From eaea9e54d1d249c592ad7f5971658e7647c3b4fb Mon Sep 17 00:00:00 2001 From: vianh Date: Sun, 1 Oct 2023 20:27:51 +0700 Subject: [PATCH 01/11] [Yurineko] New source --- .../kotatsu/parsers/site/vi/YurinekoParser.kt | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt new file mode 100644 index 00000000..2ace1e4a --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt @@ -0,0 +1,151 @@ +package org.koitharu.kotatsu.parsers.site.vi + +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.json.asIterable +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet +import org.koitharu.kotatsu.parsers.util.json.toJSONList +import org.koitharu.kotatsu.parsers.util.mapChapters +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.parseJson +import org.koitharu.kotatsu.parsers.util.parseJsonArray +import org.koitharu.kotatsu.parsers.util.requireElementById +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.tryParse +import org.koitharu.kotatsu.parsers.util.urlEncoded +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("YURINEKO", "Yurineko", "vi", ContentType.HENTAI) +class YurinekoParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.YURINEKO, 20) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("yurineko.net") + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED) + + private val apiDomain = "api.$domain" + + override suspend fun getDetails(manga: Manga): Manga { + val response = webClient.httpGet(manga.url.toAbsoluteUrl(apiDomain)).parseJson() + val df = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) + return manga.copy( + chapters = response.getJSONArray("chapters") + .toJSONList() + .mapChapters(true) { i, jo -> + val mangaId = jo.getInt("mangaID") + val chapterId = jo.getInt("id") + MangaChapter( + id = generateUid(chapterId.toLong()), + name = jo.getString("name"), + number = i + 1, + scanlator = null, + url = "/read/$mangaId/$chapterId", + uploadDate = df.tryParse(jo.getString("date")), + branch = null, + source = source, + ) + }.reversed(), + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val listUrl = when { + !query.isNullOrEmpty() -> "/search?query=${query.urlEncoded()}&page=$page" + tags.isNullOrEmpty() -> "/lastest2?page=$page" + tags.size == 1 -> "/searchType?type=tag&id=${tags.first().key}&page=$page" + else -> { + // Sort order is different when filter with multiple tags + val tagKeys = tags.joinToString(separator = ",") { it.key } + "/advancedSearch?genre=$tagKeys¬Genre=&sort=7&minChapter=1&status=0&page=$page" + } + } + val jsonResponse = webClient.httpGet(listUrl.toAbsoluteUrl(apiDomain)).parseJson() + return jsonResponse.getJSONArray("result") + .mapJSON { jo -> + val id = jo.getLong("id") + val relativeUrl = "/manga/$id" + Manga( + id = generateUid(id), + title = jo.getString("originalName"), + altTitle = jo.getStringOrNull("otherName"), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = jo.getString("thumbnail"), + tags = jo.getJSONArray("tag").mapJSONToSet { tag -> + MangaTag( + title = tag.getString("name"), + key = tag.getInt("id").toString(), + source = source, + ) + }, + state = when (jo.getInt("status")) { + 2 -> MangaState.FINISHED + 1, 3, 4 -> MangaState.ONGOING + 5, 6, 7 -> MangaState.ABANDONED + else -> null + }, + author = jo.getJSONArray("author") + .mapJSON { author -> author.getString("name") } + .joinToString { it }, + description = jo.getStringOrNull("description"), + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val jsonData = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .requireElementById("__NEXT_DATA__") + .data() + return JSONObject(jsonData).getJSONObject("props") + .getJSONObject("pageProps") + .getJSONObject("chapterData") + .getJSONArray("url") + .asIterable() + .map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + return webClient.httpGet("https://$apiDomain/tag/find?query=") + .parseJsonArray() + .mapJSONToSet { jo -> + MangaTag( + key = jo.getInt("id").toString(), + title = jo.getString("name"), + source = source, + ) + } + } +} From eda4f7521978f7b8c635610d40d63fb2d3d2cd02 Mon Sep 17 00:00:00 2001 From: vianh Date: Mon, 2 Oct 2023 21:21:58 +0700 Subject: [PATCH 02/11] [Yurineko] Fix api domain and chapters order --- .../org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt index 2ace1e4a..a7d390ad 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/YurinekoParser.kt @@ -41,7 +41,8 @@ class YurinekoParser(context: MangaLoaderContext) : PagedMangaParser(context, Ma override val sortOrders: Set get() = EnumSet.of(SortOrder.UPDATED) - private val apiDomain = "api.$domain" + private val apiDomain + get() = "api.$domain" override suspend fun getDetails(manga: Manga): Manga { val response = webClient.httpGet(manga.url.toAbsoluteUrl(apiDomain)).parseJson() @@ -62,7 +63,7 @@ class YurinekoParser(context: MangaLoaderContext) : PagedMangaParser(context, Ma branch = null, source = source, ) - }.reversed(), + }, ) } From d494bf19ee55dbe50d01a65b1b292344449ae7b7 Mon Sep 17 00:00:00 2001 From: devi Date: Mon, 2 Oct 2023 17:29:07 +0200 Subject: [PATCH 03/11] Small fix --- .../kotatsu/parsers/site/madara/all/Ero18x.kt | 3 +-- .../kotatsu/parsers/site/madara/ar/MangaStarz.kt | 2 +- .../kotatsu/parsers/site/madara/ar/MangaStarzCom.kt | 12 ------------ .../kotatsu/parsers/site/madara/tr/Jiangzaitoon.kt | 4 ++-- .../kotatsu/parsers/site/pt/UnionMangasParser.kt | 13 ++++--------- .../parsers/site/ru/grouple/AllHentaiParser.kt | 2 +- 6 files changed, 9 insertions(+), 27 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarzCom.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/all/Ero18x.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/all/Ero18x.kt index 37a796f2..fe37616a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/all/Ero18x.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/all/Ero18x.kt @@ -12,7 +12,7 @@ import java.util.Locale @MangaSourceParser("ERO18X", "Ero18x", "", ContentType.HENTAI) internal class Ero18x(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ERO18X, "ero18x.com", 10) { - override val datePattern = "MMMM d" + override val datePattern = "MM/dd" override val sourceLocale: Locale = Locale.ENGLISH override val withoutAjax = true @@ -23,7 +23,6 @@ internal class Ero18x(context: MangaLoaderContext) : val href = a.attrAsRelativeUrl("href") val link = href + stylepage val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() - val name = a.selectFirst("p")?.text() ?: a.ownText() MangaChapter( id = generateUid(href), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarz.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarz.kt index 152976ff..8104bddd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarz.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarz.kt @@ -7,6 +7,6 @@ import org.koitharu.kotatsu.parsers.site.madara.MadaraParser @MangaSourceParser("MANGASTARZ", "Manga Starz", "ar") internal class MangaStarz(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGASTARZ, "mangastarz.com", pageSize = 10) { + MadaraParser(context, MangaSource.MANGASTARZ, "mangastarz.org", pageSize = 10) { override val datePattern = "d MMMM، yyyy" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarzCom.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarzCom.kt deleted file mode 100644 index 53e93022..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaStarzCom.kt +++ /dev/null @@ -1,12 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.madara.ar - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.site.madara.MadaraParser - -@MangaSourceParser("MANGASTARZCOM", "Manga Starz .Com", "ar") -internal class MangaStarzCom(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGASTARZCOM, "mangastarz.com", 10) { - override val datePattern = "d MMMM، yyyy" -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/Jiangzaitoon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/Jiangzaitoon.kt index 3ad8c1fe..985f1fc3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/Jiangzaitoon.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/Jiangzaitoon.kt @@ -8,7 +8,7 @@ import org.koitharu.kotatsu.parsers.site.madara.MadaraParser @MangaSourceParser("JIANGZAITOON", "Jiangzaitoon", "tr", ContentType.HENTAI) internal class Jiangzaitoon(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.JIANGZAITOON, "jiangzaitoon.co") { + MadaraParser(context, MangaSource.JIANGZAITOON, "jiangzaitoon.cc") { override val postreq = true - override val datePattern = "dd MMMM yyyy" + override val datePattern = "d MMMM yyyy" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/UnionMangasParser.kt index 8454cd93..c9f2e326 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/UnionMangasParser.kt @@ -20,7 +20,7 @@ class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, SortOrder.POPULARITY, ) - override val configKeyDomain = ConfigKey.Domain("guimah.com") + override val configKeyDomain = ConfigKey.Domain("unionmangasbr.top", "guimah.com") override suspend fun getListPage( page: Int, @@ -47,7 +47,7 @@ class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, ).addPathSegment(page.toString()) val doc = webClient.httpGet(url.build()).parseHtml() val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") - return root.select(".lista-perfil-mangas-novos").map { div -> + return root.select(".lista-mangas").map { div -> val a = div.selectFirstOrThrow("a") val img = div.selectFirstOrThrow("img") val href = a.attrAsRelativeUrl("href") @@ -71,11 +71,7 @@ class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getDetails(manga: Manga): Manga { val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = if (doc.selectFirst(".perfil-d-manga") == null) { - doc.selectFirstOrThrow(".perfil-p-manga") - } else { - doc.selectFirstOrThrow(".perfil-d-manga") - } + val root = doc.selectFirstOrThrow(".perfil-d-manga, .perfil-p-manga, .manga-pagina") val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) return manga.copy( rating = root.select("h2") @@ -117,8 +113,7 @@ class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getPages(chapter: MangaChapter): List { val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("article") - return root.selectOrThrow("img[pag]").mapNotNull { img -> + return doc.body().selectOrThrow("img[pag]").mapNotNull { img -> val href = img.attrAsRelativeUrl("src") if (href.startsWith("/images/banner")) { return@mapNotNull null diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt index b487ad3b..c6cd0de9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt @@ -19,7 +19,7 @@ internal class AllHentaiParser( context: MangaLoaderContext, ) : GroupleParser(context, MangaSource.ALLHENTAI, 1) { - override val configKeyDomain = ConfigKey.Domain("z.allhen.online", "2023.allhen.online") + override val configKeyDomain = ConfigKey.Domain("24.allhen.online", "z.allhen.online", "2023.allhen.online") override val defaultIsNsfw = true override val authUrl: String From c09d484a14634b3f6ada204db9c19ea2bde58f79 Mon Sep 17 00:00:00 2001 From: Nikita Strygin Date: Tue, 3 Oct 2023 09:14:17 +0300 Subject: [PATCH 04/11] Add Line Webtoons sources (webtoons.com) It uses the API of mobile app --- .../parsers/site/en/LineWebtoonsParser.kt | 408 ++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt new file mode 100644 index 00000000..7c80eb57 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt @@ -0,0 +1,408 @@ +package org.koitharu.kotatsu.parsers.site.en + +import okhttp3.Headers +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed +import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet +import org.koitharu.kotatsu.parsers.util.json.toJSONList +import java.net.URI +import java.net.URLEncoder +import java.util.* +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec + +@MangaSourceParser("LINEWEBTOONS", "Line Webtoons", "en", type = ContentType.MANGA) +class LineWebtoonsParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.LINEWEBTOONS) { + private val signer = WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") + + override val configKeyDomain + get() = ConfigKey.Domain("webtoons.com") + private val configKeyApiDomain + get() = ConfigKey.Domain("global.apis.naver.com") + private val configKeyStaticDomain + get() = ConfigKey.Domain("webtoon-phinf.pstatic.net") + + private val apiDomain + get() = config[configKeyApiDomain] + private val staticDomain + get() = config[configKeyStaticDomain] + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + // doesn't actually sort by rating, but by likes + // this should be fine though + SortOrder.RATING, + ) + override val headers: Headers + get() = Headers.Builder() + .add("User-Agent", "nApps (Android 12;; linewebtoon; 3.1.0)") + .build() + + override suspend fun getPageUrl(page: MangaPage): String { + return page.url + } + + private suspend fun getChapters(titleNo: Long): List { + val firstResult = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30") + + val totalEpisodeCount = firstResult + .getJSONObject("episodeList") + .getInt("totalServiceEpisodeCount") + + val episodes = firstResult + .getJSONObject("episodeList") + .getJSONArray("episode") + .toJSONList() + .toMutableList() + + while (episodes.count() < totalEpisodeCount) { + val page = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30") + .getJSONObject("episodeList") + .getJSONArray("episode") + .toJSONList() + + episodes.addAll(page) + } + + return episodes.mapIndexed { i, jo -> + MangaChapter( + id = generateUid("$titleNo-$i"), + name = jo.getString("episodeTitle"), + number = jo.getInt("episodeSeq"), + url = "$titleNo-${jo.getString("episodeNo")}", + uploadDate = jo.getLong("modifyYmdt"), + // do we want to use it for anything? + branch = null, + scanlator = null, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun getDetails(manga: Manga): Manga { + val titleNo = manga.url.toLong() + + return makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}") + .getJSONObject("titleInfo") + .let { jo -> + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = "$titleNo", + publicUrl = "https://${domain}/en/canvas/a/list?title_no=${titleNo}", + rating = jo.getDouble("starScoreAverage").toFloat() / 10f, + isNsfw = jo.getBoolean("ageGradeNotice"), + coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", + largeCoverUrl = if (jo.has("thumbnailVertical")) { + "https://$staticDomain${jo.getString("thumbnailVertical")}" + } else { null }, + tags = setOf(parseTag(jo.getJSONObject("genreInfo"))), + author = jo.getString("writingAuthorName"), + description = jo.getString("synopsis"), + // I don't think the API provides this info + state = null, + chapters = getChapters(titleNo), + source = source, + ) + } + } + + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val genre = tags.oneOrThrowIfMany()?.key ?: "ALL" + + val sortOrderStr = when (sortOrder) { + SortOrder.UPDATED -> "UPDATE" + SortOrder.POPULARITY -> "READ_COUNT" + SortOrder.RATING -> "LIKEIT" + else -> { + throw Exception("Unreachable"); + } + } + + val manga = if (query != null) { + if (!tags.isNullOrEmpty()) { + throw IllegalArgumentException("This source does not support search with tags") + } + + makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset+1}&pageSize=20") + .getJSONObject("challengeSearch") + .getJSONArray("titleList") + .mapJSON { jo -> + val titleNo = jo.getLong("titleNo") + + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = "$titleNo", + publicUrl = "https://${domain}/en/canvas/a/list?title_no=${titleNo}", + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", + largeCoverUrl = null, + tags = setOf(), + author = jo.getString("writingAuthorName"), + description = null, + state = null, + source = source, + ) + } + } else { + val result = makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=${genre}&sortOrder=${sortOrderStr}&startIndex=${offset+1}&pageSize=20") + + val genres = result.getJSONObject("genreList") + .getJSONArray("challengeGenres") + .mapJSON { jo -> parseTag(jo) } + .associateBy { tag -> tag.key } + + result + .getJSONObject("titleList") + .getJSONArray("titles") + .mapJSON { jo -> + val titleNo = jo.getLong("titleNo") + + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = "$titleNo", + publicUrl = "https://${domain}/en/canvas/a/list?title_no=${titleNo}", + rating = jo.getDouble("starScoreAverage").toFloat() / 10f, + isNsfw = jo.getBoolean("ageGradeNotice"), + coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", + largeCoverUrl = if (jo.has("thumbnailVertical")) { + "https://$staticDomain${jo.getString("thumbnailVertical")}" + } else { null }, + tags = setOf(genres[jo.getString("representGenre")]!!), + author = jo.getString("writingAuthorName"), + description = jo.getString("synopsis"), + // I don't think the API provides this info + state = null, + source = source, + ) + } + } + + return manga + } + + override suspend fun getPages(chapter: MangaChapter): List { + val (titleNo, episodeNo) = chapter.url.splitTwoParts('-')!! + + return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=${titleNo}&episodeNo=${episodeNo}") + .getJSONObject("episodeInfo") + .getJSONArray("imageInfo") + .mapJSONIndexed() { i, jo -> + MangaPage( + id = generateUid("$titleNo-$episodeNo-$i"), + url = "https://$staticDomain${jo.getString("url")}", + preview = null, + source = source, + ) + } + } + + private fun parseTag(jo: JSONObject): MangaTag { + return MangaTag( + title = jo.getString("name"), + key = jo.getString("code"), + source = source, + ) + } + + override suspend fun getTags(): Set { + return makeRequest("/lineWebtoon/webtoon/challengeGenreList.json") + .getJSONObject("genreList") + .getJSONArray("challengeGenres") + .mapJSONToSet { jo -> parseTag(jo) } + } + + private suspend fun makeRequest(url: String): JSONObject { + val resp = webClient.httpGet(finalizeUrl(url)) + val message = resp.parseJson().getJSONObject("message") + if (resp.isSuccessful) { + return message.getJSONObject("result") + } else { + // TODO: handle 404 and some other error codes + val code = message.getInt("code") + val errorMessage = message.getString("message") + throw ParseException("Api error (code=$code): $errorMessage", url) + } + } + + private fun finalizeUrl(url: String): String { + val urlWithHost = "https://${apiDomain}$url" + val uri = URI(urlWithHost) + val hasVersion = (uri.rawQuery ?: "").split("&").any { it.startsWith("v=") } + val hasQuery = uri.rawQuery != null + val language = "en" + + val urlWithParams = urlWithHost + if (hasQuery) { + "&" + } else { + "?" + } + "serviceZone=GLOBAL&" + if (!hasVersion) { + "v=1" + } else { "" } + "&language=${language}&locale=${language}&platform=APP_ANDROID" + + val result = signer.makeEncryptUrl(urlWithParams); + + return result + } +} + + +private class WebtoonsUrlSigner(val secret: String) { + private val mac = Mac.getInstance("HmacSHA1").apply { + this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1")) + } + + private fun getMessage(url: String, msgpad: String): String { + return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad + } + + private fun getMessageDigest(s: String): String { + var signedMessage: ByteArray + synchronized(mac) { signedMessage = mac.doFinal(s.toByteArray()) } + + // we don't use the context.encodeBase64 here because it adds newlines and doesn't add padding + // we, however, need padding and no newlines + return encodeBase64(signedMessage) + } + + + fun makeEncryptUrl(s: String): String { + return makeEncryptUrlCore( + s, + java.lang.String.valueOf(Calendar.getInstance().timeInMillis), + ) + } + + private fun makeEncryptUrlCore(url: String, msgpad: String): String { + val digest = URLEncoder.encode(getMessageDigest(getMessage(url, msgpad)), "utf-8") + return url + if (url.contains("?")) { + "&" + } else { + "?" + } + "msgpad=${msgpad}&md=${digest}" + } +} + + +private val INT_TO_BASE64: CharArray = charArrayOf( + 'A', + 'B', + 'C', + 'D', + 'E', + 'F', + 'G', + 'H', + 'I', + 'J', + 'K', + 'L', + 'M', + 'N', + 'O', + 'P', + 'Q', + 'R', + 'S', + 'T', + 'U', + 'V', + 'W', + 'X', + 'Y', + 'Z', + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + 'l', + 'm', + 'n', + 'o', + 'p', + 'q', + 'r', + 's', + 't', + 'u', + 'v', + 'w', + 'x', + 'y', + 'z', + '0', + '1', + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + '+', + '/', +) + +private fun encodeBase64(arr: ByteArray): String { + val groupsCount = arr.size / 3 + val extraCount = arr.size - groupsCount * 3 + + val sb = StringBuffer((arr.size + 2) / 3 * 4) + val code = INT_TO_BASE64 + var groupIndex = 0 + var position = 0 + while (groupIndex < groupsCount) { + val v4 = arr[position].toInt() and 0xFF + val v5 = arr[position + 1].toInt() and 0xFF + val v6 = arr[position + 2].toInt() and 0xFF + sb.append(code[v4 shr 2]) + sb.append(code[v4 shl 4 and 0x3F or (v5 shr 4)]) + sb.append(code[v5 shl 2 and 0x3F or (v6 shr 6)]) + sb.append(code[v6 and 0x3F]) + ++groupIndex + position += 3 + } + + if (extraCount != 0) { + val v1 = arr[position].toInt() and 0xFF + sb.append(code[v1 shr 2]) + if (extraCount == 1) { + sb.append(code[v1 shl 4 and 0x3F]) + sb.append("==") + } else { + val v2 = arr[position + 1].toInt() and 0xFF + sb.append(code[v1 shl 4 and 0x3F or (v2 shr 4)]) + sb.append(code[v2 shl 2 and 0x3F]) + sb.append('=') + } + } + return sb.toString() +} From 908cc2261936a6148b7e32d5e1f4c2bbcc6487a6 Mon Sep 17 00:00:00 2001 From: devi Date: Tue, 3 Oct 2023 16:55:32 +0200 Subject: [PATCH 05/11] Minor fix --- .../kotatsu/parsers/site/en/Mangaowl.kt | 16 ++++------- .../site/mangareader/ar/PotatoManga.kt | 27 +++++++++++++++++++ .../parsers/site/mangareader/en/SuryaScans.kt | 5 +--- .../site/mangareader/id/SekaikomikParser.kt | 2 +- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Mangaowl.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Mangaowl.kt index 3930970c..7df8fd72 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Mangaowl.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Mangaowl.kt @@ -66,7 +66,6 @@ internal class Mangaowl(context: MangaLoaderContext) : } else -> { - append("/8-comics") append("?page=") append(page.toString()) @@ -129,29 +128,25 @@ internal class Mangaowl(context: MangaLoaderContext) : } private fun getChapters(mangaUrl: String, doc: Document): List { - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", sourceLocale) - val script = doc.selectFirstOrThrow("script:containsData(chapters:)") val json = script.data().substringAfter("chapters:[").substringBeforeLast(')').substringBefore("],latest_chapter:") .split("},") val slug = mangaUrl.substringAfterLast("/") - val chapter = ArrayList() - val num = 0 - json.map { t -> + var lastIndexed = 0 + json.mapIndexed { i, t -> if (t.contains("Chapter")) { val id = t.substringAfter("id:").substringBefore(",created_at") val url = "/reading/$slug/$id" - val date = t.substringAfter("created_at:\"").substringBefore("\"") val name = t.substringAfter("name:\"").substringBefore("\"") chapter.add( MangaChapter( id = generateUid(url), name = name, - number = num + 1, + number = i + 1, url = url, uploadDate = dateFormat.tryParse(date), source = source, @@ -159,6 +154,7 @@ internal class Mangaowl(context: MangaLoaderContext) : branch = null, ), ) + lastIndexed = i } } @@ -171,7 +167,7 @@ internal class Mangaowl(context: MangaLoaderContext) : MangaChapter( id = generateUid(url), name = name, - number = num + 1, + number = lastIndexed + 1, url = url, uploadDate = dateFormat.tryParse(date), source = source, @@ -179,13 +175,11 @@ internal class Mangaowl(context: MangaLoaderContext) : branch = null, ), ) - return chapter } override suspend fun getPages(chapter: MangaChapter): List { val id = chapter.url.substringAfterLast("/") - val json = webClient.httpGet("https://api.mangaowl.to/v1/chapters/$id/images?page_size=100").parseJson() return json.getJSONArray("results").mapJSON { jo -> MangaPage( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/ar/PotatoManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/ar/PotatoManga.kt index 9e45a7ec..f5ac3237 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/ar/PotatoManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/ar/PotatoManga.kt @@ -1,12 +1,39 @@ package org.koitharu.kotatsu.parsers.site.mangareader.ar +import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser +import org.koitharu.kotatsu.parsers.util.* +import java.util.ArrayList @MangaSourceParser("POTATOMANGA", "Potato Manga", "ar") internal class PotatoManga(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.POTATOMANGA, "potatomanga.xyz", pageSize = 30, searchPageSize = 10) { override val listUrl = "/series" + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val docs = webClient.httpGet(chapterUrl).parseHtml() + val script = docs.selectFirstOrThrow(selectTestScript) + val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i).replace("\\", "")), + url = images.getString(i).replace("\\", ""), + preview = null, + source = source, + ), + ) + } + return pages + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/SuryaScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/SuryaScans.kt index e7945ac2..73d56850 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/SuryaScans.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/SuryaScans.kt @@ -7,7 +7,4 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser @MangaSourceParser("SURYASCANS", "Surya Scans", "en") internal class SuryaScans(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.SURYASCANS, "suryascans.com", pageSize = 5, searchPageSize = 5) { - - override val datePattern = "MMM d, yyyy" -} + MangaReaderParser(context, MangaSource.SURYASCANS, "suryareader.com", pageSize = 5, searchPageSize = 5) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/SekaikomikParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/SekaikomikParser.kt index 66649556..c8ed39e3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/SekaikomikParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/SekaikomikParser.kt @@ -7,4 +7,4 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser @MangaSourceParser("SEKAIKOMIK", "Sekai Komik", "id") internal class SekaikomikParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.SEKAIKOMIK, "sekaikomik.pro", pageSize = 20, searchPageSize = 100) + MangaReaderParser(context, MangaSource.SEKAIKOMIK, "sekaikomik.bio", pageSize = 20, searchPageSize = 100) From 543e3da1949c0b3729d20b338373d216789b31d6 Mon Sep 17 00:00:00 2001 From: devi Date: Tue, 3 Oct 2023 18:25:43 +0200 Subject: [PATCH 06/11] Add sources --- .../parsers/site/fmreader/FmreaderParser.kt | 52 +++++++---- .../parsers/site/fmreader/es/OlimpoScans.kt | 89 +++++++++++++++++++ .../parsers/site/heancms/pt/ReaperScansPt.kt | 5 +- .../parsers/site/madara/ar/MangaLekNet.kt | 10 +++ .../parsers/site/madara/ar/MangaLionz.kt | 2 +- .../parsers/site/madara/es/GanzoScan.kt | 12 +++ .../parsers/site/madara/pt/FlowerManga.kt | 12 +++ .../parsers/site/madara/pt/KakuseiProject.kt | 10 +++ .../parsers/site/madara/pt/SweetScan.kt | 12 +++ .../site/mangareader/en/CypherScans.kt | 10 +++ .../site/mangareader/id/WarungKomik.kt | 10 +++ 11 files changed, 204 insertions(+), 20 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/es/OlimpoScans.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLekNet.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/GanzoScan.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/FlowerManga.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/KakuseiProject.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/SweetScan.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/CypherScans.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/WarungKomik.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt index a70ff36c..457e834d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt @@ -29,6 +29,7 @@ internal abstract class FmreaderParser( protected open val listeurl = "/manga-list.html" protected open val datePattern = "MMMM d, yyyy" + protected open val tagPrefix = "manga-list-genre-" init { paginator.firstPage = 1 @@ -39,11 +40,20 @@ internal abstract class FmreaderParser( protected val ongoing: Set = setOf( "On going", "Incomplete", + "En curso", ) @JvmField protected val finished: Set = setOf( "Completed", + "Completado", + ) + + @JvmField + protected val abandoned: Set = hashSetOf( + "Canceled", + "Cancelled", + "Drop", ) override suspend fun getListPage( @@ -106,7 +116,7 @@ internal abstract class FmreaderParser( override suspend fun getTags(): Set { val doc = webClient.httpGet("https://$domain/$listeurl").parseHtml() return doc.select(selectBodyTag).mapNotNullToSet { a -> - val href = a.attr("href").substringAfter("manga-list-genre-").substringBeforeLast(".html") + val href = a.attr("href").substringAfter(tagPrefix).substringBeforeLast(".html") MangaTag( key = href, title = a.text(), @@ -131,6 +141,7 @@ internal abstract class FmreaderParser( when (it.text()) { in ongoing -> MangaState.ONGOING in finished -> MangaState.FINISHED + in abandoned -> MangaState.ABANDONED else -> null } } @@ -140,7 +151,7 @@ internal abstract class FmreaderParser( manga.copy( tags = doc.body().select(selectTag).mapNotNullToSet { a -> MangaTag( - key = a.attr("href").substringAfter("manga-list-genre-").substringBeforeLast(".html"), + key = a.attr("href").substringAfter(tagPrefix).substringBeforeLast(".html"), title = a.text().toTitleCase(), source = source, ) @@ -201,6 +212,7 @@ internal abstract class FmreaderParser( val d = date?.lowercase() ?: return 0 return when { d.endsWith(" ago") || + d.endsWith(" atrás") || // short Hours d.endsWith(" h") || // short Day @@ -240,40 +252,44 @@ internal abstract class FmreaderParser( val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 val cal = Calendar.getInstance() return when { - WordSet( - "day", - "days", - ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + WordSet("min", "minute", "minutes", "minuto", "minutos").anyWordIn(date) -> cal.apply { + add( + Calendar.MINUTE, + -number, + ) + }.timeInMillis - WordSet("hour", "hours", "h").anyWordIn(date) -> cal.apply { + WordSet("hour", "hours", "hora", "horas", "h").anyWordIn(date) -> cal.apply { add( Calendar.HOUR, -number, ) }.timeInMillis - WordSet( - "min", - "minute", - "minutes", - ).anyWordIn(date) -> cal.apply { + WordSet("day", "days", "día", "dia").anyWordIn(date) -> cal.apply { add( - Calendar.MINUTE, + Calendar.DAY_OF_MONTH, -number, ) }.timeInMillis - WordSet("second").anyWordIn(date) -> cal.apply { + WordSet("week", "weeks", "semana", "semanas").anyWordIn(date) -> cal.apply { add( - Calendar.SECOND, + Calendar.WEEK_OF_YEAR, -number, ) }.timeInMillis - WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + WordSet("month", "months", "mes", "meses").anyWordIn(date) -> cal.apply { + add( + Calendar.MONTH, + -number, + ) + }.timeInMillis + + WordSet("year", "año", "años").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis else -> 0 } } - } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/es/OlimpoScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/es/OlimpoScans.kt new file mode 100644 index 00000000..78e171a7 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/es/OlimpoScans.kt @@ -0,0 +1,89 @@ +package org.koitharu.kotatsu.parsers.site.fmreader.es + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser +import org.koitharu.kotatsu.parsers.util.* + +@MangaSourceParser("OLIMPOSCANS", "Olimpo Scans", "es") +internal class OlimpoScans(context: MangaLoaderContext) : + FmreaderParser(context, MangaSource.OLIMPOSCANS, "olimposcans.com") { + + override val selectState = "ul.manga-info li:contains(Estado) a" + override val selectAlt = "ul.manga-info li:contains(Otros nombres)" + override val selectTag = "ul.manga-info li:contains(Género) a" + override val tagPrefix = "lista-de-comics-genero-" + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val tag = tags.oneOrThrowIfMany() + val url = buildString { + append("https://") + append(domain) + append(listeurl) + append("?page=") + append(page.toString()) + when { + !query.isNullOrEmpty() -> { + append("&name=") + append(query.urlEncoded()) + } + + !tags.isNullOrEmpty() -> { + append("&genre=") + append(tag?.key.orEmpty()) + } + } + append("&sort=") + when (sortOrder) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("last_update") + SortOrder.ALPHABETICAL -> append("name") + else -> append("last_update") + } + } + val doc = webClient.httpGet(url).parseHtml() + val lastPage = + doc.selectLast(".pagination a")?.attr("href")?.substringAfterLast("page=")?.substringBeforeLast("&artist") + ?.toInt() ?: 1 + if (lastPage < page) { + return emptyList() + } + return doc.select("div.thumb-item-flow").map { div -> + val href = "/" + div.selectFirstOrThrow("div.series-title a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("data-bg").toAbsoluteUrl(domain), + title = div.selectFirstOrThrow("div.series-title").text().orEmpty(), + altTitle = null, + rating = RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = null, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = ("/" + chapter.url).toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select(selectPage).map { img -> + val url = ("/proxy.php?link=" + img.src()).toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/pt/ReaperScansPt.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/pt/ReaperScansPt.kt index 7ecc4fea..e253fa15 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/pt/ReaperScansPt.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/pt/ReaperScansPt.kt @@ -2,9 +2,12 @@ package org.koitharu.kotatsu.parsers.site.heancms.pt import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.site.heancms.HeanCms @MangaSourceParser("REAPERSCANSPT", "Reaper Scans", "pt") internal class ReaperScansPt(context: MangaLoaderContext) : - HeanCms(context, MangaSource.REAPERSCANSPT, "reaperscans.net") + HeanCms(context, MangaSource.REAPERSCANSPT, "reaperscans.net") { + override val configKeyDomain = ConfigKey.Domain("reaperscans.net", "reaperbr.online") +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLekNet.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLekNet.kt new file mode 100644 index 00000000..011889d5 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLekNet.kt @@ -0,0 +1,10 @@ +package org.koitharu.kotatsu.parsers.site.madara.ar + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madara.MadaraParser + +@MangaSourceParser("MANGALEK_NET", "Manga Lek .Net", "ar") +internal class MangaLekNet(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGALEK_NET, "manga-lek.net", pageSize = 10) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLionz.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLionz.kt index 0c1be10b..b8b581c1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLionz.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/MangaLionz.kt @@ -7,4 +7,4 @@ import org.koitharu.kotatsu.parsers.site.madara.MadaraParser @MangaSourceParser("MANGALIONZ", "Manga Lionz", "ar") internal class MangaLionz(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGALIONZ, "mangalionz.com", pageSize = 10) + MadaraParser(context, MangaSource.MANGALIONZ, "mangalionz.org", pageSize = 10) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/GanzoScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/GanzoScan.kt new file mode 100644 index 00000000..edac9167 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/GanzoScan.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madara.es + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madara.MadaraParser + +@MangaSourceParser("GANZOSCAN", "Ganzo Scan", "es") +internal class GanzoScan(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.GANZOSCAN, "ganzoscan.com") { + override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/FlowerManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/FlowerManga.kt new file mode 100644 index 00000000..944cd396 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/FlowerManga.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madara.pt + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madara.MadaraParser + +@MangaSourceParser("FLOWERMANGA", "Flower Manga", "pt") +internal class FlowerManga(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.FLOWERMANGA, "flowermanga.com", 24) { + override val datePattern = "d MMMM yyyy" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/KakuseiProject.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/KakuseiProject.kt new file mode 100644 index 00000000..793b7483 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/KakuseiProject.kt @@ -0,0 +1,10 @@ +package org.koitharu.kotatsu.parsers.site.madara.pt + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madara.MadaraParser + +@MangaSourceParser("KAKUSEIPROJECT", "Kakusei Project", "pt") +internal class KakuseiProject(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.KAKUSEIPROJECT, "kakuseiproject.com.br", 10) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/SweetScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/SweetScan.kt new file mode 100644 index 00000000..1a962d32 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/SweetScan.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madara.pt + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madara.MadaraParser + +@MangaSourceParser("SWEETSCAN", "Sweet Scan", "pt") +internal class SweetScan(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.SWEETSCAN, "sweetscan.net") { + override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/CypherScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/CypherScans.kt new file mode 100644 index 00000000..e1b105ed --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/CypherScans.kt @@ -0,0 +1,10 @@ +package org.koitharu.kotatsu.parsers.site.mangareader.en + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser + +@MangaSourceParser("CYPHERSCANS", "Cypher Scans", "en") +internal class CypherScans(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.CYPHERSCANS, "cypherscans.xyz", pageSize = 20, searchPageSize = 10) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/WarungKomik.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/WarungKomik.kt new file mode 100644 index 00000000..2a1c0e83 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/WarungKomik.kt @@ -0,0 +1,10 @@ +package org.koitharu.kotatsu.parsers.site.mangareader.id + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser + +@MangaSourceParser("WARUNGKOMIK", "Warung Komik", "id") +internal class WarungKomik(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.WARUNGKOMIK, "warungkomik.com", pageSize = 20, searchPageSize = 10) From 887fc691d1f3b5766848d1bbe8798c2829bd46ae Mon Sep 17 00:00:00 2001 From: Nikita Strygin Date: Wed, 4 Oct 2023 12:03:21 +0300 Subject: [PATCH 07/11] Add support for more languages in Line Webtoons parser Also change the default sorting to POPULARITY & clean up --- .../parsers/site/en/LineWebtoonsParser.kt | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt index 7c80eb57..d5dbb115 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt @@ -19,8 +19,7 @@ import java.util.* import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec -@MangaSourceParser("LINEWEBTOONS", "Line Webtoons", "en", type = ContentType.MANGA) -class LineWebtoonsParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.LINEWEBTOONS) { +internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: MangaSource) : MangaParser(context, source) { private val signer = WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") override val configKeyDomain @@ -36,11 +35,11 @@ class LineWebtoonsParser(context: MangaLoaderContext) : MangaParser(context, Man get() = config[configKeyStaticDomain] override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, SortOrder.POPULARITY, // doesn't actually sort by rating, but by likes // this should be fine though SortOrder.RATING, + SortOrder.UPDATED, ) override val headers: Headers get() = Headers.Builder() @@ -130,7 +129,7 @@ class LineWebtoonsParser(context: MangaLoaderContext) : MangaParser(context, Man SortOrder.POPULARITY -> "READ_COUNT" SortOrder.RATING -> "LIKEIT" else -> { - throw Exception("Unreachable"); + throw Exception("Unreachable") } } @@ -250,7 +249,12 @@ class LineWebtoonsParser(context: MangaLoaderContext) : MangaParser(context, Man val uri = URI(urlWithHost) val hasVersion = (uri.rawQuery ?: "").split("&").any { it.startsWith("v=") } val hasQuery = uri.rawQuery != null - val language = "en" + // some language tags do not map perfectly to the ones used by the API + val language = when (val tag = sourceLocale.toLanguageTag()) { + "in" -> "id" + "zh" -> "zh-hant" + else -> tag + } val urlWithParams = urlWithHost + if (hasQuery) { "&" @@ -260,10 +264,24 @@ class LineWebtoonsParser(context: MangaLoaderContext) : MangaParser(context, Man "v=1" } else { "" } + "&language=${language}&locale=${language}&platform=APP_ANDROID" - val result = signer.makeEncryptUrl(urlWithParams); - - return result + return signer.makeEncryptUrl(urlWithParams) } + + @MangaSourceParser("LINEWEBTOONS_EN", "Line Webtoons English", "en", type = ContentType.MANGA) + class English(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_EN) + @MangaSourceParser("LINEWEBTOONS_ZH", "Line Webtoons Chinese", "zh", type = ContentType.MANGA) + class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ZH) + @MangaSourceParser("LINEWEBTOONS_TH", "Line Webtoons Thai", "th", type = ContentType.MANGA) + class Thai(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_TH) + @MangaSourceParser("LINEWEBTOONS_ID", "Line Webtoons Indonesian", "id", type = ContentType.MANGA) + class Indonesian(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ID) + @MangaSourceParser("LINEWEBTOONS_ES", "Line Webtoons Spanish", "es", type = ContentType.MANGA) + class Spanish(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ES) + @MangaSourceParser("LINEWEBTOONS_FR", "Line Webtoons French", "fr", type = ContentType.MANGA) + class French(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_FR) + @MangaSourceParser("LINEWEBTOONS_DE", "Line Webtoons German", "de", type = ContentType.MANGA) + class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_DE) + } From 3d7e62d2fe52accbd833a3360b3b77daae0ba0a2 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 4 Oct 2023 13:15:12 +0300 Subject: [PATCH 08/11] [MintManga] Update domain --- .../koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt index 63c75a29..f8aed227 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt @@ -11,9 +11,9 @@ internal class MintMangaParser( ) : GroupleParser(context, MangaSource.MINTMANGA, 2) { override val configKeyDomain = ConfigKey.Domain( + "23.mintmanga.live", "mintmanga.live", "mintmanga.com", "m.mintmanga.live", ) - } From 753c27a90ce153ee308191147c674c9bf5ca9e99 Mon Sep 17 00:00:00 2001 From: Nikita Strygin Date: Wed, 4 Oct 2023 15:19:27 +0300 Subject: [PATCH 09/11] Clean up code after review - Do not use ConfigKey.Domain for supplementary domains - use IllegalArgumentException - use "$var" instead of "${var}" where possible - fix the language in `publicUrl` - use `?` to handle `largeCoverUrl` - use `requireNotNull` instead of `!!` --- .../parsers/site/en/LineWebtoonsParser.kt | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt index d5dbb115..b80c28da 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt @@ -1,6 +1,7 @@ package org.koitharu.kotatsu.parsers.site.en import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser @@ -9,11 +10,11 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet import org.koitharu.kotatsu.parsers.util.json.toJSONList -import java.net.URI import java.net.URLEncoder import java.util.* import javax.crypto.Mac @@ -22,17 +23,17 @@ import javax.crypto.spec.SecretKeySpec internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: MangaSource) : MangaParser(context, source) { private val signer = WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") + // we don't __really__ support changing this domain because: + // 1. I don't think other websites have this exact API + // 2. most communication is done with other domains (hosting API and static content), which are not configurable + // 3. we rely on the HTTP client setting the referer header to webtoons.com + // + // This effectively means that changing the domain will break the source. Yikes override val configKeyDomain get() = ConfigKey.Domain("webtoons.com") - private val configKeyApiDomain - get() = ConfigKey.Domain("global.apis.naver.com") - private val configKeyStaticDomain - get() = ConfigKey.Domain("webtoon-phinf.pstatic.net") - private val apiDomain - get() = config[configKeyApiDomain] - private val staticDomain - get() = config[configKeyStaticDomain] + private val apiDomain = "global.apis.naver.com" + private val staticDomain = "webtoon-phinf.pstatic.net" override val sortOrders: Set = EnumSet.of( SortOrder.POPULARITY, @@ -50,6 +51,14 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: return page.url } + // some language tags do not map perfectly to the ones used by the API + private val languageCode: String + get() = when (val tag = sourceLocale.toLanguageTag()) { + "in" -> "id" + "zh" -> "zh-hant" + else -> tag + } + private suspend fun getChapters(titleNo: Long): List { val firstResult = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30") @@ -79,7 +88,6 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: number = jo.getInt("episodeSeq"), url = "$titleNo-${jo.getString("episodeNo")}", uploadDate = jo.getLong("modifyYmdt"), - // do we want to use it for anything? branch = null, scanlator = null, source = source, @@ -98,7 +106,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: title = jo.getString("title"), altTitle = null, url = "$titleNo", - publicUrl = "https://${domain}/en/canvas/a/list?title_no=${titleNo}", + publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=${titleNo}", rating = jo.getDouble("starScoreAverage").toFloat() / 10f, isNsfw = jo.getBoolean("ageGradeNotice"), coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", @@ -129,7 +137,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: SortOrder.POPULARITY -> "READ_COUNT" SortOrder.RATING -> "LIKEIT" else -> { - throw Exception("Unreachable") + throw IllegalArgumentException("Unsupported sort order: $sortOrder") } } @@ -149,7 +157,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: title = jo.getString("title"), altTitle = null, url = "$titleNo", - publicUrl = "https://${domain}/en/canvas/a/list?title_no=${titleNo}", + publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo", rating = RATING_UNKNOWN, isNsfw = false, coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", @@ -162,7 +170,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: ) } } else { - val result = makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=${genre}&sortOrder=${sortOrderStr}&startIndex=${offset+1}&pageSize=20") + val result = makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset+1}&pageSize=20") val genres = result.getJSONObject("genreList") .getJSONArray("challengeGenres") @@ -180,13 +188,11 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: title = jo.getString("title"), altTitle = null, url = "$titleNo", - publicUrl = "https://${domain}/en/canvas/a/list?title_no=${titleNo}", + publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo", rating = jo.getDouble("starScoreAverage").toFloat() / 10f, isNsfw = jo.getBoolean("ageGradeNotice"), coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", - largeCoverUrl = if (jo.has("thumbnailVertical")) { - "https://$staticDomain${jo.getString("thumbnailVertical")}" - } else { null }, + largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), tags = setOf(genres[jo.getString("representGenre")]!!), author = jo.getString("writingAuthorName"), description = jo.getString("synopsis"), @@ -201,9 +207,9 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: } override suspend fun getPages(chapter: MangaChapter): List { - val (titleNo, episodeNo) = chapter.url.splitTwoParts('-')!! + val (titleNo, episodeNo) = requireNotNull(chapter.url.splitTwoParts('-')) - return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=${titleNo}&episodeNo=${episodeNo}") + return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=$titleNo&episodeNo=$episodeNo") .getJSONObject("episodeInfo") .getJSONArray("imageInfo") .mapJSONIndexed() { i, jo -> @@ -245,24 +251,18 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: } private fun finalizeUrl(url: String): String { - val urlWithHost = "https://${apiDomain}$url" - val uri = URI(urlWithHost) - val hasVersion = (uri.rawQuery ?: "").split("&").any { it.startsWith("v=") } - val hasQuery = uri.rawQuery != null - // some language tags do not map perfectly to the ones used by the API - val language = when (val tag = sourceLocale.toLanguageTag()) { - "in" -> "id" - "zh" -> "zh-hant" - else -> tag - } + val absoluteUrl = url.toAbsoluteUrl(apiDomain) + val parsedUrl = absoluteUrl.toHttpUrl() + val hasVersion = parsedUrl.queryParameter("v") != null + val hasQuery = parsedUrl.query != null - val urlWithParams = urlWithHost + if (hasQuery) { + val urlWithParams = absoluteUrl + if (hasQuery) { "&" } else { "?" } + "serviceZone=GLOBAL&" + if (!hasVersion) { "v=1" - } else { "" } + "&language=${language}&locale=${language}&platform=APP_ANDROID" + } else { "" } + "&language=$languageCode&locale=$languageCode&platform=APP_ANDROID" return signer.makeEncryptUrl(urlWithParams) } @@ -317,7 +317,7 @@ private class WebtoonsUrlSigner(val secret: String) { "&" } else { "?" - } + "msgpad=${msgpad}&md=${digest}" + } + "msgpad=$msgpad&md=$digest" } } From 27c29f99d97a53c318584a206c5537ef269de583 Mon Sep 17 00:00:00 2001 From: devi Date: Wed, 4 Oct 2023 17:14:12 +0200 Subject: [PATCH 10/11] minor fixes and add small sources --- .../parsers/site/madara/en/BestManhuaCom.kt | 113 +----------------- .../parsers/site/madara/es/BarManga.kt | 12 ++ .../kotatsu/parsers/site/madara/pt/Taberu.kt | 12 ++ .../site/mangareader/pt/Origamiorpheans.kt | 10 +- 4 files changed, 26 insertions(+), 121 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/BarManga.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/Taberu.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/en/BestManhuaCom.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/en/BestManhuaCom.kt index 72d513f0..7f8fa1cf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/en/BestManhuaCom.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/en/BestManhuaCom.kt @@ -2,122 +2,11 @@ package org.koitharu.kotatsu.parsers.site.madara.en import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaPage import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.site.madara.MadaraParser -import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("BESTMANHUACOM", "Best Manhua .Com", "en") internal class BestManhuaCom(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.BESTMANHUACOM, "bestmanhua.com", 20) { - override val datePattern = "dd MMMM yyyy" - override val tagPrefix = "genres/" - override val listUrl = "all-manga/" + MadaraParser(context, MangaSource.BESTMANHUACOM, "bestmanhua.com", 10) { override val withoutAjax = true - override val selectDesc = "div.dsct" - override val selectTestAsync = "div.panel-manga-chapter" - override val selectDate = "span.chapter-time" - override val selectChapter = "li.a-h" - override val selectBodyPage = "div.manga-content div.read-content" - override val selectPage = "div.image-placeholder" - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val tag = tags.oneOrThrowIfMany() - val url = buildString { - append("https://") - append(domain) - val pages = page + 1 - when { - !query.isNullOrEmpty() -> { - append("/page/") - append(pages.toString()) - append("/?s=") - append(query.urlEncoded()) - append("&post_type=wp-manga&") - } - - !tags.isNullOrEmpty() -> { - append("/$tagPrefix") - append(tag?.key.orEmpty()) - append("/") - append(pages.toString()) - append("?") - } - - else -> { - append("/$listUrl") - append(pages.toString()) - append("?") - } - } - append("sort=") - when (sortOrder) { - SortOrder.POPULARITY -> append("most-viewd") - SortOrder.UPDATED -> append("latest-updated") - SortOrder.NEWEST -> append("release-date") - SortOrder.ALPHABETICAL -> append("name-az") - SortOrder.RATING -> append("rating") - } - } - val doc = webClient.httpGet(url).parseHtml() - - return doc.select("div.page-item").map { div -> - val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") - val summary = div.selectFirstOrThrow(".bigor-manga") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src().orEmpty(), - title = summary.selectFirst("h3")?.text().orEmpty(), - altTitle = null, - rating = div.selectFirstOrThrow("div.item-rate span").ownText().toFloatOrNull()?.div(5f) ?: -1f, - tags = emptySet(), - author = null, - state = null, - source = source, - isNsfw = isNsfwSource, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val chapterId = - doc.selectFirst("script:containsData(chapter_id = )")?.toString()?.substringAfter("chapter_id = ") - ?.substringBefore(",") - - val json = - webClient.httpGet("https://$domain/ajax/image/list/chap/$chapterId?mode=vertical&quality=high").parseJson() - - val html = json.getString("html").split("/div>") - - val pages = ArrayList() - - html.map { t -> - if (t.contains("data-src=")) { - val url = t.substringAfter("data-src=\"").substringBefore("\"") - pages.add( - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ), - ) - } - } - return pages - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/BarManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/BarManga.kt new file mode 100644 index 00000000..0509103e --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/BarManga.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madara.es + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madara.MadaraParser + +@MangaSourceParser("BARMANGA", "Bar Manga", "es") +internal class BarManga(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.BARMANGA, "barmanga.com") { + override val datePattern = "MM/dd/yyyy" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/Taberu.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/Taberu.kt new file mode 100644 index 00000000..121c8cbe --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/Taberu.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madara.pt + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madara.MadaraParser + +@MangaSourceParser("TABERU", "Taberu", "pt") +internal class Taberu(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.TABERU, "taberu.org", 10) { + override val datePattern: String = "dd/MM/yyyy" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/pt/Origamiorpheans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/pt/Origamiorpheans.kt index 4955efa8..443ff174 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/pt/Origamiorpheans.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/pt/Origamiorpheans.kt @@ -7,12 +7,4 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser @MangaSourceParser("ORIGAMIORPHEANS", "Origami Orpheans", "pt") internal class Origamiorpheans(context: MangaLoaderContext) : - MangaReaderParser( - context, - MangaSource.ORIGAMIORPHEANS, - "origami-orpheans.com.br", - pageSize = 20, - searchPageSize = 20, - ) { - override val datePattern = "MMM d, yyyy" -} + MangaReaderParser(context, MangaSource.ORIGAMIORPHEANS, "origami-orpheans.com", pageSize = 20, searchPageSize = 10) From 400a90464e205195c01b173002dea1a5011e1f99 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 6 Oct 2023 10:36:51 +0300 Subject: [PATCH 11/11] [LineWebtoons] Refactoring --- .../parsers/site/en/LineWebtoonsParser.kt | 296 ++++++------------ 1 file changed, 94 insertions(+), 202 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt index b80c28da..f60a07de 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt @@ -1,27 +1,32 @@ package org.koitharu.kotatsu.parsers.site.en +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import okhttp3.Headers +import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed -import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet -import org.koitharu.kotatsu.parsers.util.json.toJSONList -import java.net.URLEncoder +import org.koitharu.kotatsu.parsers.util.json.* import java.util.* import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec -internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: MangaSource) : MangaParser(context, source) { - private val signer = WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") +internal abstract class LineWebtoonsParser( + context: MangaLoaderContext, + source: MangaSource, +) : MangaParser(context, source) { + + private val signer by lazy { + WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") + } // we don't __really__ support changing this domain because: // 1. I don't think other websites have this exact API @@ -29,8 +34,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: // 3. we rely on the HTTP client setting the referer header to webtoons.com // // This effectively means that changing the domain will break the source. Yikes - override val configKeyDomain - get() = ConfigKey.Domain("webtoons.com") + override val configKeyDomain = ConfigKey.Domain("webtoons.com") private val apiDomain = "global.apis.naver.com" private val staticDomain = "webtoon-phinf.pstatic.net" @@ -48,7 +52,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: .build() override suspend fun getPageUrl(page: MangaPage): String { - return page.url + return page.url.toAbsoluteUrl(staticDomain) } // some language tags do not map perfectly to the ones used by the API @@ -60,7 +64,9 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: } private suspend fun getChapters(titleNo: Long): List { - val firstResult = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30") + val firstResult = makeRequest( + url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30", + ) val totalEpisodeCount = firstResult .getJSONObject("episodeList") @@ -73,20 +79,21 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: .toMutableList() while (episodes.count() < totalEpisodeCount) { - val page = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30") - .getJSONObject("episodeList") + val page = makeRequest( + url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30", + ).getJSONObject("episodeList") .getJSONArray("episode") .toJSONList() episodes.addAll(page) } - return episodes.mapIndexed { i, jo -> + return episodes.mapChapters { i, jo -> MangaChapter( id = generateUid("$titleNo-$i"), name = jo.getString("episodeTitle"), number = jo.getInt("episodeSeq"), - url = "$titleNo-${jo.getString("episodeNo")}", + url = "$titleNo-${jo.get("episodeNo")}", uploadDate = jo.getLong("modifyYmdt"), branch = null, scanlator = null, @@ -95,10 +102,11 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: }.sortedBy { it.number } } - override suspend fun getDetails(manga: Manga): Manga { + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val titleNo = manga.url.toLong() + val chaptersDeferred = async { getChapters(titleNo) } - return makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}") + makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}") .getJSONObject("titleInfo") .let { jo -> Manga( @@ -107,18 +115,16 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: altTitle = null, url = "$titleNo", publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=${titleNo}", - rating = jo.getDouble("starScoreAverage").toFloat() / 10f, - isNsfw = jo.getBoolean("ageGradeNotice"), - coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", - largeCoverUrl = if (jo.has("thumbnailVertical")) { - "https://$staticDomain${jo.getString("thumbnailVertical")}" - } else { null }, + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), tags = setOf(parseTag(jo.getJSONObject("genreInfo"))), - author = jo.getString("writingAuthorName"), + author = jo.getStringOrNull("writingAuthorName"), description = jo.getString("synopsis"), // I don't think the API provides this info state = null, - chapters = getChapters(titleNo), + chapters = chaptersDeferred.await(), source = source, ) } @@ -136,9 +142,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: SortOrder.UPDATED -> "UPDATE" SortOrder.POPULARITY -> "READ_COUNT" SortOrder.RATING -> "LIKEIT" - else -> { - throw IllegalArgumentException("Unsupported sort order: $sortOrder") - } + else -> throw IllegalArgumentException("Unsupported sort order: $sortOrder") } val manga = if (query != null) { @@ -146,7 +150,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: throw IllegalArgumentException("This source does not support search with tags") } - makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset+1}&pageSize=20") + makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20") .getJSONObject("challengeSearch") .getJSONArray("titleList") .mapJSON { jo -> @@ -156,21 +160,22 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: id = generateUid(titleNo), title = jo.getString("title"), altTitle = null, - url = "$titleNo", + url = titleNo.toString(), publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo", rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", + isNsfw = isNsfwSource, + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), largeCoverUrl = null, - tags = setOf(), - author = jo.getString("writingAuthorName"), + tags = emptySet(), + author = jo.getStringOrNull("writingAuthorName"), description = null, state = null, source = source, ) } } else { - val result = makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset+1}&pageSize=20") + val result = + makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset + 1}&pageSize=20") val genres = result.getJSONObject("genreList") .getJSONArray("challengeGenres") @@ -187,14 +192,14 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: id = generateUid(titleNo), title = jo.getString("title"), altTitle = null, - url = "$titleNo", + url = titleNo.toString(), publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo", - rating = jo.getDouble("starScoreAverage").toFloat() / 10f, - isNsfw = jo.getBoolean("ageGradeNotice"), - coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), - tags = setOf(genres[jo.getString("representGenre")]!!), - author = jo.getString("writingAuthorName"), + tags = setOfNotNull(genres[jo.getString("representGenre")]), + author = jo.getStringOrNull("writingAuthorName"), description = jo.getString("synopsis"), // I don't think the API provides this info state = null, @@ -212,10 +217,10 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=$titleNo&episodeNo=$episodeNo") .getJSONObject("episodeInfo") .getJSONArray("imageInfo") - .mapJSONIndexed() { i, jo -> + .mapJSONIndexed { i, jo -> MangaPage( id = generateUid("$titleNo-$episodeNo-$i"), - url = "https://$staticDomain${jo.getString("url")}", + url = jo.getString("url"), preview = null, source = source, ) @@ -239,188 +244,75 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: private suspend fun makeRequest(url: String): JSONObject { val resp = webClient.httpGet(finalizeUrl(url)) - val message = resp.parseJson().getJSONObject("message") - if (resp.isSuccessful) { - return message.getJSONObject("result") - } else { - // TODO: handle 404 and some other error codes - val code = message.getInt("code") - val errorMessage = message.getString("message") - throw ParseException("Api error (code=$code): $errorMessage", url) + val message: JSONObject? = resp.parseJson().optJSONObject("message") + return when (resp.code) { + in 200..299 -> checkNotNull(message).getJSONObject("result") + 404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url) + else -> { + val code = message?.getIntOrDefault("code", 0) + val errorMessage = message?.getStringOrNull("message") + throw ParseException("Api error (code=$code): $errorMessage", url) + } } } - private fun finalizeUrl(url: String): String { - val absoluteUrl = url.toAbsoluteUrl(apiDomain) - val parsedUrl = absoluteUrl.toHttpUrl() - val hasVersion = parsedUrl.queryParameter("v") != null - val hasQuery = parsedUrl.query != null - - val urlWithParams = absoluteUrl + if (hasQuery) { - "&" - } else { - "?" - } + "serviceZone=GLOBAL&" + if (!hasVersion) { - "v=1" - } else { "" } + "&language=$languageCode&locale=$languageCode&platform=APP_ANDROID" - - return signer.makeEncryptUrl(urlWithParams) + private fun finalizeUrl(url: String): HttpUrl { + val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl() + val builder = httpUrl.newBuilder() + .addQueryParameter("serviceZone", "GLOBAL") + if (httpUrl.queryParameter("v") == null) { + builder.addQueryParameter("v", "1") + } + builder.addQueryParameter("language", languageCode) + .addQueryParameter("locale", "languageCode") + .addQueryParameter("platform", "APP_ANDROID") + signer.makeEncryptUrl(builder) + return builder.build() } @MangaSourceParser("LINEWEBTOONS_EN", "Line Webtoons English", "en", type = ContentType.MANGA) class English(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_EN) + @MangaSourceParser("LINEWEBTOONS_ZH", "Line Webtoons Chinese", "zh", type = ContentType.MANGA) class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ZH) + @MangaSourceParser("LINEWEBTOONS_TH", "Line Webtoons Thai", "th", type = ContentType.MANGA) class Thai(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_TH) + @MangaSourceParser("LINEWEBTOONS_ID", "Line Webtoons Indonesian", "id", type = ContentType.MANGA) class Indonesian(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ID) + @MangaSourceParser("LINEWEBTOONS_ES", "Line Webtoons Spanish", "es", type = ContentType.MANGA) class Spanish(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ES) + @MangaSourceParser("LINEWEBTOONS_FR", "Line Webtoons French", "fr", type = ContentType.MANGA) class French(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_FR) + @MangaSourceParser("LINEWEBTOONS_DE", "Line Webtoons German", "de", type = ContentType.MANGA) class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_DE) -} - + private inner class WebtoonsUrlSigner(private val secret: String) { -private class WebtoonsUrlSigner(val secret: String) { - private val mac = Mac.getInstance("HmacSHA1").apply { - this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1")) - } - - private fun getMessage(url: String, msgpad: String): String { - return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad - } - - private fun getMessageDigest(s: String): String { - var signedMessage: ByteArray - synchronized(mac) { signedMessage = mac.doFinal(s.toByteArray()) } - - // we don't use the context.encodeBase64 here because it adds newlines and doesn't add padding - // we, however, need padding and no newlines - return encodeBase64(signedMessage) - } - - - fun makeEncryptUrl(s: String): String { - return makeEncryptUrlCore( - s, - java.lang.String.valueOf(Calendar.getInstance().timeInMillis), - ) - } - - private fun makeEncryptUrlCore(url: String, msgpad: String): String { - val digest = URLEncoder.encode(getMessageDigest(getMessage(url, msgpad)), "utf-8") - return url + if (url.contains("?")) { - "&" - } else { - "?" - } + "msgpad=$msgpad&md=$digest" - } -} + private val mac = Mac.getInstance("HmacSHA1").apply { + this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1")) + } + private fun getMessage(url: String, msgpad: String): String { + return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad + } -private val INT_TO_BASE64: CharArray = charArrayOf( - 'A', - 'B', - 'C', - 'D', - 'E', - 'F', - 'G', - 'H', - 'I', - 'J', - 'K', - 'L', - 'M', - 'N', - 'O', - 'P', - 'Q', - 'R', - 'S', - 'T', - 'U', - 'V', - 'W', - 'X', - 'Y', - 'Z', - 'a', - 'b', - 'c', - 'd', - 'e', - 'f', - 'g', - 'h', - 'i', - 'j', - 'k', - 'l', - 'm', - 'n', - 'o', - 'p', - 'q', - 'r', - 's', - 't', - 'u', - 'v', - 'w', - 'x', - 'y', - 'z', - '0', - '1', - '2', - '3', - '4', - '5', - '6', - '7', - '8', - '9', - '+', - '/', -) - -private fun encodeBase64(arr: ByteArray): String { - val groupsCount = arr.size / 3 - val extraCount = arr.size - groupsCount * 3 - - val sb = StringBuffer((arr.size + 2) / 3 * 4) - val code = INT_TO_BASE64 - var groupIndex = 0 - var position = 0 - while (groupIndex < groupsCount) { - val v4 = arr[position].toInt() and 0xFF - val v5 = arr[position + 1].toInt() and 0xFF - val v6 = arr[position + 2].toInt() and 0xFF - sb.append(code[v4 shr 2]) - sb.append(code[v4 shl 4 and 0x3F or (v5 shr 4)]) - sb.append(code[v5 shl 2 and 0x3F or (v6 shr 6)]) - sb.append(code[v6 and 0x3F]) - ++groupIndex - position += 3 - } + private fun getMessageDigest(s: String): String { + val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) } + return context.encodeBase64(signedMessage) + } - if (extraCount != 0) { - val v1 = arr[position].toInt() and 0xFF - sb.append(code[v1 shr 2]) - if (extraCount == 1) { - sb.append(code[v1 shl 4 and 0x3F]) - sb.append("==") - } else { - val v2 = arr[position + 1].toInt() and 0xFF - sb.append(code[v1 shl 4 and 0x3F or (v2 shr 4)]) - sb.append(code[v2 shl 2 and 0x3F]) - sb.append('=') + fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) { + val msgpad = Calendar.getInstance().timeInMillis.toString() + val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgpad)) + urlBuilder + .addQueryParameter("msgpad", msgpad) + .addQueryParameter("md", digest) +// .addEncodedQueryParameter("md", digest.urlEncoded()) } } - return sb.toString() }