From 34d2e1530252612d22a7da9c0d4498dc7183e484 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Tue, 18 Apr 2023 18:34:22 +0300 Subject: [PATCH 1/5] [New source] HoneyManga --- .../kotatsu/parsers/site/HoneyMangaParser.kt | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt new file mode 100644 index 00000000..161943d2 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -0,0 +1,206 @@ +package org.koitharu.kotatsu.core.parser + +import okhttp3.Headers +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed +import org.koitharu.kotatsu.parsers.util.parseJson +import org.koitharu.kotatsu.parsers.util.parseJsonArray +import org.koitharu.kotatsu.parsers.util.removeSuffix +import org.koitharu.kotatsu.parsers.util.tryParse +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + + +private const val PAGE_SIZE = 20 + +@MangaSourceParser("HONEYMANGA", "Honey Manga", "uk") +class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HONEYMANGA, PAGE_SIZE), Interceptor { + + private val urlApi = "https://data.api.$domain" + private val mangaApi = "$urlApi/v2/manga/cursor-list" + private val chapterApi = "$urlApi/v2/chapter/cursor-list" + private val genresListApi = "$urlApi/genres-tags/genres-list" + private val framesApi = "$urlApi/chapter/frames" + private val searchApi = "https://search.api.$domain/api/v1/title/search-matching?query=" + + private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources" + override val headers + get() = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("honey-manga.com.ua", null) + + override val sortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + + override suspend fun getDetails(manga: Manga): Manga { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + val body = JSONObject() + body.put("mangaId", manga.url) + body.put("pageSize", 999999) // Hack lol (no) + body.put("page", 1) + body.put("sortOrder", "ASC") + val chapterRequest = webClient.httpPost(chapterApi, body).parseJson() + return manga.copy( + chapters = chapterRequest.getJSONArray("data").mapJSONIndexed() { i, jo -> + MangaChapter( + id = generateUid(jo.getString("id")), + name = buildString { + append("Том ") + append(jo.optString("volume", "0")) + append(". ") + append("Розділ ") + append(jo.optString("chapterNum", "0")) + if (jo.optString("title") != "Title") { + append(" - ") + append(jo.optString("title")) + } + }, + number = i + 1, + url = jo.optString("chapterResourcesId"), + scanlator = null, + uploadDate = dateFormat.tryParse(jo.getString("lastUpdated")), + branch = null, + source = source + ) + } + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder + ): List { + val body = JSONObject() + var content: JSONArray? = null + content = if (!query.isNullOrEmpty()) { + if((query.length < 3) || (page > 1)) return emptyList() + body.put("query", query) + webClient.httpGet(searchApi + query).parseJsonArray() + } else { + body.put("page", page) + body.put("pageSize", PAGE_SIZE) + body.put("filters", JSONArray()) + val sort = JSONObject() + sort.put("sortBy", getSortKey(sortOrder)) + sort.put("sortOrder", "DESC") + body.put("sort", sort) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + } + return content!!.mapJSON { jo -> + val id = jo.getString("id") + val posterUrl = jo.getString("posterUrl") + Manga( + id = generateUid(id), + title = jo.getString("title"), + altTitle = jo.getStringOrNull("alternativeTitle"), + url = id, + publicUrl = "https://$domain/book/$id", + rating = RATING_UNKNOWN, + isNsfw = isNsfw(jo.getStringOrNull("adult")), + coverUrl = getCoverUrl(posterUrl, 256), + tags = getTags(jo.optJSONArray("genresAndTags")!!), + state = when (jo.getStringOrNull("titleStatus")) { + "Онгоінг" -> MangaState.ONGOING + "Завершено" -> MangaState.FINISHED + else -> null + }, + author = null, + largeCoverUrl = getCoverUrl(posterUrl, 1080), + description = jo.getStringOrNull("description"), + chapters = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val body = JSONObject() + body.put("chapterId", chapter.url) + + val content = webClient.httpPost(framesApi, body).parseJson().getJSONObject("resourceIds") + val mangaPage = mutableListOf() + (0 until content.length()).forEach { i -> + val item = content.get(i.toString()).toString() + + mangaPage.add(MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source)) + } + return mangaPage + } + + override suspend fun getTags(): Set { + // https://data.api.honey-manga.com.ua/genres-tags/genres-list + val tagsSet = mutableListOf() + val content = webClient.httpGet(genresListApi).parseJsonArray() + (0 until content.length()).forEach { i -> + val item = content.get(i).toString() + + tagsSet.add(MangaTag(item, item, source)) + } + + return tagsSet.toSet() + } + + // Need for disable encoding (with encoding not working) + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = if (request.header("Content-Encoding") != null) { + request.newBuilder().removeHeader("CommonHeaders.CONTENT_ENCODING").build() + } else { + request + } + return chain.proceed(newRequest) + } + + private fun isNsfw(adultValue: String?): Boolean { + val intValue = adultValue?.removeSuffix('+')?.toIntOrNull() + return intValue != null && intValue >= 18 + } + + private fun getCoverUrl(id: String, w: Int): String { + return "https://$domain/_next/image?url=https%3A%2F%2Fmanga-storage.fra1.digitaloceanspaces.com%2Fpublic-resources%2F$id&w=$w&q=75" + } + + private fun getTags(jsonTags: JSONArray): Set { + val tagsSet = mutableListOf() + (0 until jsonTags.length()).forEach { i -> + val item = jsonTags.get(i) + + tagsSet.add(MangaTag(item.toString(), item.toString(), source)) + } + + return tagsSet.toSet() + } + + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.POPULARITY -> "likes" + SortOrder.NEWEST -> "lastUpdated" + else -> "likes" + } +} From a3565abdc37d2fb5e287ca79e2dd4c1adb9dfb25 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Tue, 18 Apr 2023 19:35:15 +0300 Subject: [PATCH 2/5] [HoneyManga] Improve code --- .../kotatsu/parsers/site/HoneyMangaParser.kt | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index 161943d2..d9d8ead9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.core.parser +import androidx.collection.ArraySet import okhttp3.Headers import okhttp3.Interceptor import okhttp3.Response @@ -26,6 +27,7 @@ import org.koitharu.kotatsu.parsers.util.parseJson import org.koitharu.kotatsu.parsers.util.parseJsonArray import org.koitharu.kotatsu.parsers.util.removeSuffix import org.koitharu.kotatsu.parsers.util.tryParse +import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -36,14 +38,15 @@ private const val PAGE_SIZE = 20 @MangaSourceParser("HONEYMANGA", "Honey Manga", "uk") class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HONEYMANGA, PAGE_SIZE), Interceptor { - private val urlApi = "https://data.api.$domain" - private val mangaApi = "$urlApi/v2/manga/cursor-list" - private val chapterApi = "$urlApi/v2/chapter/cursor-list" - private val genresListApi = "$urlApi/genres-tags/genres-list" - private val framesApi = "$urlApi/chapter/frames" + private val urlApi get() = "https://data.api.$domain" + private val mangaApi get() = "$urlApi/v2/manga/cursor-list" + private val chapterApi get() = "$urlApi/v2/chapter/cursor-list" + private val genresListApi get() = "$urlApi/genres-tags/genres-list" + private val framesApi get() = "$urlApi/chapter/frames" private val searchApi = "https://search.api.$domain/api/v1/title/search-matching?query=" private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources" + override val headers get() = Headers.Builder() .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") @@ -98,12 +101,10 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, sortOrder: SortOrder ): List { val body = JSONObject() - var content: JSONArray? = null - content = if (!query.isNullOrEmpty()) { - if((query.length < 3) || (page > 1)) return emptyList() - body.put("query", query) - webClient.httpGet(searchApi + query).parseJsonArray() - } else { + var content: JSONArray = JSONArray() + + // Popular/Newest + content = if (query.isNullOrEmpty()) { body.put("page", page) body.put("pageSize", PAGE_SIZE) body.put("filters", JSONArray()) @@ -112,8 +113,12 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, sort.put("sortOrder", "DESC") body.put("sort", sort) webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + // Search + } else { + if((query.length < 3) || (page > 1)) return emptyList() + webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() } - return content!!.mapJSON { jo -> + return content.mapJSON { jo -> val id = jo.getString("id") val posterUrl = jo.getString("posterUrl") Manga( @@ -125,7 +130,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, rating = RATING_UNKNOWN, isNsfw = isNsfw(jo.getStringOrNull("adult")), coverUrl = getCoverUrl(posterUrl, 256), - tags = getTags(jo.optJSONArray("genresAndTags")!!), + tags = getTitleTags(jo.optJSONArray("genresAndTags")), state = when (jo.getStringOrNull("titleStatus")) { "Онгоінг" -> MangaState.ONGOING "Завершено" -> MangaState.FINISHED @@ -145,13 +150,10 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, body.put("chapterId", chapter.url) val content = webClient.httpPost(framesApi, body).parseJson().getJSONObject("resourceIds") - val mangaPage = mutableListOf() - (0 until content.length()).forEach { i -> + return List(content.length()) { i -> val item = content.get(i.toString()).toString() - - mangaPage.add(MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source)) + MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source) } - return mangaPage } override suspend fun getTags(): Set { @@ -187,8 +189,14 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, return "https://$domain/_next/image?url=https%3A%2F%2Fmanga-storage.fra1.digitaloceanspaces.com%2Fpublic-resources%2F$id&w=$w&q=75" } - private fun getTags(jsonTags: JSONArray): Set { - val tagsSet = mutableListOf() + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.POPULARITY -> "likes" + SortOrder.NEWEST -> "lastUpdated" + else -> "likes" + } + + private fun getTitleTags(jsonTags: JSONArray): Set { + val tagsSet = ArraySet(jsonTags.length()) (0 until jsonTags.length()).forEach { i -> val item = jsonTags.get(i) @@ -197,10 +205,4 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, return tagsSet.toSet() } - - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.POPULARITY -> "likes" - SortOrder.NEWEST -> "lastUpdated" - else -> "likes" - } } From e5ececa42db3f7af745723f4f54450ad6864de9f Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Wed, 19 Apr 2023 19:58:25 +0300 Subject: [PATCH 3/5] [HoneyManga] Update code, get(), CONTENT_ENCODING --- .../org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index d9d8ead9..e2fa848f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -43,7 +43,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, private val chapterApi get() = "$urlApi/v2/chapter/cursor-list" private val genresListApi get() = "$urlApi/genres-tags/genres-list" private val framesApi get() = "$urlApi/chapter/frames" - private val searchApi = "https://search.api.$domain/api/v1/title/search-matching?query=" + private val searchApi get() = "https://search.api.$domain/api/v1/title/search-matching?query=" private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources" @@ -173,7 +173,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() val newRequest = if (request.header("Content-Encoding") != null) { - request.newBuilder().removeHeader("CommonHeaders.CONTENT_ENCODING").build() + request.newBuilder().removeHeader("Content-Encoding").build() } else { request } From 647a694d9171bb4c510c4deaa0fb54909114e127 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Wed, 19 Apr 2023 20:18:45 +0300 Subject: [PATCH 4/5] [HoneyManga] throw IllegalArgumentException if search < 3 --- .../org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index e2fa848f..ce6de408 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -115,7 +115,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") // Search } else { - if((query.length < 3) || (page > 1)) return emptyList() + if((query.length < 3) || (page > 1)) throw IllegalArgumentException("Запит має містити щонайменше 3 символи / The query must contain at least 3 characters") webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() } return content.mapJSON { jo -> From b4a4c860494371daa80f2412e06b2f037f5521d3 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Wed, 26 Apr 2023 16:35:42 +0300 Subject: [PATCH 5/5] [HoneyManga] Impl. filter by genres and fix cover url --- .../kotatsu/parsers/site/HoneyMangaParser.kt | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index ce6de408..a7057cc1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -1,4 +1,4 @@ -package org.koitharu.kotatsu.core.parser +package org.koitharu.kotatsu.parsers.site import androidx.collection.ArraySet import okhttp3.Headers @@ -102,21 +102,39 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, ): List { val body = JSONObject() var content: JSONArray = JSONArray() - + body.put("page", page) + body.put("pageSize", PAGE_SIZE) + val sort = JSONObject() + sort.put("sortBy", getSortKey(sortOrder)) + sort.put("sortOrder", "DESC") + body.put("sort", sort) + + // Tags + content = if (!tags.isNullOrEmpty()) { + val filters: JSONArray = JSONArray() + val tag_filter = JSONObject() + tag_filter.put("filterBy", "genres") + tag_filter.put("filterOperator", "ALL") + val tag: JSONArray = JSONArray() + tags.forEach { + tag.put(it.title) + } + tag_filter.put("filterValue", tag) + filters.put(tag_filter) + body.put("filters", filters) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + // Search + } else if(!query.isNullOrEmpty()) { + if((query.length < 3)) throw IllegalArgumentException("Запит має містити щонайменше 3 символи / The query must contain at least 3 characters") + if(page == 1) { + webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() + } else { + JSONArray() + } // Popular/Newest - content = if (query.isNullOrEmpty()) { - body.put("page", page) - body.put("pageSize", PAGE_SIZE) + } else { body.put("filters", JSONArray()) - val sort = JSONObject() - sort.put("sortBy", getSortKey(sortOrder)) - sort.put("sortOrder", "DESC") - body.put("sort", sort) webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") - // Search - } else { - if((query.length < 3) || (page > 1)) throw IllegalArgumentException("Запит має містити щонайменше 3 символи / The query must contain at least 3 characters") - webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() } return content.mapJSON { jo -> val id = jo.getString("id") @@ -186,7 +204,8 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, } private fun getCoverUrl(id: String, w: Int): String { - return "https://$domain/_next/image?url=https%3A%2F%2Fmanga-storage.fra1.digitaloceanspaces.com%2Fpublic-resources%2F$id&w=$w&q=75" + // https://honey-manga.com.ua/_next/image?url=https%3A%2F%2Fhoneymangastorage.b-cdn.net%2Fpublic-resources%2F1c4613c2-ffe3-405a-b26a-2cab59ddd223%3Foptimizer%3Dimage%26width%3D512%26height%3D512&w=3840&q=75 + return "https://$domain/_next/image?url=https%3A%2F%2Fhoneymangastorage.b-cdn.net%2Fpublic-resources%2F$id&w=$w&q=75" } private fun getSortKey(order: SortOrder?) = when (order) {