From b6dd7fb5f65eae854f58d43dd139debb34f319a2 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 12 Sep 2023 18:23:34 +0300 Subject: [PATCH] [HentaiUkr] Fixes --- .../site/ru/grouple/AllHentaiParser.kt | 8 +- .../kotatsu/parsers/site/uk/HentaiUkr.kt | 146 ---------------- .../parsers/site/uk/HentaiUkrParser.kt | 158 ++++++++++++++++++ .../koitharu/kotatsu/parsers/MangaSources.kt | 2 +- 4 files changed, 164 insertions(+), 150 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkr.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkrParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt index 5a3f0691..86254824 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/AllHentaiParser.kt @@ -14,7 +14,7 @@ import org.koitharu.kotatsu.parsers.util.parseFailed import org.koitharu.kotatsu.parsers.util.parseHtml import org.koitharu.kotatsu.parsers.util.urlEncoded -@MangaSourceParser("ALLHENTAI", "ALlHentai", "ru", type = ContentType.HENTAI) +@MangaSourceParser("ALLHENTAI", "AllHentai", "ru", type = ContentType.HENTAI) internal class AllHentaiParser( context: MangaLoaderContext, ) : GroupleParser(context, MangaSource.ALLHENTAI, 1) { @@ -24,7 +24,7 @@ internal class AllHentaiParser( override val authUrl: String get() { - val targetUri = "https://${domain}/".urlEncoded() + val targetUri = "https://$domain/".urlEncoded() return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" } @@ -46,6 +46,8 @@ internal class AllHentaiParser( val res = element.parent()?.text() return if (res.isNullOrEmpty()) { root.parseFailed("Cannot find username") - } else res + } else { + res + } } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkr.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkr.kt deleted file mode 100644 index 0f0758cb..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkr.kt +++ /dev/null @@ -1,146 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.uk - -import androidx.collection.ArraySet -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.PagedMangaParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.toJSONList -import java.text.SimpleDateFormat -import java.util.* - -private const val HEADER_ENCODING = "Content-Encoding" -private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US) -@MangaSourceParser("HENTAIUKR", "Hentaiukr", "uk") -class HentaiUkrParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HENTAIUKR, 1), - Interceptor { - - private val allManga get() = "https://$domain/search/objects.json" - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("hentaiukr.com") - - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - ) - - override suspend fun getDetails(manga: Manga): Manga { - val json= webClient.httpGet(allManga).parseJson().getJSONArray("manga").toJSONList().find { it.getString("name") == manga.title } - val html_manga = webClient.httpGet("https://$domain${manga.url}").parseHtml() - val about = html_manga.body().requireElementById("about").text() - - return manga.copy( - description = about, - chapters = listOf( - MangaChapter( - id = generateUid(manga.id), - name = manga.title, - number = 1, - url = manga.url, - scanlator = null, - uploadDate = date.tryParse(json!!.getString("add_date")), - branch = null, - source = source, - ) - ) - ) - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - // No pagination - if(page != 1){ - return emptyList() - } - - // Get all manga - var allManga = webClient.httpGet(allManga).parseJson().getJSONArray("manga").toJSONList() - - // Search - if (!query.isNullOrBlank()){ - val queryArray = ArraySet() - allManga.map { item -> - when (query.toString()) { - in item.getString("name") -> queryArray.add(item) - in item.getString("eng_name") -> queryArray.add(item) - in item.getString("orig_name") -> queryArray.add(item) - in item.getString("author") -> queryArray.add(item) - in item.getString("team") -> queryArray.add(item) - else -> {} - } - } - allManga = queryArray.toList() - } - - // Return to app - return allManga.map { jo -> - val id = jo.getString("id") - Manga( - id = generateUid(id), - title = jo.getString("name"), - altTitle = jo.getStringOrNull("eng_name"), - url = jo.getString("url"), - publicUrl = "https://$domain${jo.getString("url")}", - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = "https://$domain${jo.getString("thumb")}", - tags = getTags(jo.optJSONArray("tags")), - state = null, - author = jo.getString("author"), - largeCoverUrl = null, - description = null, - chapters = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val html_pages = webClient.httpGet( "https://$domain${chapter.url}vertical_reader.html").parseHtml() - return html_pages.select("img.image").mapIndexed { i, page -> - MangaPage( - id = generateUid(i.toString()), - "https://$domain${page.attr("src")}", - null, - source - ) - } - } - - override suspend fun getTags(): Set { - return emptySet() - } - - private fun getTags(jsonTags: JSONArray): Set { - val tagsSet = ArraySet(jsonTags.length()) - repeat(jsonTags.length()) { i -> - val item = jsonTags.getJSONObject(i) - - tagsSet.add(MangaTag(title = item.getString("name"), key = item.getString("id"), source = source)) - } - return tagsSet - } - - // Need for disable encoding (with encoding not working) - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = if (request.header(HEADER_ENCODING) != null) { - request.newBuilder().removeHeader(HEADER_ENCODING).build() - } else { - request - } - return chain.proceed(newRequest) - } -} - diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkrParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkrParser.kt new file mode 100644 index 00000000..f31670f3 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkrParser.kt @@ -0,0 +1,158 @@ +package org.koitharu.kotatsu.parsers.site.uk + +import androidx.collection.ArraySet +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.toJSONList +import java.text.SimpleDateFormat +import java.util.* + +private const val HEADER_ENCODING = "Content-Encoding" +private const val PAGE_SIZE = 60 + +@MangaSourceParser("HENTAIUKR", "Hentaiukr", "uk", ContentType.HENTAI) +class HentaiUkrParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.HENTAIUKR), Interceptor { + + private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US) + + private val allManga = SoftSuspendLazy { + webClient.httpGet("https://$domain/search/objects.json").parseJson().getJSONArray("manga").toJSONList() + } + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("hentaiukr.com") + + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + ) + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val jsonDeferred = async { allManga.get().first { it.getString("url") == manga.url } } + val htmlDeferred = async { webClient.httpGet("https://$domain${manga.url}").parseHtml() } + + val about = htmlDeferred.await().body().requireElementById("about").text() + + manga.copy( + description = about, + chapters = listOf( + MangaChapter( + id = generateUid(manga.id), + name = manga.title, + number = 1, + url = manga.url, + scanlator = null, + uploadDate = date.tryParse(jsonDeferred.await().getString("add_date")), + branch = null, + source = source, + ), + ), + ) + } + + override suspend fun getList(offset: Int, query: String?, tags: Set?, sortOrder: SortOrder): List { + // Get all manga + val json = allManga.get().toMutableList() + + // Search + if (!query.isNullOrEmpty()) { + json.retainAll { item -> + item.getString("name").contains(query, ignoreCase = true) || + item.getStringOrNull("eng_name")?.contains(query, ignoreCase = true) == true || + item.getStringOrNull("orig_name")?.contains(query, ignoreCase = true) == true || + item.getStringOrNull("author")?.contains(query, ignoreCase = true) == true || + item.getStringOrNull("team")?.contains(query, ignoreCase = true) == true + } + } + + if (!tags.isNullOrEmpty()) { + val ids = tags.mapToSet { it.key } + json.retainAll { item -> + item.getJSONArray("tags") + .mapJSON { it.getInt("id").toString() } + .any { x -> x in ids } + } + } + + // Return to app + return json.drop(offset).take(PAGE_SIZE).map { jo -> + val id = jo.getLong("id") + Manga( + id = generateUid(id), + title = jo.getString("name"), + altTitle = jo.getStringOrNull("eng_name"), + url = jo.getString("url"), + publicUrl = jo.getString("url").toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = jo.getString("thumb").toAbsoluteUrl(domain), + tags = getTags(jo.optJSONArray("tags")), + state = null, + author = jo.getStringOrNull("author"), + largeCoverUrl = null, + description = null, + chapters = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val htmlPages = webClient.httpGet("https://$domain${chapter.url}vertical_reader.html").parseHtml() + return htmlPages.select("img.image").mapIndexed { i, page -> + MangaPage( + id = generateUid(i.toString()), + "https://$domain${page.attr("src")}", + null, + source, + ) + } + } + + override suspend fun getTags(): Set { + return allManga.get().flatMapTo(HashSet()) { x -> + x.getJSONArray("tags").mapJSON { t -> + MangaTag( + title = t.getString("name"), + key = t.getInt("id").toString(), + source = source, + ) + } + } + } + + private fun getTags(jsonTags: JSONArray): Set { + val tagsSet = ArraySet(jsonTags.length()) + repeat(jsonTags.length()) { i -> + val item = jsonTags.getJSONObject(i) + tagsSet.add( + MangaTag( + title = item.getString("name"), + key = item.getInt("id").toString(), + source = source, + ), + ) + } + return tagsSet + } + + // Need for disable encoding (with encoding not working) + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = if (request.header(HEADER_ENCODING) != null) { + request.newBuilder().removeHeader(HEADER_ENCODING).build() + } else { + request + } + return chain.proceed(newRequest) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt index ef660faa..9c1d2780 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt @@ -3,5 +3,5 @@ package org.koitharu.kotatsu.parsers import org.junit.jupiter.params.provider.EnumSource import org.koitharu.kotatsu.parsers.model.MangaSource -@EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE) +@EnumSource(MangaSource::class, names = ["HENTAIUKR"], mode = EnumSource.Mode.INCLUDE) internal annotation class MangaSources