From 3771b8d26baf58087d0af20dffa4e925cefc3aa4 Mon Sep 17 00:00:00 2001 From: devi Date: Fri, 18 Apr 2025 20:20:42 +0200 Subject: [PATCH] Fix PhenixScans close #1599 close #1675 Fix KeyoappParser close #1640 --- .../parsers/site/fr/LegacyScansParser.kt | 2 + .../parsers/site/fr/PhenixscansParser.kt | 263 ++++++++++++++++++ .../parsers/site/keyoapp/KeyoappParser.kt | 6 +- .../parsers/site/madara/fr/InovaScanManga.kt | 2 + .../site/mangareader/fr/PhenixscansParser.kt | 19 -- 5 files changed, 270 insertions(+), 22 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/PhenixscansParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/PhenixscansParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/LegacyScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/LegacyScansParser.kt index d47b83e3..48194ade 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/LegacyScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/LegacyScansParser.kt @@ -2,6 +2,7 @@ package org.koitharu.kotatsu.parsers.site.fr import org.json.JSONArray import org.json.JSONObject +import org.koitharu.kotatsu.parsers.Broken import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey @@ -12,6 +13,7 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSON import java.text.SimpleDateFormat import java.util.* +@Broken("images canvas need to refactor") @MangaSourceParser("LEGACY_SCANS", "LegacyScans", "fr") internal class LegacyScansParser(context: MangaLoaderContext) : LegacyPagedMangaParser(context, MangaParserSource.LEGACY_SCANS, 18) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/PhenixscansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/PhenixscansParser.kt new file mode 100644 index 00000000..d560a843 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/PhenixscansParser.kt @@ -0,0 +1,263 @@ +package org.koitharu.kotatsu.parsers.site.fr + +import kotlinx.coroutines.* +import org.json.JSONArray +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.LegacyPagedMangaParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("PHENIXSCANS", "PhenixScans", "fr") +internal class PhenixscansParser(context: MangaLoaderContext) : + LegacyPagedMangaParser(context, MangaParserSource.PHENIXSCANS, 18) { + + override val configKeyDomain = ConfigKey.Domain("phenix-scans.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override val availableSortOrders: Set = + EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://api.") + append(domain) + append("/front/manga") + + when { + !filter.query.isNullOrEmpty() -> { + if (page > 1) { + return emptyList() + } + append("/search?query=") + append(filter.query.urlEncoded()) + } + + else -> { + append("?page=") + append(page.toString()) + append("&limit=18&sort=") + when (order) { + SortOrder.POPULARITY -> append("rating") + SortOrder.UPDATED -> append("updatedAt") + SortOrder.ALPHABETICAL -> append("title") + else -> append("updatedAt") + } + + if (filter.tags.isNotEmpty()) { + append("&genre=") + filter.tags.joinTo(this, separator = ",") { it.key } + } + + filter.types.oneOrThrowIfMany()?.let { + append("&type=") + append( + when (it) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + else -> "" + }, + ) + } + + if (filter.states.isNotEmpty()) { + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + when (it) { + MangaState.ONGOING -> append("Ongoing") + MangaState.FINISHED -> append("Completed") + MangaState.PAUSED -> append("Hiatus") + else -> append("") + } + } + } + + } + } + } + + return parseMangaList(webClient.httpGet(url).parseJson().getJSONArray("mangas")) + } + + private fun parseMangaList(json: JSONArray): List { + return json.mapJSON { j -> + val slug = j.getString("slug") + val urlManga = "https://$domain/manga/$slug" + Manga( + id = generateUid(j.getString("_id")), + title = j.getString("title"), + altTitles = emptySet(), + url = urlManga, + publicUrl = urlManga, + rating = j.getFloatOrDefault("averageRating", RATING_UNKNOWN) / 10f, + contentRating = null, + description = j.getStringOrNull("synopsis"), + coverUrl = "https://cdn.phenix-scans.com/?url=https://api.phenix-scans.com/" + j.getString("coverImage") + "&output=webp&w=400&ll", + tags = emptySet(), + state = when (j.getStringOrNull("status")) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + "Hiatus" -> MangaState.FINISHED + else -> null + }, + authors = emptySet(), + source = source, + ) + } + } + + private val dateFormat = SimpleDateFormat("d MMM yyyy", sourceLocale) + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val mangaUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(mangaUrl).parseHtml() + + manga.copy( + tags = doc.select("div.project__content-tags a").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast("tag="), + title = a.text().toTitleCase(), + source = source, + ) + }, + chapters = doc.select(" div.project__chapters a.project__chapter") + .mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst(".project__chapter-title")?.text().orEmpty() + val dateText = a.selectFirst(".project__chapter-date")?.text().orEmpty() + MangaChapter( + id = generateUid(href), + title = name, + number = i.toFloat(), + volume = 0, + url = href, + scanlator = null, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.chapter-images img.chapter-image").map { img -> + val url = img.src() ?: img.parseFailed("Image src not found") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set { + val json = webClient.httpGet("https://api.$domain/front/manga?page=1&limit=18&sort=updatedAt").parseJson() + .getJSONArray("genres") + return json.mapJSONToSet { + MangaTag( + key = it.getString("_id"), + title = it.getString("name"), + source = source, + ) + } + } + + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val d = date?.lowercase() ?: return 0 + return when { + + WordSet( + " sec", " min", " h", " j", " sem", " mois", " année", + ).endsWith(d) -> { + parseRelativeDate(d) + } + + WordSet("il y a").startsWith(d) -> { + parseRelativeDate(d) + } + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + }.let { dateFormat.tryParse(it.joinToString(" ")) } + + else -> dateFormat.tryParse(date) + } + } + + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + return when { + WordSet("second", "sec") + .anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + + WordSet("min", "minute", "minutes") + .anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + + WordSet("heures", "heure", "h") + .anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + + WordSet("jour", "jours", "j") + .anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("sem", "semaine", "semaines").anyWordIn(date) -> cal.apply { + add( + Calendar.WEEK_OF_YEAR, + -number, + ) + }.timeInMillis + + WordSet("mois") + .anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + + WordSet("année") + .anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + + else -> 0 + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/keyoapp/KeyoappParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/keyoapp/KeyoappParser.kt index 66ac6f0d..2ed2f97d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/keyoapp/KeyoappParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/keyoapp/KeyoappParser.kt @@ -99,7 +99,7 @@ internal abstract class KeyoappParser( doc.select("div.grid > div.group") }.map { div -> - val title = div.selectFirstOrThrow("h3").text().orEmpty() + val title = div.selectFirst("h3")?.text().orEmpty() if (query.isNotEmpty() && title.contains(query, ignoreCase = true)) { manga.add(addManga(div)) } @@ -122,13 +122,13 @@ internal abstract class KeyoappParser( private fun addManga(div: Element): Manga { val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") - val cover = div.selectFirst("div.h-full") ?: div.selectFirst("a") + val cover = div.selectFirst("a div.bg-cover") return Manga( id = generateUid(href), url = href, publicUrl = href.toAbsoluteUrl(div.host ?: domain), coverUrl = cover?.styleValueOrNull("background-image")?.cssUrl(), - title = div.selectFirstOrThrow("h3").text().orEmpty(), + title = (div.selectFirst("h3")?.text() ?: div.selectFirst("a")?.attr("title")).orEmpty(), altTitles = emptySet(), rating = RATING_UNKNOWN, tags = div.select("div.gap-1 a").mapToSet { a -> diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/fr/InovaScanManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/fr/InovaScanManga.kt index 976adf3d..1c3a13e9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/fr/InovaScanManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/fr/InovaScanManga.kt @@ -1,10 +1,12 @@ package org.koitharu.kotatsu.parsers.site.madara.fr +import org.koitharu.kotatsu.parsers.Broken import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.site.madara.MadaraParser +@Broken("Source change template") @MangaSourceParser("INOVASCANMANGA", "InovaScanManga", "fr") internal class InovaScanManga(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.INOVASCANMANGA, "inovascanmanga.com") { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/PhenixscansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/PhenixscansParser.kt deleted file mode 100644 index 53b943b0..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/PhenixscansParser.kt +++ /dev/null @@ -1,19 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.mangareader.fr - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser -import org.koitharu.kotatsu.parsers.Broken - -@Broken -@MangaSourceParser("PHENIXSCANS", "PhenixScans", "fr") -internal class PhenixscansParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaParserSource.PHENIXSCANS, "phenixscans.fr", pageSize = 20, searchPageSize = 10) { - override val datePattern = "MMM d, yyyy" - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isTagsExclusionSupported = false, - ) -}