diff --git a/.github/ISSUE_TEMPLATE/report_issue.yml b/.github/ISSUE_TEMPLATE/report_issue.yml index 9a9562ecf..cc7e0e4f2 100644 --- a/.github/ISSUE_TEMPLATE/report_issue.yml +++ b/.github/ISSUE_TEMPLATE/report_issue.yml @@ -24,6 +24,7 @@ body: 1. First step 2. Second step 3. Issue here + Please use English language validations: required: false diff --git a/.github/ISSUE_TEMPLATE/request_feature.yml b/.github/ISSUE_TEMPLATE/request_feature.yml index 023943e91..e2ac8b5b5 100644 --- a/.github/ISSUE_TEMPLATE/request_feature.yml +++ b/.github/ISSUE_TEMPLATE/request_feature.yml @@ -1,30 +1,31 @@ name: ⭐ Feature request description: Suggest a feature to improve a source -labels: [feature request] +labels: [ feature request ] body: - - type: textarea - id: feature-description - attributes: - label: Describe your suggested feature - description: How can an existing source be improved? - placeholder: | - Example: - "It should work like this..." - validations: - required: true + - type: textarea + id: feature-description + attributes: + label: Describe your suggested feature + description: How can an existing source be improved? + placeholder: | + Example: + "It should work like this..." + Please use English language + validations: + required: true - - type: textarea - id: other-details - attributes: - label: Other details - placeholder: | - Additional details and attachments. + - type: textarea + id: other-details + attributes: + label: Other details + placeholder: | + Additional details and attachments. - - type: checkboxes - id: acknowledgements - attributes: - label: Acknowledgements - options: - - label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue. - required: true \ No newline at end of file + - type: checkboxes + id: acknowledgements + attributes: + label: Acknowledgements + options: + - label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue. + required: true diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/Broken.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/Broken.kt index 24e68479f..74d03077c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/Broken.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/Broken.kt @@ -5,4 +5,10 @@ package org.koitharu.kotatsu.parsers */ @Target(AnnotationTarget.CLASS) @Retention(AnnotationRetention.SOURCE) -internal annotation class Broken +internal annotation class Broken( + + /** + * Reason why this parser is broken + */ + val message: String = "", +) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/Mangarbic.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/Mangarbic.kt index 386d2c653..461beb298 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/Mangarbic.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ar/Mangarbic.kt @@ -1,10 +1,12 @@ package org.koitharu.kotatsu.parsers.site.madara.ar +import org.koitharu.kotatsu.parsers.Broken import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.site.madara.MadaraParser +@Broken("Website is down or domain has been changed") @MangaSourceParser("MANGARBIC", "MangaArabic", "ar") internal class Mangarbic(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.MANGARBIC, "lekmanga.net") { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/MerlinScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/MerlinScans.kt index 49c2256d6..78ad60d87 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/MerlinScans.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/tr/MerlinScans.kt @@ -1,11 +1,13 @@ package org.koitharu.kotatsu.parsers.site.madara.tr +import org.koitharu.kotatsu.parsers.Broken import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.site.madara.MadaraParser //Images blocked by ReCAPTCHA +@Broken // https://github.com/KotatsuApp/kotatsu-parsers/issues/1250 @MangaSourceParser("MERLINSCANS", "MerlinScans", "tr") internal class MerlinScans(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.MERLINSCANS, "merlinscans.com", 10) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/FlameComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/FlameComics.kt index ac037d9b9..d94b72488 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/FlameComics.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/FlameComics.kt @@ -1,12 +1,211 @@ package org.koitharu.kotatsu.parsers.site.mangareader.en +import androidx.collection.ArraySet +import okhttp3.HttpUrl +import org.json.JSONArray +import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser +import org.koitharu.kotatsu.parsers.SinglePageMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.* +import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy +import java.util.* +import java.util.concurrent.TimeUnit @MangaSourceParser("FLAMECOMICS", "FlameComics", "en") internal class FlameComics(context: MangaLoaderContext) : - MangaReaderParser(context, MangaParserSource.FLAMECOMICS, "flamecomics.xyz", pageSize = 24, searchPageSize = 10) { - //override val listUrl = "" + SinglePageMangaParser(context, MangaParserSource.FLAMECOMICS) { + + private val commonPrefix = suspendLazy(initializer = ::fetchCommonPrefix) + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override val configKeyDomain = ConfigKey.Domain("flamecomics.xyz") + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("browse.json") + if (!filter.query.isNullOrEmpty()) { + url.addQueryParameter("search", filter.query) + } + val json = webClient.httpGet(url.build()).parseJson().getJSONObject("pageProps").getJSONArray("series") + return json.mapJSONNotNull { jo -> + parseManga(jo).takeIf { it.tags.matches(filter) } + } + } + + override suspend fun getDetails(manga: Manga): Manga = getDetailsImpl(manga.url.toLong()) + + override suspend fun getPages(chapter: MangaChapter): List { + val (seriesId, token) = chapter.url.split('?') + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("series") + .addPathSegment(seriesId) + .addPathSegment("$token.json") + .addQueryParameter("id", seriesId) + .addQueryParameter("token", token) + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps") + .getJSONObject("chapter") + .getJSONObject("images") + .entries() + return json.map { (i, jo) -> + MangaPage( + id = generateUid("$i|$token"), + url = imageUrl(seriesId, token + "/" + jo.getString("name"), 1920), + preview = imageUrl(seriesId, token + "/" + jo.getString("name"), 128), + source = source, + ) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val seriesId = link.pathSegments.lastOrNull()?.toLongOrNull() ?: return null + return getDetailsImpl(seriesId) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + private suspend fun fetchCommonPrefix(): String { + val raw = webClient.httpGet(urlBuilder().build()).parseRaw() + val regex = Regex("/_next/static/([^/]+)/_buildManifest\\.js") + return checkNotNull(raw.findGroupValue(regex)) { "Unable to find common prefix" } + } + + private fun imageUrl(seriesId: Any, url: String, width: Int) = urlBuilder() + .addPathSegment("_next") + .addPathSegment("image") + .addQueryParameter( + "url", + urlBuilder("cdn") + .addPathSegment("series") + .addPathSegment(seriesId.toString()) + .addPathSegments(url) + .build().toString(), + ) + .addQueryParameter("w", width.toString()) + .addQueryParameter("q", "100") + .build() + .toString() + + private suspend fun fetchAvailableTags(): Set { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("browse.json") + .build() + return webClient.httpGet(url).parseJson() + .getJSONObject("pageProps") + .getJSONArray("series") + .mapJSONNotNull { it.getStringOrNull("categories") } + .flatMapTo(ArraySet()) { + JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } + } + } + + private fun parseManga(jo: JSONObject): Manga { + val seriesId = jo.getLong("series_id") + val cover = jo.getStringOrNull("cover") + return Manga( + id = generateUid(seriesId), + title = jo.getString("title"), + altTitle = jo.getStringOrNull("altTitles")?.let { + JSONArray(it).optString(0) + }, + url = seriesId.toString(), + publicUrl = "https://${domain}/series/$seriesId", + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = if (cover != null) { + imageUrl(seriesId, cover, 256) + } else { + "" + }, + tags = jo.getStringOrNull("categories")?.let { + JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } + }.orEmpty(), + state = when (jo.getStringOrNull("status")) { + "Dropped" -> MangaState.ABANDONED + "Completed" -> MangaState.FINISHED + "Hiatus" -> MangaState.PAUSED + "Ongoing" -> MangaState.ONGOING + else -> null + }, + author = jo.getStringOrNull("author"), + largeCoverUrl = if (cover != null) { + imageUrl(seriesId, cover, 640) + } else { + null + }, + description = jo.getStringOrNull("description"), + source = source, + ) + } + + private suspend fun getDetailsImpl(seriesId: Long): Manga { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("series") + .addPathSegment("$seriesId.json") + .addQueryParameter("id", seriesId.toString()) + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps") + val series = json.getJSONObject("series") + val chapters = json.getJSONArray("chapters") + return parseManga(series).copy( + chapters = chapters.mapJSON { jo -> + val chapterId = jo.getLong("chapter_id") + val number = jo.getFloatOrDefault("chapter", 0f) + MangaChapter( + id = generateUid(longOf(seriesId.toInt(), chapterId.toInt())), + name = jo.getStringOrNull("name") ?: ("Chapter " + number.formatSimple()), + number = number, + volume = 0, + url = seriesId.toString() + "?" + jo.getStringOrNull("token").orEmpty(), + scanlator = null, + uploadDate = TimeUnit.SECONDS.toMillis(jo.getLongOrDefault("release_date", 0L)), + branch = jo.getStringOrNull("language"), + source = source, + ) + }, + ) + } + + private fun Set.matches(filter: MangaListFilter): Boolean { + if (filter.tags.isNotEmpty() && !containsAll(filter.tags)) { + return false + } + for (tag in filter.tagsExclude) { + if (contains(tag)) { + return false + } + } + return true + } + + private fun String.toMangaTag() = MangaTag(this.toTitleCase(sourceLocale), this, source) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Number.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Number.kt index 96becafae..6d4d2232a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Number.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Number.kt @@ -68,3 +68,7 @@ public inline fun Int.ifZero(defaultVale: () -> Int): Int { this } } + +public fun longOf(a: Int, b: Int): Long { + return a.toLong() shl 32 or (b.toLong() and 0xffffffffL) +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 7ca225976..31880af83 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -248,7 +248,9 @@ internal class MangaParserTest { for (item in list) { assert(item.url.isNotEmpty()) { "Url is empty" } assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" } - assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } + if (item.coverUrl.isNotEmpty()) { // TODO nullable cover + assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } + } assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } assert(item.publicUrl.isUrlAbsolute()) }