diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt index 15510f29..05684178 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt @@ -1,12 +1,15 @@ package org.koitharu.kotatsu.parsers import okhttp3.CookieJar +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient import okhttp3.Response import org.koitharu.kotatsu.parsers.bitmap.Bitmap import org.koitharu.kotatsu.parsers.config.MangaSourceConfig import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.LinkResolver import java.util.* public abstract class MangaLoaderContext { @@ -17,6 +20,10 @@ public abstract class MangaLoaderContext { public fun newParserInstance(source: MangaParserSource): MangaParser = source.newParser(this) + public fun newLinkResolver(link: HttpUrl): LinkResolver = LinkResolver(this, link) + + public fun newLinkResolver(link: String): LinkResolver = newLinkResolver(link.toHttpUrl()) + public open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data) public open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 9986a201..3c78d855 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -2,6 +2,7 @@ package org.koitharu.kotatsu.parsers import androidx.annotation.CallSuper import okhttp3.Headers +import okhttp3.HttpUrl import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.MangaSourceConfig import org.koitharu.kotatsu.parsers.model.* @@ -105,4 +106,10 @@ public abstract class MangaParser @InternalParsersApi constructor( public open suspend fun getRelatedManga(seed: Manga): List { return RelatedMangaFinder(listOf(this)).invoke(seed) } + + /** + * Return [Manga] object by web link to it + * @see [Manga.publicUrl] + */ + public open suspend fun resolveLink(link: HttpUrl): Manga? = null } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt index 391d09a9..ee59270a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt @@ -4,6 +4,8 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl +import org.json.JSONArray import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser @@ -199,73 +201,28 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context } } val json = webClient.httpGet(url).parseJson().getJSONArray("data") - return json.mapJSON { jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - val relations = jo.getJSONArray("relationships").associateByKey("type") - val cover = relations["cover_art"] - ?.getJSONObject("attributes") - ?.getString("fileName") - ?.let { - "https://uploads.$domain/covers/$id/$it" - } - Manga( - id = generateUid(id), - title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { - "Title should not be null" - }, - altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), - url = id, - publicUrl = "https://$domain/title/$id", - rating = RATING_UNKNOWN, - isNsfw = when (attrs.getStringOrNull("contentRating")) { - "erotica", "pornographic" -> true - else -> false - }, - coverUrl = cover?.plus(".256.jpg").orEmpty(), - largeCoverUrl = cover, - description = attrs.optJSONObject("description")?.selectByLocale(), - tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> - MangaTag( - title = tag.getJSONObject("attributes") - .getJSONObject("name") - .firstStringValue() - .toTitleCase(), - key = tag.getString("id"), - source = source, - ) - }, - state = when (attrs.getStringOrNull("status")) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "cancelled" -> MangaState.ABANDONED - else -> null - }, - author = (relations["author"] ?: relations["artist"]) - ?.getJSONObject("attributes") - ?.getStringOrNull("name"), - source = source, - ) - } + return json.mapJSON { jo -> jo.fetchManga(null) } } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val domain = domain + override suspend fun getDetails(manga: Manga): Manga { val mangaId = manga.url.removePrefix("/") - val attrsDeferred = async { + return getDetails(mangaId) + } + + override suspend fun resolveLink(link: HttpUrl): Manga? { + val regex = Regex("[0-9a-f\\-]{10,}", RegexOption.IGNORE_CASE) + val mangaId = link.pathSegments.find { regex.matches(it) } ?: return null + return getDetails(mangaId) + } + + private suspend fun getDetails(mangaId: String): Manga = coroutineScope { + val jsonDeferred = async { webClient.httpGet( "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", - ).parseJson().getJSONObject("data").getJSONObject("attributes") + ).parseJson().getJSONObject("data") } val feedDeferred = async { loadChapters(mangaId) } - val mangaAttrs = attrsDeferred.await() - val feed = feedDeferred.await() - manga.copy( - description = mangaAttrs.optJSONObject("description")?.selectByLocale() - ?: manga.description, - chapters = mapChapters(feed), - ) + jsonDeferred.await().fetchManga(mapChapters(feedDeferred.await())) } override suspend fun getPages(chapter: MangaChapter): List { @@ -312,6 +269,57 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context } } + private fun JSONObject.fetchManga(chapters: List?): Manga { + val id = getString("id") + val attrs = getJSONObject("attributes") + val relations = getJSONArray("relationships").associateByKey("type") + val cover = relations["cover_art"] + ?.getJSONObject("attributes") + ?.getString("fileName") + ?.let { + "https://uploads.$domain/covers/$id/$it" + } + return Manga( + id = generateUid(id), + title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { + "Title should not be null" + }, + altTitle = attrs.optJSONArray("altTitles")?.flatten()?.selectByLocale(), + url = id, + publicUrl = "https://$domain/title/$id", + rating = RATING_UNKNOWN, + isNsfw = when (attrs.getStringOrNull("contentRating")) { + "erotica", "pornographic" -> true + else -> false + }, + coverUrl = cover?.plus(".256.jpg").orEmpty(), + largeCoverUrl = cover, + description = attrs.optJSONObject("description")?.selectByLocale(), + tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> + MangaTag( + title = tag.getJSONObject("attributes") + .getJSONObject("name") + .firstStringValue() + .toTitleCase(), + key = tag.getString("id"), + source = source, + ) + }, + state = when (attrs.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + }, + author = (relations["author"] ?: relations["artist"]) + ?.getJSONObject("attributes") + ?.getStringOrNull("name"), + chapters = chapters, + source = source, + ) + } + private fun JSONObject.firstStringValue() = values().next() as String private fun JSONObject.selectByLocale(): String? { @@ -323,6 +331,19 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String } + private fun JSONArray.flatten(): JSONObject { + val result = JSONObject() + repeat(length()) { i -> + val jo = optJSONObject(i) + if (jo != null) { + for (key in jo.keys()) { + result.put(key, jo.get(key)) + } + } + } + return result + } + private suspend fun loadChapters(mangaId: String): List { val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) if (firstPage.size >= firstPage.total) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolver.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolver.kt new file mode 100644 index 00000000..a328c3df --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolver.kt @@ -0,0 +1,80 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runInterruptible +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.model.* + +public class LinkResolver internal constructor( + private val context: MangaLoaderContext, + public val link: HttpUrl, +) { + + private val source = SuspendLazy(::resolveSource) + + public suspend fun getSource(): MangaParserSource? = source.get() + + public suspend fun getManga(): Manga? { + val parser = context.newParserInstance(source.get() ?: return null) + return parser.resolveLink(link) ?: parser.resolveLinkLongPath() + } + + private suspend fun resolveSource(): MangaParserSource? = runInterruptible(Dispatchers.Default) { + val domains = setOfNotNull(link.host, link.topPrivateDomain()) + for (s in MangaParserSource.entries) { + val parser = context.newParserInstance(s) + for (d in parser.configKeyDomain.presetValues) { + if (d in domains) { + return@runInterruptible s + } + } + } + null + } + + private suspend fun MangaParser.resolveLinkLongPath(): Manga? { + val stubTitle = link.pathSegments.lastOrNull().orEmpty() + val seed = Manga( + id = 0L, + title = stubTitle, + altTitle = null, + url = link.toString().toRelativeUrl(link.host), + publicUrl = link.toString(), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = "", + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + chapters = null, + source = source, + ).let { manga -> + getDetails(manga) + } + val query = when { + seed.title != stubTitle && seed.title.isNotEmpty() -> seed.title + !seed.altTitle.isNullOrEmpty() -> seed.altTitle + !seed.author.isNullOrEmpty() -> seed.author + else -> return seed // unfortunately we do not know a real manga title so unable to find it + } + return runCatchingCancellable { + val order = if (SortOrder.RELEVANCE in availableSortOrders) SortOrder.RELEVANCE else defaultSortOrder + val list = getList(0, order, MangaListFilter(query = query)) + list.single { manga -> isSameUrl(manga.publicUrl) } + }.getOrDefault(seed) + } + + private fun isSameUrl(publicUrl: String): Boolean { + if (publicUrl == link.toString()) { + return true + } + val httpUrl = publicUrl.toHttpUrlOrNull() ?: return false + return link.host == httpUrl.host + && link.encodedPath == httpUrl.encodedPath + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolverTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolverTest.kt new file mode 100644 index 00000000..78bf4e5c --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolverTest.kt @@ -0,0 +1,29 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.MangaLoaderContextMock +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import kotlin.time.Duration.Companion.minutes + +internal class LinkResolverTest { + + private val context = MangaLoaderContextMock + + @Test + fun supportedSource() = runTest(timeout = 2.minutes) { + val resolver = context.newLinkResolver("REDACTED" /* do not publish links to manga on GitHub */) + Assertions.assertEquals(MangaParserSource.MANGADEX, resolver.getSource()) + val manga = resolver.getManga() + Assertions.assertEquals(resolver.link.toString(), manga?.publicUrl) + } + + @Test + fun unsupportedSource2() = runTest(timeout = 2.minutes) { + val resolver = context.newLinkResolver("REDACTED" /* do not publish links to manga on GitHub */) + Assertions.assertEquals(MangaParserSource.BATOTO, resolver.getSource()) + val manga = resolver.getManga() + Assertions.assertEquals(resolver.link.toString(), manga?.publicUrl) + } +}