KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Link resolver implementation

Browse Source
This commit is contained in:
Koitharu
2024-10-13 21:12:36 +03:00
parent 797a91a037
commit 1d040e8291
5 changed files with 204 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt
View File

@@ -1,12 +1,15 @@
package org.koitharu.kotatsu.parsers
import okhttp3.CookieJar
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.Response
import org.koitharu.kotatsu.parsers.bitmap.Bitmap
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.LinkResolver
import java.util.*
public abstract class MangaLoaderContext {
@@ -17,6 +20,10 @@ public abstract class MangaLoaderContext {
public fun newParserInstance(source: MangaParserSource): MangaParser = source.newParser(this)
public fun newLinkResolver(link: HttpUrl): LinkResolver = LinkResolver(this, link)
public fun newLinkResolver(link: String): LinkResolver = newLinkResolver(link.toHttpUrl())
public open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data)
public open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data)

7
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
View File

@@ -2,6 +2,7 @@ package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import okhttp3.Headers
import okhttp3.HttpUrl
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.*
@@ -105,4 +106,10 @@ public abstract class MangaParser @InternalParsersApi constructor(
public open suspend fun getRelatedManga(seed: Manga): List<Manga> {
return RelatedMangaFinder(listOf(this)).invoke(seed)
}
/**
* Return [Manga] object by web link to it
* @see [Manga.publicUrl]
*/
public open suspend fun resolveLink(link: HttpUrl): Manga? = null
}

141
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt
View File

@@ -4,6 +4,8 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import okhttp3.HttpUrl
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
@@ -199,73 +201,28 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
}
val json = webClient.httpGet(url).parseJson().getJSONArray("data")
return json.mapJSON { jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
val relations = jo.getJSONArray("relationships").associateByKey("type")
val cover = relations["cover_art"]
?.getJSONObject("attributes")
?.getString("fileName")
?.let {
"https://uploads.$domain/covers/$id/$it"
}
Manga(
id = generateUid(id),
title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) {
"Title should not be null"
},
altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(),
url = id,
publicUrl = "https://$domain/title/$id",
rating = RATING_UNKNOWN,
isNsfw = when (attrs.getStringOrNull("contentRating")) {
"erotica", "pornographic" -> true
else -> false
},
coverUrl = cover?.plus(".256.jpg").orEmpty(),
largeCoverUrl = cover,
description = attrs.optJSONObject("description")?.selectByLocale(),
tags = attrs.getJSONArray("tags").mapJSONToSet { tag ->
MangaTag(
title = tag.getJSONObject("attributes")
.getJSONObject("name")
.firstStringValue()
.toTitleCase(),
key = tag.getString("id"),
source = source,
)
},
state = when (attrs.getStringOrNull("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
"hiatus" -> MangaState.PAUSED
"cancelled" -> MangaState.ABANDONED
else -> null
},
author = (relations["author"] ?: relations["artist"])
?.getJSONObject("attributes")
?.getStringOrNull("name"),
source = source,
)
}
return json.mapJSON { jo -> jo.fetchManga(null) }
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val domain = domain
override suspend fun getDetails(manga: Manga): Manga {
val mangaId = manga.url.removePrefix("/")
val attrsDeferred = async {
return getDetails(mangaId)
}
override suspend fun resolveLink(link: HttpUrl): Manga? {
val regex = Regex("[0-9a-f\\-]{10,}", RegexOption.IGNORE_CASE)
val mangaId = link.pathSegments.find { regex.matches(it) } ?: return null
return getDetails(mangaId)
}
private suspend fun getDetails(mangaId: String): Manga = coroutineScope {
val jsonDeferred = async {
webClient.httpGet(
"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
).parseJson().getJSONObject("data").getJSONObject("attributes")
).parseJson().getJSONObject("data")
}
val feedDeferred = async { loadChapters(mangaId) }
val mangaAttrs = attrsDeferred.await()
val feed = feedDeferred.await()
manga.copy(
description = mangaAttrs.optJSONObject("description")?.selectByLocale()
?: manga.description,
chapters = mapChapters(feed),
)
jsonDeferred.await().fetchManga(mapChapters(feedDeferred.await()))
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
@@ -312,6 +269,57 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
}
private fun JSONObject.fetchManga(chapters: List<MangaChapter>?): Manga {
val id = getString("id")
val attrs = getJSONObject("attributes")
val relations = getJSONArray("relationships").associateByKey("type")
val cover = relations["cover_art"]
?.getJSONObject("attributes")
?.getString("fileName")
?.let {
"https://uploads.$domain/covers/$id/$it"
}
return Manga(
id = generateUid(id),
title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) {
"Title should not be null"
},
altTitle = attrs.optJSONArray("altTitles")?.flatten()?.selectByLocale(),
url = id,
publicUrl = "https://$domain/title/$id",
rating = RATING_UNKNOWN,
isNsfw = when (attrs.getStringOrNull("contentRating")) {
"erotica", "pornographic" -> true
else -> false
},
coverUrl = cover?.plus(".256.jpg").orEmpty(),
largeCoverUrl = cover,
description = attrs.optJSONObject("description")?.selectByLocale(),
tags = attrs.getJSONArray("tags").mapJSONToSet { tag ->
MangaTag(
title = tag.getJSONObject("attributes")
.getJSONObject("name")
.firstStringValue()
.toTitleCase(),
key = tag.getString("id"),
source = source,
)
},
state = when (attrs.getStringOrNull("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
"hiatus" -> MangaState.PAUSED
"cancelled" -> MangaState.ABANDONED
else -> null
},
author = (relations["author"] ?: relations["artist"])
?.getJSONObject("attributes")
?.getStringOrNull("name"),
chapters = chapters,
source = source,
)
}
private fun JSONObject.firstStringValue() = values().next() as String
private fun JSONObject.selectByLocale(): String? {
@@ -323,6 +331,19 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String
}
private fun JSONArray.flatten(): JSONObject {
val result = JSONObject()
repeat(length()) { i ->
val jo = optJSONObject(i)
if (jo != null) {
for (key in jo.keys()) {
result.put(key, jo.get(key))
}
}
}
return result
}
private suspend fun loadChapters(mangaId: String): List<JSONObject> {
val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE)
if (firstPage.size >= firstPage.total) {

80
src/main/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolver.kt Normal file
View File

@@ -0,0 +1,80 @@
package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.*
public class LinkResolver internal constructor(
private val context: MangaLoaderContext,
public val link: HttpUrl,
) {
private val source = SuspendLazy(::resolveSource)
public suspend fun getSource(): MangaParserSource? = source.get()
public suspend fun getManga(): Manga? {
val parser = context.newParserInstance(source.get() ?: return null)
return parser.resolveLink(link) ?: parser.resolveLinkLongPath()
}
private suspend fun resolveSource(): MangaParserSource? = runInterruptible(Dispatchers.Default) {
val domains = setOfNotNull(link.host, link.topPrivateDomain())
for (s in MangaParserSource.entries) {
val parser = context.newParserInstance(s)
for (d in parser.configKeyDomain.presetValues) {
if (d in domains) {
return@runInterruptible s
}
}
}
null
}
private suspend fun MangaParser.resolveLinkLongPath(): Manga? {
val stubTitle = link.pathSegments.lastOrNull().orEmpty()
val seed = Manga(
id = 0L,
title = stubTitle,
altTitle = null,
url = link.toString().toRelativeUrl(link.host),
publicUrl = link.toString(),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = "",
tags = emptySet(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
chapters = null,
source = source,
).let { manga ->
getDetails(manga)
}
val query = when {
seed.title != stubTitle && seed.title.isNotEmpty() -> seed.title
!seed.altTitle.isNullOrEmpty() -> seed.altTitle
!seed.author.isNullOrEmpty() -> seed.author
else -> return seed // unfortunately we do not know a real manga title so unable to find it
}
return runCatchingCancellable {
val order = if (SortOrder.RELEVANCE in availableSortOrders) SortOrder.RELEVANCE else defaultSortOrder
val list = getList(0, order, MangaListFilter(query = query))
list.single { manga -> isSameUrl(manga.publicUrl) }
}.getOrDefault(seed)
}
private fun isSameUrl(publicUrl: String): Boolean {
if (publicUrl == link.toString()) {
return true
}
val httpUrl = publicUrl.toHttpUrlOrNull() ?: return false
return link.host == httpUrl.host
&& link.encodedPath == httpUrl.encodedPath
}
}

29
src/test/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolverTest.kt Normal file
View File

@@ -0,0 +1,29 @@
package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.test.runTest
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test
import org.koitharu.kotatsu.parsers.MangaLoaderContextMock
import org.koitharu.kotatsu.parsers.model.MangaParserSource
import kotlin.time.Duration.Companion.minutes
internal class LinkResolverTest {
private val context = MangaLoaderContextMock
@Test
fun supportedSource() = runTest(timeout = 2.minutes) {
val resolver = context.newLinkResolver("REDACTED" /* do not publish links to manga on GitHub */)
Assertions.assertEquals(MangaParserSource.MANGADEX, resolver.getSource())
val manga = resolver.getManga()
Assertions.assertEquals(resolver.link.toString(), manga?.publicUrl)
}
@Test
fun unsupportedSource2() = runTest(timeout = 2.minutes) {
val resolver = context.newLinkResolver("REDACTED" /* do not publish links to manga on GitHub */)
Assertions.assertEquals(MangaParserSource.BATOTO, resolver.getSource())
val manga = resolver.getManga()
Assertions.assertEquals(resolver.link.toString(), manga?.publicUrl)
}
}