KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve utils

Browse Source
This commit is contained in:
Koitharu
2024-11-09 19:03:04 +02:00
parent 60f1fb1f70
commit 3b173dc6fc
36 changed files with 265 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/summary.yaml vendored
View File

@@ -1 +1 @@
total: 1152
total: 1152

6
build.gradle
View File

@@ -2,8 +2,8 @@ import tasks.ReportGenerateTask
plugins {
id 'java-library'
id 'org.jetbrains.kotlin.jvm' version '2.0.20'
id 'com.google.devtools.ksp' version '2.0.20-1.0.25'
id 'org.jetbrains.kotlin.jvm' version '2.0.21'
id 'com.google.devtools.ksp' version '2.0.21-1.0.27'
id 'maven-publish'
}
@@ -63,7 +63,7 @@ dependencies {
implementation 'com.squareup.okio:okio:3.9.0'
api 'org.jsoup:jsoup:1.18.1'
implementation 'org.json:json:20240303'
implementation 'androidx.collection:collection:1.4.2'
implementation 'androidx.collection:collection:1.4.5'
ksp project(':kotatsu-parsers-ksp')

2
buildSrc/build.gradle
View File

@@ -1,5 +1,5 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '2.0.20'
id 'org.jetbrains.kotlin.jvm' version '2.0.21'
}
repositories {

2
kotatsu-parsers-ksp/build.gradle
View File

@@ -7,5 +7,5 @@ kotlin {
}
dependencies {
implementation 'com.google.devtools.ksp:symbol-processing-api:2.0.20-1.0.25'
implementation 'com.google.devtools.ksp:symbol-processing-api:2.0.21-1.0.27'
}

24
src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt
View File

@@ -1,6 +1,8 @@
package org.koitharu.kotatsu.parsers.model
import androidx.collection.ArrayMap
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.util.findById
public class Manga(
/**
@@ -76,8 +78,26 @@ public class Manga(
public val hasRating: Boolean
get() = rating > 0f && rating <= 1f
public fun getChapters(branch: String?): List<MangaChapter>? {
return chapters?.filter { x -> x.branch == branch }
public fun getChapters(branch: String?): List<MangaChapter> {
return chapters?.filter { x -> x.branch == branch }.orEmpty()
}
public fun findChapterById(id: Long): MangaChapter? = chapters?.findById(id)
public fun requireChapterById(id: Long): MangaChapter = requireNotNull(findChapterById(id)) {
"Chapter with id $id not found"
}
public fun getBranches(): Map<String?, Int> {
if (chapters.isNullOrEmpty()) {
return emptyMap()
}
val result = ArrayMap<String?, Int>()
chapters.forEach {
val key = it.branch
result[key] = result.getOrDefault(key, 0) + 1
}
return result
}
@InternalParsersApi

14
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt
View File

@@ -1,5 +1,7 @@
package org.koitharu.kotatsu.parsers.model
import org.koitharu.kotatsu.parsers.util.formatSimple
public class MangaChapter(
/**
* An unique id of chapter
@@ -38,6 +40,18 @@ public class MangaChapter(
@JvmField public val source: MangaSource,
) {
public fun numberString(): String? = if (number > 0f) {
number.formatSimple()
} else {
null
}
public fun volumeString(): String? = if (volume > 0) {
volume.toString()
} else {
null
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false

2
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt
View File

@@ -4,7 +4,7 @@ import org.koitharu.kotatsu.parsers.MangaParser
public class MangaPage(
/**
* Unique identifier for manga
* Unique identifier for page
*/
@JvmField public val id: Long,
/**

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ComickFunParser.kt
View File

@@ -12,6 +12,7 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
@@ -256,7 +257,7 @@ internal class ComickFunParser(context: MangaLoaderContext) :
return resolver.resolveManga(this, url = slug, id = generateUid(slug))
}
private val tagsArray = SuspendLazy(::loadTags)
private val tagsArray = suspendLazy(initializer = ::loadTags)
private suspend fun fetchAvailableTags(): Set<MangaTag> {
val sparseArray = tagsArray.get()

89
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
View File

@@ -18,6 +18,7 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.security.MessageDigest
@@ -387,7 +388,7 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
return nozomi
}
private val galleriesIndexVersion = SuspendLazy {
private val galleriesIndexVersion = suspendLazy {
webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw()
}
@@ -480,14 +481,14 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
title = doc.selectFirstOrThrow("h1").text(),
url = id.toString(),
coverUrl =
"https:" +
doc.selectFirstOrThrow("picture > source")
.attr("data-srcset")
.substringBefore(" "),
"https:" +
doc.selectFirstOrThrow("picture > source")
.attr("data-srcset")
.substringBefore(" "),
publicUrl =
doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href")
.toAbsoluteUrl(domain),
doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href")
.toAbsoluteUrl(domain),
author = null,
tags = emptySet(),
isNsfw = true,
@@ -510,37 +511,37 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
return manga.copy(
title = json.getString("title"),
largeCoverUrl =
json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash")
val commonId = commonImageId()
val imageId = imageIdFromHash(hash)
val subDomain = 'a' + subdomainOffset(imageId)
json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash")
val commonId = commonImageId()
val imageId = imageIdFromHash(hash)
val subDomain = 'a' + subdomainOffset(imageId)
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
},
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
},
author =
json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(),
json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(),
publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain),
tags =
buildSet {
json.optJSONArray("characters")
?.mapToTags("character")
?.let(::addAll)
json.optJSONArray("tags")
?.mapToTags("tag")
?.let(::addAll)
json.optJSONArray("artists")
?.mapToTags("artist")
?.let(::addAll)
json.optJSONArray("parodys")
?.mapToTags("parody")
?.let(::addAll)
json.optJSONArray("groups")
?.mapToTags("group")
?.let(::addAll)
},
buildSet {
json.optJSONArray("characters")
?.mapToTags("character")
?.let(::addAll)
json.optJSONArray("tags")
?.mapToTags("tag")
?.let(::addAll)
json.optJSONArray("artists")
?.mapToTags("artist")
?.let(::addAll)
json.optJSONArray("parodys")
?.mapToTags("parody")
?.let(::addAll)
json.optJSONArray("groups")
?.mapToTags("group")
?.let(::addAll)
},
chapters = listOf(
MangaChapter(
id = generateUid(manga.url),
@@ -564,15 +565,15 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
mapJSON {
MangaTag(
title =
it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title ♀"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
"$title ♂"
} else {
title
}
},
it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title ♀"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
"$title ♂"
} else {
title
}
},
key = it.getString("url").tagUrlToTag(),
source = source,
).let(tags::add)

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaFireParser.kt
View File

@@ -16,6 +16,7 @@ import org.koitharu.kotatsu.parsers.bitmap.Rect
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
import kotlin.math.min
@@ -60,7 +61,7 @@ internal abstract class MangaFireParser(
?: body.parseFailed("Cannot find username")
}
private val tags = SoftSuspendLazy {
private val tags = suspendLazy(soft = true) {
webClient.httpGet("https://$domain/filter").parseHtml()
.select(".genres > li").map {
MangaTag(

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaPark.kt
View File

@@ -8,6 +8,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
@@ -152,7 +153,7 @@ internal class MangaPark(context: MangaLoaderContext) :
}
}
private val tagsMap = SuspendLazy(::parseTags)
private val tagsMap = suspendLazy(initializer = ::parseTags)
private suspend fun parseTags(): Map<String, MangaTag> {
val tagElements = webClient.httpGet("https://$domain/search").parseHtml()

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaPlusParser.kt
View File

@@ -18,6 +18,7 @@ import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
internal abstract class MangaPlusParser(
@@ -82,7 +83,7 @@ internal abstract class MangaPlusParser(
}
// since search is local, save network calls on related manga call
private val allTitleCache = SuspendLazy {
private val allTitleCache = suspendLazy {
apiCall("/title_list/allV2")
.getJSONObject("allTitlesViewV2")
.getJSONArray("AllTitlesGroup")

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaReaderToParser.kt
View File

@@ -15,6 +15,7 @@ import org.koitharu.kotatsu.parsers.bitmap.Rect
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
import javax.crypto.Cipher
import javax.crypto.spec.SecretKeySpec
@@ -56,7 +57,7 @@ internal class MangaReaderToParser(context: MangaLoaderContext) :
SortOrder.ALPHABETICAL,
)
val tags = SoftSuspendLazy {
val tags = suspendLazy(soft = true) {
val document = webClient.httpGet("https://$domain/filter").parseHtml()
document.select("div.f-genre-item").map {

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt
View File

@@ -15,6 +15,7 @@ import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
@@ -72,7 +73,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) :
return chain.proceed(newRequest)
}
private val cdnHost = SuspendLazy(::getUpdatedCdnHost)
private val cdnHost = suspendLazy(initializer = ::getUpdatedCdnHost)
private suspend fun getUpdatedCdnHost(): String {
val url = "https://$domain/manga-home"

7
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt
View File

@@ -16,6 +16,7 @@ import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec
@@ -148,12 +149,12 @@ internal abstract class WebtoonsParser(
}
}
private val allGenreCache = SuspendLazy {
private val allGenreCache = suspendLazy {
makeRequest("/lineWebtoon/webtoon/genreList.json").getJSONObject("genreList").getJSONArray("genres")
.mapJSON { jo -> parseTag(jo) }.associateBy { tag -> tag.key }
}
private val allTitleCache = SoftSuspendLazy {
private val allTitleCache = suspendLazy(soft = true) {
makeRequest("/lineWebtoon/webtoon/titleList.json?").getJSONObject("titleList").getJSONArray("titles")
.mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
@@ -333,6 +334,6 @@ internal abstract class WebtoonsParser(
private inner class MangaWebtoon(
val manga: Manga,
@JvmField val date: Long? = null,
@JvmField val readCount: Long? = null,
@JvmField val readCount: Long? = null, // FIXME get rid of boxing
)
}

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Manhwa18Com.kt
View File

@@ -7,6 +7,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
@MangaSourceParser("MANHWA18COM", "Manhwa18.com", "en", type = ContentType.HENTAI)
@@ -222,7 +223,7 @@ internal class Manhwa18Com(context: MangaLoaderContext) :
}
}
private val tagsMap = SuspendLazy(::parseTags)
private val tagsMap = suspendLazy(initializer = ::parseTags)
private suspend fun parseTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml()

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Manhwa18Parser.kt
View File

@@ -7,6 +7,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
@MangaSourceParser("MANHWA18", "Manhwa18.net", "en", type = ContentType.HENTAI)
@@ -222,7 +223,7 @@ internal class Manhwa18Parser(context: MangaLoaderContext) :
}
}
private val tagsMap = SuspendLazy(::parseTags)
private val tagsMap = suspendLazy(initializer = ::parseTags)
private suspend fun parseTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml()

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/en/RizzComic.kt
View File

@@ -12,6 +12,7 @@ import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
@MangaSourceParser("RIZZCOMIC", "RizzComic", "en")
@@ -37,7 +38,7 @@ internal class RizzComic(context: MangaLoaderContext) :
private val filterUrl = "/Index/filter_series"
private val searchUrl = "/Index/live_search"
private var randomPartCache = SuspendLazy(::getRandomPart)
private var randomPartCache = suspendLazy(initializer = ::getRandomPart)
private val randomPartRegex = Regex("""^(r\d+-)""")
private val slugRegex = Regex("""[^a-z0-9]+""")
private val searchMangaSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx"

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt
View File

@@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
@@ -33,7 +34,7 @@ internal abstract class NepnepParser(
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.POPULARITY, SortOrder.UPDATED)
private val searchDoc = SoftSuspendLazy {
private val searchDoc = suspendLazy(soft = true) {
webClient.httpGet("https://$domain/search/").parseHtml()
}

6
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/DesuMeParser.kt
View File

@@ -12,6 +12,8 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.getOrNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
@MangaSourceParser("DESUME", "Desu", "ru")
@@ -41,7 +43,7 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
.add("User-Agent", UserAgents.KOTATSU)
.build()
private val tagsCache = SuspendLazy(::fetchTags)
private val tagsCache = suspendLazy(initializer = ::fetchTags)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
if (!filter.query.isNullOrEmpty() && page != searchPaginator.firstPage) {
@@ -68,7 +70,7 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
?: throw ParseException("Invalid response", url)
val total = json.length()
val list = ArrayList<Manga>(total)
val tagsMap = tagsCache.tryGet().getOrNull()
val tagsMap = tagsCache.getOrNull()
for (i in 0 until total) {
val jo = json.getJSONObject(i)
val cover = jo.getJSONObject("image")

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt
View File

@@ -25,6 +25,7 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
@@ -50,7 +51,7 @@ internal abstract class GroupleParser(
"Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0",
)
private val splitTranslationsKey = ConfigKey.SplitByTranslations(false)
private val tagsIndex = SuspendLazy(::fetchTagsMap)
private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap)
override fun getRequestHeaders(): Headers = Headers.Builder()
.add("User-Agent", config[userAgentKey])

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/rulib/LibSocialParser.kt
View File

@@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
@@ -52,7 +53,7 @@ internal abstract class LibSocialParser(
4, MangaState.PAUSED,
5, MangaState.ABANDONED,
)
private val imageServers = SuspendLazy(::fetchServers)
private val imageServers = suspendLazy(initializer = ::fetchServers)
private val splitTranslationsKey = ConfigKey.SplitByTranslations(true)
private val preferredServerKey = ConfigKey.PreferredImageServer(
presetValues = mapOf(

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkrParser.kt
View File

@@ -16,6 +16,7 @@ import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
@@ -29,7 +30,7 @@ internal class HentaiUkrParser(context: MangaLoaderContext) : MangaParser(contex
private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US)
private val allManga = SoftSuspendLazy {
private val allManga = suspendLazy(soft = true) {
runCatchingCancellable {
webClient.httpGet("https://$domain/search/objects.json").parseJson()
}.recoverCatchingCancellable {

8
src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HoneyMangaParser.kt
View File

@@ -15,6 +15,8 @@ import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault
import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.getOrNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
@@ -35,7 +37,7 @@ internal class HoneyMangaParser(context: MangaLoaderContext) :
private val framesApi get() = "$urlApi/chapter/frames"
private val searchApi get() = "https://search.api.$domain/v2/manga/pattern?query="
private val imageStorageUrl = SuspendLazy(::fetchCoversBaseUrl)
private val imageStorageUrl = suspendLazy(initializer = ::fetchCoversBaseUrl)
override val configKeyDomain = ConfigKey.Domain("honey-manga.com.ua")
@@ -173,7 +175,7 @@ internal class HoneyMangaParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val content = webClient.httpGet("$framesApi/${chapter.url}").parseJson().getJSONObject("resourceIds")
val baseUrl = imageStorageUrl.tryGet().getOrDefault(IMAGE_BASEURL_FALLBACK)
val baseUrl = imageStorageUrl.getOrNull() ?: IMAGE_BASEURL_FALLBACK
return List(content.length()) { i ->
val item = content.getString(i.toString())
MangaPage(id = generateUid(item), "$baseUrl/$item", getCoverUrl(item, 256), source)
@@ -208,7 +210,7 @@ internal class HoneyMangaParser(context: MangaLoaderContext) :
}
private suspend fun getCoverUrl(id: String, w: Int): String {
val baseUrl = imageStorageUrl.tryGet().getOrDefault(IMAGE_BASEURL_FALLBACK)
val baseUrl = imageStorageUrl.getOrNull() ?: IMAGE_BASEURL_FALLBACK
return concatUrl(baseUrl, "$id?optimizer=image&width=$w&height=$w")
}

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/BlogTruyenVN.kt
View File

@@ -4,6 +4,7 @@ import androidx.collection.ArrayMap
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
@@ -11,9 +12,10 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.getOrNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat
import java.util.*
import org.koitharu.kotatsu.parsers.Broken
@Broken
@MangaSourceParser("BLOGTRUYENVN", "BlogTruyenVN", "vi")
@@ -43,7 +45,7 @@ internal class BlogTruyenVN(context: MangaLoaderContext) :
)
private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
private var cacheTags = SuspendLazy(::fetchTags)
private var cacheTags = suspendLazy(initializer = ::fetchTags)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
return when {
@@ -79,7 +81,7 @@ internal class BlogTruyenVN(context: MangaLoaderContext) :
return listElements.mapNotNull { el ->
val linkTag = el.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap ->
val tags = cacheTags.getOrNull()?.let { tagMap ->
el.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
@@ -169,7 +171,7 @@ internal class BlogTruyenVN(context: MangaLoaderContext) :
}
}
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap ->
val tags = cacheTags.getOrNull()?.let { tagMap ->
descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/zh/Baozimh.kt
View File

@@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.*
@MangaSourceParser("BAOZIMH", "Baozimh", "zh")
@@ -42,7 +43,7 @@ internal class Baozimh(context: MangaLoaderContext) :
),
)
private val tagsMap = SuspendLazy(::parseTags)
private val tagsMap = suspendLazy(initializer = ::parseTags)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
when {

2
src/main/kotlin/org/koitharu/kotatsu/parsers/util/Enum.kt
View File

@@ -4,7 +4,7 @@ package org.koitharu.kotatsu.parsers.util
import kotlin.enums.EnumEntries
public fun <E : Enum<E>> EnumEntries<E>.names() = Array(size) { i ->
public fun <E : Enum<E>> EnumEntries<E>.names(): Array<String> = Array(size) { i ->
get(i).name
}

3
src/main/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolver.kt
View File

@@ -7,13 +7,14 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
public class LinkResolver internal constructor(
private val context: MangaLoaderContext,
public val link: HttpUrl,
) {
private val source = SuspendLazy(::resolveSource)
private val source = suspendLazy(initializer = ::resolveSource)
public suspend fun getSource(): MangaParserSource? = source.get()

5
src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParsersUtils.kt
View File

@@ -2,6 +2,7 @@
package org.koitharu.kotatsu.parsers.util
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaListFilter
import kotlin.contracts.contract
@@ -11,3 +12,7 @@ public fun MangaListFilter?.isNullOrEmpty(): Boolean {
}
return this == null || this.isEmpty()
}
public fun Collection<MangaChapter>.findById(chapterId: Long): MangaChapter? = find { x ->
x.id == chapterId
}

1
src/main/kotlin/org/koitharu/kotatsu/parsers/util/Number.kt
View File

@@ -62,7 +62,6 @@ public inline fun Int.ifZero(defaultVale: () -> Int): Int {
contract {
callsInPlace(defaultVale, InvocationKind.AT_MOST_ONCE)
}
@Suppress("KotlinConstantConditions")
return if (this == 0) {
defaultVale()
} else {

37
src/main/kotlin/org/koitharu/kotatsu/parsers/util/SoftSuspendLazy.kt
View File

@@ -1,37 +0,0 @@
package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import java.lang.ref.SoftReference
/**
* Like a [SuspendLazy] but with [SoftReference] under the hood
*/
public class SoftSuspendLazy<T : Any>(
private val initializer: suspend () -> T,
) {
private val mutex = Mutex()
private var cachedValue: SoftReference<T>? = null
public suspend fun get(): T {
// fast way
cachedValue?.get()?.let {
return it
}
return mutex.withLock {
cachedValue?.get()?.let {
return it
}
val result = initializer()
cachedValue = SoftReference(result)
result
}
}
public suspend fun tryGet(): Result<T> = runCatchingCancellable { get() }
public fun peek(): T? {
return cachedValue?.get()
}
}

3
src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt
View File

@@ -2,7 +2,6 @@
package org.koitharu.kotatsu.parsers.util
import androidx.annotation.FloatRange
import androidx.collection.MutableIntList
import androidx.collection.arraySetOf
import java.math.BigInteger
@@ -225,7 +224,7 @@ public fun String.levenshteinDistance(other: String): Int {
/**
* @param threshold 0 = exact match
*/
public fun String.almostEquals(other: String, @FloatRange(from = 0.0) threshold: Float): Boolean {
public fun String.almostEquals(other: String, threshold: Float): Boolean {
if (threshold <= 0f) {
return equals(other, ignoreCase = true)
}

41
src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt
View File

@@ -1,41 +0,0 @@
package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
public class SuspendLazy<T>(
private val initializer: suspend () -> T,
) {
private val mutex = Mutex()
private var cachedValue: Any? = Uninitialized
@Suppress("UNCHECKED_CAST")
public suspend fun get(): T {
// fast way
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
return mutex.withLock {
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
val result = initializer()
cachedValue = result
result
}
}
public suspend fun tryGet(): Result<T> = runCatchingCancellable { get() }
@Suppress("UNCHECKED_CAST")
public fun peek(): T? {
return cachedValue?.takeUnless { it === Uninitialized } as T?
}
private object Uninitialized
}

41
src/main/kotlin/org/koitharu/kotatsu/parsers/util/suspendlazy/SoftSuspendLazyImpl.kt Normal file
View File

@@ -0,0 +1,41 @@
package org.koitharu.kotatsu.parsers.util.suspendlazy
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import java.lang.ref.SoftReference
import kotlin.coroutines.CoroutineContext
/**
* Like a [SuspendLazy] but with [SoftReference] under the hood
*/
internal class SoftSuspendLazyImpl<T : Any>(
private val coroutineContext: CoroutineContext,
private val initializer: SuspendLazyInitializer<T>,
) : SuspendLazy<T> {
private val mutex: Mutex = Mutex()
private var cachedValue: SoftReference<T>? = null
override val isInitialized: Boolean
get() = cachedValue?.get() != null
override suspend fun get(): T {
// fast way
cachedValue?.get()?.let {
return it
}
return mutex.withLock {
cachedValue?.get()?.let {
return it
}
val result = withContext(coroutineContext) {
initializer()
}
cachedValue = SoftReference(result)
result
}
}
override fun peek(): T? = cachedValue?.get()
}

33
src/main/kotlin/org/koitharu/kotatsu/parsers/util/suspendlazy/SuspendLazy.kt Normal file
View File

@@ -0,0 +1,33 @@
package org.koitharu.kotatsu.parsers.util.suspendlazy
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
internal typealias SuspendLazyInitializer<T> = suspend () -> T
public interface SuspendLazy<T> {
public val isInitialized: Boolean
public suspend fun get(): T
public fun peek(): T?
}
public suspend fun <T> SuspendLazy<T>.getOrNull(): T? = runCatchingCancellable { get() }.getOrNull()
public fun <T> suspendLazy(
context: CoroutineContext = EmptyCoroutineContext,
initializer: SuspendLazyInitializer<T>,
): SuspendLazy<T> = SuspendLazyImpl(context, initializer)
public fun <T : Any> suspendLazy(
context: CoroutineContext = EmptyCoroutineContext,
soft: Boolean,
initializer: SuspendLazyInitializer<T>,
): SuspendLazy<T> = if (soft) {
SoftSuspendLazyImpl(context, initializer)
} else {
SuspendLazyImpl(context, initializer)
}

47
src/main/kotlin/org/koitharu/kotatsu/parsers/util/suspendlazy/SuspendLazyImpl.kt Normal file
View File

@@ -0,0 +1,47 @@
package org.koitharu.kotatsu.parsers.util.suspendlazy
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlin.coroutines.CoroutineContext
internal class SuspendLazyImpl<T>(
private val coroutineContext: CoroutineContext,
private val initializer: SuspendLazyInitializer<T>,
) : SuspendLazy<T> {
private val mutex: Mutex = Mutex()
private var cachedValue: Any? = Uninitialized
override val isInitialized: Boolean
get() = cachedValue !== Uninitialized
@Suppress("UNCHECKED_CAST")
override suspend fun get(): T {
// fast way
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
return mutex.withLock {
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
val result = withContext(coroutineContext) {
initializer()
}
cachedValue = result
result
}
}
@Suppress("UNCHECKED_CAST")
override fun peek(): T? {
return cachedValue?.takeUnless { it === Uninitialized } as T?
}
private object Uninitialized
}