Improve utils

master
Koitharu 1 year ago
parent 60f1fb1f70
commit 3b173dc6fc
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7

@ -2,8 +2,8 @@ import tasks.ReportGenerateTask
plugins { plugins {
id 'java-library' id 'java-library'
id 'org.jetbrains.kotlin.jvm' version '2.0.20' id 'org.jetbrains.kotlin.jvm' version '2.0.21'
id 'com.google.devtools.ksp' version '2.0.20-1.0.25' id 'com.google.devtools.ksp' version '2.0.21-1.0.27'
id 'maven-publish' id 'maven-publish'
} }
@ -63,7 +63,7 @@ dependencies {
implementation 'com.squareup.okio:okio:3.9.0' implementation 'com.squareup.okio:okio:3.9.0'
api 'org.jsoup:jsoup:1.18.1' api 'org.jsoup:jsoup:1.18.1'
implementation 'org.json:json:20240303' implementation 'org.json:json:20240303'
implementation 'androidx.collection:collection:1.4.2' implementation 'androidx.collection:collection:1.4.5'
ksp project(':kotatsu-parsers-ksp') ksp project(':kotatsu-parsers-ksp')

@ -1,5 +1,5 @@
plugins { plugins {
id 'org.jetbrains.kotlin.jvm' version '2.0.20' id 'org.jetbrains.kotlin.jvm' version '2.0.21'
} }
repositories { repositories {

@ -7,5 +7,5 @@ kotlin {
} }
dependencies { dependencies {
implementation 'com.google.devtools.ksp:symbol-processing-api:2.0.20-1.0.25' implementation 'com.google.devtools.ksp:symbol-processing-api:2.0.21-1.0.27'
} }

@ -1,6 +1,8 @@
package org.koitharu.kotatsu.parsers.model package org.koitharu.kotatsu.parsers.model
import androidx.collection.ArrayMap
import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.util.findById
public class Manga( public class Manga(
/** /**
@ -76,8 +78,26 @@ public class Manga(
public val hasRating: Boolean public val hasRating: Boolean
get() = rating > 0f && rating <= 1f get() = rating > 0f && rating <= 1f
public fun getChapters(branch: String?): List<MangaChapter>? { public fun getChapters(branch: String?): List<MangaChapter> {
return chapters?.filter { x -> x.branch == branch } return chapters?.filter { x -> x.branch == branch }.orEmpty()
}
public fun findChapterById(id: Long): MangaChapter? = chapters?.findById(id)
public fun requireChapterById(id: Long): MangaChapter = requireNotNull(findChapterById(id)) {
"Chapter with id $id not found"
}
public fun getBranches(): Map<String?, Int> {
if (chapters.isNullOrEmpty()) {
return emptyMap()
}
val result = ArrayMap<String?, Int>()
chapters.forEach {
val key = it.branch
result[key] = result.getOrDefault(key, 0) + 1
}
return result
} }
@InternalParsersApi @InternalParsersApi

@ -1,5 +1,7 @@
package org.koitharu.kotatsu.parsers.model package org.koitharu.kotatsu.parsers.model
import org.koitharu.kotatsu.parsers.util.formatSimple
public class MangaChapter( public class MangaChapter(
/** /**
* An unique id of chapter * An unique id of chapter
@ -38,6 +40,18 @@ public class MangaChapter(
@JvmField public val source: MangaSource, @JvmField public val source: MangaSource,
) { ) {
public fun numberString(): String? = if (number > 0f) {
number.formatSimple()
} else {
null
}
public fun volumeString(): String? = if (volume > 0) {
volume.toString()
} else {
null
}
override fun equals(other: Any?): Boolean { override fun equals(other: Any?): Boolean {
if (this === other) return true if (this === other) return true
if (javaClass != other?.javaClass) return false if (javaClass != other?.javaClass) return false

@ -4,7 +4,7 @@ import org.koitharu.kotatsu.parsers.MangaParser
public class MangaPage( public class MangaPage(
/** /**
* Unique identifier for manga * Unique identifier for page
*/ */
@JvmField public val id: Long, @JvmField public val id: Long,
/** /**

@ -12,6 +12,7 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.* import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -256,7 +257,7 @@ internal class ComickFunParser(context: MangaLoaderContext) :
return resolver.resolveManga(this, url = slug, id = generateUid(slug)) return resolver.resolveManga(this, url = slug, id = generateUid(slug))
} }
private val tagsArray = SuspendLazy(::loadTags) private val tagsArray = suspendLazy(initializer = ::loadTags)
private suspend fun fetchAvailableTags(): Set<MangaTag> { private suspend fun fetchAvailableTags(): Set<MangaTag> {
val sparseArray = tagsArray.get() val sparseArray = tagsArray.get()

@ -18,6 +18,7 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.nio.ByteBuffer import java.nio.ByteBuffer
import java.nio.ByteOrder import java.nio.ByteOrder
import java.security.MessageDigest import java.security.MessageDigest
@ -387,7 +388,7 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
return nozomi return nozomi
} }
private val galleriesIndexVersion = SuspendLazy { private val galleriesIndexVersion = suspendLazy {
webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw()
} }
@ -480,14 +481,14 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
title = doc.selectFirstOrThrow("h1").text(), title = doc.selectFirstOrThrow("h1").text(),
url = id.toString(), url = id.toString(),
coverUrl = coverUrl =
"https:" + "https:" +
doc.selectFirstOrThrow("picture > source") doc.selectFirstOrThrow("picture > source")
.attr("data-srcset") .attr("data-srcset")
.substringBefore(" "), .substringBefore(" "),
publicUrl = publicUrl =
doc.selectFirstOrThrow("h1 > a") doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href") .attrAsRelativeUrl("href")
.toAbsoluteUrl(domain), .toAbsoluteUrl(domain),
author = null, author = null,
tags = emptySet(), tags = emptySet(),
isNsfw = true, isNsfw = true,
@ -510,37 +511,37 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
return manga.copy( return manga.copy(
title = json.getString("title"), title = json.getString("title"),
largeCoverUrl = largeCoverUrl =
json.getJSONArray("files").getJSONObject(0).let { json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash") val hash = it.getString("hash")
val commonId = commonImageId() val commonId = commonImageId()
val imageId = imageIdFromHash(hash) val imageId = imageIdFromHash(hash)
val subDomain = 'a' + subdomainOffset(imageId) val subDomain = 'a' + subdomainOffset(imageId)
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp" "https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
}, },
author = author =
json.optJSONArray("artists") json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() } ?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(), ?.joinToString(),
publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain), publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain),
tags = tags =
buildSet { buildSet {
json.optJSONArray("characters") json.optJSONArray("characters")
?.mapToTags("character") ?.mapToTags("character")
?.let(::addAll) ?.let(::addAll)
json.optJSONArray("tags") json.optJSONArray("tags")
?.mapToTags("tag") ?.mapToTags("tag")
?.let(::addAll) ?.let(::addAll)
json.optJSONArray("artists") json.optJSONArray("artists")
?.mapToTags("artist") ?.mapToTags("artist")
?.let(::addAll) ?.let(::addAll)
json.optJSONArray("parodys") json.optJSONArray("parodys")
?.mapToTags("parody") ?.mapToTags("parody")
?.let(::addAll) ?.let(::addAll)
json.optJSONArray("groups") json.optJSONArray("groups")
?.mapToTags("group") ?.mapToTags("group")
?.let(::addAll) ?.let(::addAll)
}, },
chapters = listOf( chapters = listOf(
MangaChapter( MangaChapter(
id = generateUid(manga.url), id = generateUid(manga.url),
@ -564,15 +565,15 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
mapJSON { mapJSON {
MangaTag( MangaTag(
title = title =
it.getString(key).toCamelCase().let { title -> it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) { if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title" "$title"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
"$title" "$title"
} else { } else {
title title
} }
}, },
key = it.getString("url").tagUrlToTag(), key = it.getString("url").tagUrlToTag(),
source = source, source = source,
).let(tags::add) ).let(tags::add)

@ -16,6 +16,7 @@ import org.koitharu.kotatsu.parsers.bitmap.Rect
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
import kotlin.math.min import kotlin.math.min
@ -60,7 +61,7 @@ internal abstract class MangaFireParser(
?: body.parseFailed("Cannot find username") ?: body.parseFailed("Cannot find username")
} }
private val tags = SoftSuspendLazy { private val tags = suspendLazy(soft = true) {
webClient.httpGet("https://$domain/filter").parseHtml() webClient.httpGet("https://$domain/filter").parseHtml()
.select(".genres > li").map { .select(".genres > li").map {
MangaTag( MangaTag(

@ -8,6 +8,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.DateFormat import java.text.DateFormat
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -152,7 +153,7 @@ internal class MangaPark(context: MangaLoaderContext) :
} }
} }
private val tagsMap = SuspendLazy(::parseTags) private val tagsMap = suspendLazy(initializer = ::parseTags)
private suspend fun parseTags(): Map<String, MangaTag> { private suspend fun parseTags(): Map<String, MangaTag> {
val tagElements = webClient.httpGet("https://$domain/search").parseHtml() val tagElements = webClient.httpGet("https://$domain/search").parseHtml()

@ -18,6 +18,7 @@ import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
internal abstract class MangaPlusParser( internal abstract class MangaPlusParser(
@ -82,7 +83,7 @@ internal abstract class MangaPlusParser(
} }
// since search is local, save network calls on related manga call // since search is local, save network calls on related manga call
private val allTitleCache = SuspendLazy { private val allTitleCache = suspendLazy {
apiCall("/title_list/allV2") apiCall("/title_list/allV2")
.getJSONObject("allTitlesViewV2") .getJSONObject("allTitlesViewV2")
.getJSONArray("AllTitlesGroup") .getJSONArray("AllTitlesGroup")

@ -15,6 +15,7 @@ import org.koitharu.kotatsu.parsers.bitmap.Rect
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
import javax.crypto.Cipher import javax.crypto.Cipher
import javax.crypto.spec.SecretKeySpec import javax.crypto.spec.SecretKeySpec
@ -56,7 +57,7 @@ internal class MangaReaderToParser(context: MangaLoaderContext) :
SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL,
) )
val tags = SoftSuspendLazy { val tags = suspendLazy(soft = true) {
val document = webClient.httpGet("https://$domain/filter").parseHtml() val document = webClient.httpGet("https://$domain/filter").parseHtml()
document.select("div.f-genre-item").map { document.select("div.f-genre-item").map {

@ -15,6 +15,7 @@ import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -72,7 +73,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) :
return chain.proceed(newRequest) return chain.proceed(newRequest)
} }
private val cdnHost = SuspendLazy(::getUpdatedCdnHost) private val cdnHost = suspendLazy(initializer = ::getUpdatedCdnHost)
private suspend fun getUpdatedCdnHost(): String { private suspend fun getUpdatedCdnHost(): String {
val url = "https://$domain/manga-home" val url = "https://$domain/manga-home"

@ -16,6 +16,7 @@ import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.* import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
import javax.crypto.Mac import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec import javax.crypto.spec.SecretKeySpec
@ -148,12 +149,12 @@ internal abstract class WebtoonsParser(
} }
} }
private val allGenreCache = SuspendLazy { private val allGenreCache = suspendLazy {
makeRequest("/lineWebtoon/webtoon/genreList.json").getJSONObject("genreList").getJSONArray("genres") makeRequest("/lineWebtoon/webtoon/genreList.json").getJSONObject("genreList").getJSONArray("genres")
.mapJSON { jo -> parseTag(jo) }.associateBy { tag -> tag.key } .mapJSON { jo -> parseTag(jo) }.associateBy { tag -> tag.key }
} }
private val allTitleCache = SoftSuspendLazy { private val allTitleCache = suspendLazy(soft = true) {
makeRequest("/lineWebtoon/webtoon/titleList.json?").getJSONObject("titleList").getJSONArray("titles") makeRequest("/lineWebtoon/webtoon/titleList.json?").getJSONObject("titleList").getJSONArray("titles")
.mapJSON { jo -> .mapJSON { jo ->
val titleNo = jo.getLong("titleNo") val titleNo = jo.getLong("titleNo")
@ -333,6 +334,6 @@ internal abstract class WebtoonsParser(
private inner class MangaWebtoon( private inner class MangaWebtoon(
val manga: Manga, val manga: Manga,
@JvmField val date: Long? = null, @JvmField val date: Long? = null,
@JvmField val readCount: Long? = null, @JvmField val readCount: Long? = null, // FIXME get rid of boxing
) )
} }

@ -7,6 +7,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
@MangaSourceParser("MANHWA18COM", "Manhwa18.com", "en", type = ContentType.HENTAI) @MangaSourceParser("MANHWA18COM", "Manhwa18.com", "en", type = ContentType.HENTAI)
@ -222,7 +223,7 @@ internal class Manhwa18Com(context: MangaLoaderContext) :
} }
} }
private val tagsMap = SuspendLazy(::parseTags) private val tagsMap = suspendLazy(initializer = ::parseTags)
private suspend fun parseTags(): Map<String, MangaTag> { private suspend fun parseTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml()

@ -7,6 +7,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
@MangaSourceParser("MANHWA18", "Manhwa18.net", "en", type = ContentType.HENTAI) @MangaSourceParser("MANHWA18", "Manhwa18.net", "en", type = ContentType.HENTAI)
@ -222,7 +223,7 @@ internal class Manhwa18Parser(context: MangaLoaderContext) :
} }
} }
private val tagsMap = SuspendLazy(::parseTags) private val tagsMap = suspendLazy(initializer = ::parseTags)
private suspend fun parseTags(): Map<String, MangaTag> { private suspend fun parseTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml()

@ -12,6 +12,7 @@ import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
@MangaSourceParser("RIZZCOMIC", "RizzComic", "en") @MangaSourceParser("RIZZCOMIC", "RizzComic", "en")
@ -37,7 +38,7 @@ internal class RizzComic(context: MangaLoaderContext) :
private val filterUrl = "/Index/filter_series" private val filterUrl = "/Index/filter_series"
private val searchUrl = "/Index/live_search" private val searchUrl = "/Index/live_search"
private var randomPartCache = SuspendLazy(::getRandomPart) private var randomPartCache = suspendLazy(initializer = ::getRandomPart)
private val randomPartRegex = Regex("""^(r\d+-)""") private val randomPartRegex = Regex("""^(r\d+-)""")
private val slugRegex = Regex("""[^a-z0-9]+""") private val slugRegex = Regex("""[^a-z0-9]+""")
private val searchMangaSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" private val searchMangaSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx"

@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.asTypedList import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -33,7 +34,7 @@ internal abstract class NepnepParser(
override val availableSortOrders: Set<SortOrder> = override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.POPULARITY, SortOrder.UPDATED) EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.POPULARITY, SortOrder.UPDATED)
private val searchDoc = SoftSuspendLazy { private val searchDoc = suspendLazy(soft = true) {
webClient.httpGet("https://$domain/search/").parseHtml() webClient.httpGet("https://$domain/search/").parseHtml()
} }

@ -12,6 +12,8 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.* import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.getOrNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
@MangaSourceParser("DESUME", "Desu", "ru") @MangaSourceParser("DESUME", "Desu", "ru")
@ -41,7 +43,7 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
.add("User-Agent", UserAgents.KOTATSU) .add("User-Agent", UserAgents.KOTATSU)
.build() .build()
private val tagsCache = SuspendLazy(::fetchTags) private val tagsCache = suspendLazy(initializer = ::fetchTags)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
if (!filter.query.isNullOrEmpty() && page != searchPaginator.firstPage) { if (!filter.query.isNullOrEmpty() && page != searchPaginator.firstPage) {
@ -68,7 +70,7 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
?: throw ParseException("Invalid response", url) ?: throw ParseException("Invalid response", url)
val total = json.length() val total = json.length()
val list = ArrayList<Manga>(total) val list = ArrayList<Manga>(total)
val tagsMap = tagsCache.tryGet().getOrNull() val tagsMap = tagsCache.getOrNull()
for (i in 0 until total) { for (i in 0 until total) {
val jo = json.getJSONObject(i) val jo = json.getJSONObject(i)
val cover = jo.getJSONObject("image") val cover = jo.getJSONObject("image")

@ -25,6 +25,7 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -50,7 +51,7 @@ internal abstract class GroupleParser(
"Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0",
) )
private val splitTranslationsKey = ConfigKey.SplitByTranslations(false) private val splitTranslationsKey = ConfigKey.SplitByTranslations(false)
private val tagsIndex = SuspendLazy(::fetchTagsMap) private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap)
override fun getRequestHeaders(): Headers = Headers.Builder() override fun getRequestHeaders(): Headers = Headers.Builder()
.add("User-Agent", config[userAgentKey]) .add("User-Agent", config[userAgentKey])

@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.* import org.koitharu.kotatsu.parsers.util.json.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -52,7 +53,7 @@ internal abstract class LibSocialParser(
4, MangaState.PAUSED, 4, MangaState.PAUSED,
5, MangaState.ABANDONED, 5, MangaState.ABANDONED,
) )
private val imageServers = SuspendLazy(::fetchServers) private val imageServers = suspendLazy(initializer = ::fetchServers)
private val splitTranslationsKey = ConfigKey.SplitByTranslations(true) private val splitTranslationsKey = ConfigKey.SplitByTranslations(true)
private val preferredServerKey = ConfigKey.PreferredImageServer( private val preferredServerKey = ConfigKey.PreferredImageServer(
presetValues = mapOf( presetValues = mapOf(

@ -16,6 +16,7 @@ import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.asTypedList import org.koitharu.kotatsu.parsers.util.json.asTypedList
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -29,7 +30,7 @@ internal class HentaiUkrParser(context: MangaLoaderContext) : MangaParser(contex
private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US) private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US)
private val allManga = SoftSuspendLazy { private val allManga = suspendLazy(soft = true) {
runCatchingCancellable { runCatchingCancellable {
webClient.httpGet("https://$domain/search/objects.json").parseJson() webClient.httpGet("https://$domain/search/objects.json").parseJson()
}.recoverCatchingCancellable { }.recoverCatchingCancellable {

@ -15,6 +15,8 @@ import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault
import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.getOrNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -35,7 +37,7 @@ internal class HoneyMangaParser(context: MangaLoaderContext) :
private val framesApi get() = "$urlApi/chapter/frames" private val framesApi get() = "$urlApi/chapter/frames"
private val searchApi get() = "https://search.api.$domain/v2/manga/pattern?query=" private val searchApi get() = "https://search.api.$domain/v2/manga/pattern?query="
private val imageStorageUrl = SuspendLazy(::fetchCoversBaseUrl) private val imageStorageUrl = suspendLazy(initializer = ::fetchCoversBaseUrl)
override val configKeyDomain = ConfigKey.Domain("honey-manga.com.ua") override val configKeyDomain = ConfigKey.Domain("honey-manga.com.ua")
@ -173,7 +175,7 @@ internal class HoneyMangaParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val content = webClient.httpGet("$framesApi/${chapter.url}").parseJson().getJSONObject("resourceIds") val content = webClient.httpGet("$framesApi/${chapter.url}").parseJson().getJSONObject("resourceIds")
val baseUrl = imageStorageUrl.tryGet().getOrDefault(IMAGE_BASEURL_FALLBACK) val baseUrl = imageStorageUrl.getOrNull() ?: IMAGE_BASEURL_FALLBACK
return List(content.length()) { i -> return List(content.length()) { i ->
val item = content.getString(i.toString()) val item = content.getString(i.toString())
MangaPage(id = generateUid(item), "$baseUrl/$item", getCoverUrl(item, 256), source) MangaPage(id = generateUid(item), "$baseUrl/$item", getCoverUrl(item, 256), source)
@ -208,7 +210,7 @@ internal class HoneyMangaParser(context: MangaLoaderContext) :
} }
private suspend fun getCoverUrl(id: String, w: Int): String { private suspend fun getCoverUrl(id: String, w: Int): String {
val baseUrl = imageStorageUrl.tryGet().getOrDefault(IMAGE_BASEURL_FALLBACK) val baseUrl = imageStorageUrl.getOrNull() ?: IMAGE_BASEURL_FALLBACK
return concatUrl(baseUrl, "$id?optimizer=image&width=$w&height=$w") return concatUrl(baseUrl, "$id?optimizer=image&width=$w&height=$w")
} }

@ -4,6 +4,7 @@ import androidx.collection.ArrayMap
import org.json.JSONArray import org.json.JSONArray
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.jsoup.nodes.Element import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.PagedMangaParser
@ -11,9 +12,10 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.getOrNull
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
import org.koitharu.kotatsu.parsers.Broken
@Broken @Broken
@MangaSourceParser("BLOGTRUYENVN", "BlogTruyenVN", "vi") @MangaSourceParser("BLOGTRUYENVN", "BlogTruyenVN", "vi")
@ -43,7 +45,7 @@ internal class BlogTruyenVN(context: MangaLoaderContext) :
) )
private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
private var cacheTags = SuspendLazy(::fetchTags) private var cacheTags = suspendLazy(initializer = ::fetchTags)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
return when { return when {
@ -79,7 +81,7 @@ internal class BlogTruyenVN(context: MangaLoaderContext) :
return listElements.mapNotNull { el -> return listElements.mapNotNull { el ->
val linkTag = el.selectFirst("div.fl-l > a") ?: return@mapNotNull null val linkTag = el.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href") val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap -> val tags = cacheTags.getOrNull()?.let { tagMap ->
el.select("footer > div.category > a").mapNotNullToSet { a -> el.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()] tagMap[a.text()]
} }
@ -169,7 +171,7 @@ internal class BlogTruyenVN(context: MangaLoaderContext) :
} }
} }
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap -> val tags = cacheTags.getOrNull()?.let { tagMap ->
descriptionElement.select("p > span.category").mapNotNullToSet { descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName] tagMap[tagName]

@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
import java.util.* import java.util.*
@MangaSourceParser("BAOZIMH", "Baozimh", "zh") @MangaSourceParser("BAOZIMH", "Baozimh", "zh")
@ -42,7 +43,7 @@ internal class Baozimh(context: MangaLoaderContext) :
), ),
) )
private val tagsMap = SuspendLazy(::parseTags) private val tagsMap = suspendLazy(initializer = ::parseTags)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
when { when {

@ -4,7 +4,7 @@ package org.koitharu.kotatsu.parsers.util
import kotlin.enums.EnumEntries import kotlin.enums.EnumEntries
public fun <E : Enum<E>> EnumEntries<E>.names() = Array(size) { i -> public fun <E : Enum<E>> EnumEntries<E>.names(): Array<String> = Array(size) { i ->
get(i).name get(i).name
} }

@ -7,13 +7,14 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
public class LinkResolver internal constructor( public class LinkResolver internal constructor(
private val context: MangaLoaderContext, private val context: MangaLoaderContext,
public val link: HttpUrl, public val link: HttpUrl,
) { ) {
private val source = SuspendLazy(::resolveSource) private val source = suspendLazy(initializer = ::resolveSource)
public suspend fun getSource(): MangaParserSource? = source.get() public suspend fun getSource(): MangaParserSource? = source.get()

@ -2,6 +2,7 @@
package org.koitharu.kotatsu.parsers.util package org.koitharu.kotatsu.parsers.util
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaListFilter import org.koitharu.kotatsu.parsers.model.MangaListFilter
import kotlin.contracts.contract import kotlin.contracts.contract
@ -11,3 +12,7 @@ public fun MangaListFilter?.isNullOrEmpty(): Boolean {
} }
return this == null || this.isEmpty() return this == null || this.isEmpty()
} }
public fun Collection<MangaChapter>.findById(chapterId: Long): MangaChapter? = find { x ->
x.id == chapterId
}

@ -62,7 +62,6 @@ public inline fun Int.ifZero(defaultVale: () -> Int): Int {
contract { contract {
callsInPlace(defaultVale, InvocationKind.AT_MOST_ONCE) callsInPlace(defaultVale, InvocationKind.AT_MOST_ONCE)
} }
@Suppress("KotlinConstantConditions")
return if (this == 0) { return if (this == 0) {
defaultVale() defaultVale()
} else { } else {

@ -1,37 +0,0 @@
package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import java.lang.ref.SoftReference
/**
* Like a [SuspendLazy] but with [SoftReference] under the hood
*/
public class SoftSuspendLazy<T : Any>(
private val initializer: suspend () -> T,
) {
private val mutex = Mutex()
private var cachedValue: SoftReference<T>? = null
public suspend fun get(): T {
// fast way
cachedValue?.get()?.let {
return it
}
return mutex.withLock {
cachedValue?.get()?.let {
return it
}
val result = initializer()
cachedValue = SoftReference(result)
result
}
}
public suspend fun tryGet(): Result<T> = runCatchingCancellable { get() }
public fun peek(): T? {
return cachedValue?.get()
}
}

@ -2,7 +2,6 @@
package org.koitharu.kotatsu.parsers.util package org.koitharu.kotatsu.parsers.util
import androidx.annotation.FloatRange
import androidx.collection.MutableIntList import androidx.collection.MutableIntList
import androidx.collection.arraySetOf import androidx.collection.arraySetOf
import java.math.BigInteger import java.math.BigInteger
@ -225,7 +224,7 @@ public fun String.levenshteinDistance(other: String): Int {
/** /**
* @param threshold 0 = exact match * @param threshold 0 = exact match
*/ */
public fun String.almostEquals(other: String, @FloatRange(from = 0.0) threshold: Float): Boolean { public fun String.almostEquals(other: String, threshold: Float): Boolean {
if (threshold <= 0f) { if (threshold <= 0f) {
return equals(other, ignoreCase = true) return equals(other, ignoreCase = true)
} }

@ -1,41 +0,0 @@
package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
public class SuspendLazy<T>(
private val initializer: suspend () -> T,
) {
private val mutex = Mutex()
private var cachedValue: Any? = Uninitialized
@Suppress("UNCHECKED_CAST")
public suspend fun get(): T {
// fast way
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
return mutex.withLock {
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
val result = initializer()
cachedValue = result
result
}
}
public suspend fun tryGet(): Result<T> = runCatchingCancellable { get() }
@Suppress("UNCHECKED_CAST")
public fun peek(): T? {
return cachedValue?.takeUnless { it === Uninitialized } as T?
}
private object Uninitialized
}

@ -0,0 +1,41 @@
package org.koitharu.kotatsu.parsers.util.suspendlazy
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import java.lang.ref.SoftReference
import kotlin.coroutines.CoroutineContext
/**
* Like a [SuspendLazy] but with [SoftReference] under the hood
*/
internal class SoftSuspendLazyImpl<T : Any>(
private val coroutineContext: CoroutineContext,
private val initializer: SuspendLazyInitializer<T>,
) : SuspendLazy<T> {
private val mutex: Mutex = Mutex()
private var cachedValue: SoftReference<T>? = null
override val isInitialized: Boolean
get() = cachedValue?.get() != null
override suspend fun get(): T {
// fast way
cachedValue?.get()?.let {
return it
}
return mutex.withLock {
cachedValue?.get()?.let {
return it
}
val result = withContext(coroutineContext) {
initializer()
}
cachedValue = SoftReference(result)
result
}
}
override fun peek(): T? = cachedValue?.get()
}

@ -0,0 +1,33 @@
package org.koitharu.kotatsu.parsers.util.suspendlazy
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
internal typealias SuspendLazyInitializer<T> = suspend () -> T
public interface SuspendLazy<T> {
public val isInitialized: Boolean
public suspend fun get(): T
public fun peek(): T?
}
public suspend fun <T> SuspendLazy<T>.getOrNull(): T? = runCatchingCancellable { get() }.getOrNull()
public fun <T> suspendLazy(
context: CoroutineContext = EmptyCoroutineContext,
initializer: SuspendLazyInitializer<T>,
): SuspendLazy<T> = SuspendLazyImpl(context, initializer)
public fun <T : Any> suspendLazy(
context: CoroutineContext = EmptyCoroutineContext,
soft: Boolean,
initializer: SuspendLazyInitializer<T>,
): SuspendLazy<T> = if (soft) {
SoftSuspendLazyImpl(context, initializer)
} else {
SuspendLazyImpl(context, initializer)
}

@ -0,0 +1,47 @@
package org.koitharu.kotatsu.parsers.util.suspendlazy
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlin.coroutines.CoroutineContext
internal class SuspendLazyImpl<T>(
private val coroutineContext: CoroutineContext,
private val initializer: SuspendLazyInitializer<T>,
) : SuspendLazy<T> {
private val mutex: Mutex = Mutex()
private var cachedValue: Any? = Uninitialized
override val isInitialized: Boolean
get() = cachedValue !== Uninitialized
@Suppress("UNCHECKED_CAST")
override suspend fun get(): T {
// fast way
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
return mutex.withLock {
cachedValue.let {
if (it !== Uninitialized) {
return it as T
}
}
val result = withContext(coroutineContext) {
initializer()
}
cachedValue = result
result
}
}
@Suppress("UNCHECKED_CAST")
override fun peek(): T? {
return cachedValue?.takeUnless { it === Uninitialized } as T?
}
private object Uninitialized
}
Loading…
Cancel
Save