KotatsuApp
/
Kotatsu
2
0
Fork 0
Code Issues 124 Pull Requests Packages Projects Releases 163 Wiki Activity

hitomi: formatting

Browse Source
AwkwardPeak7 2 years ago
parent b61c5e8f12
commit 495c9fad33
No known key found for this signature in database
1 changed files with 305 additions and 244 deletions
Show Stats Download Patch File Download Diff File

167
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
Unescape Escape View File

@ -20,17 +20,18 @@ import kotlin.math.min
@OptIn(ExperimentalUnsignedTypes::class) @OptIn(ExperimentalUnsignedTypes::class)
@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI) @MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI)
class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.HITOMILA) { class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.HITOMILA) {
override val configKeyDomain = ConfigKey.Domain("hitomi.la") override val configKeyDomain = ConfigKey.Domain("hitomi.la")
private val ltnBaseUrl get() = "https://${getDomain("ltn")}" private val ltnBaseUrl get() = "https://${getDomain("ltn")}"
override val availableSortOrders: Set<SortOrder> = EnumSet.of( override val availableSortOrders: Set<SortOrder> =
EnumSet.of(
SortOrder.NEWEST, SortOrder.NEWEST,
SortOrder.POPULARITY, SortOrder.POPULARITY,
) )
private val localeMap: Map<Locale, String> = mapOf( private val localeMap: Map<Locale, String> =
mapOf(
Locale("id") to "indonesian", Locale("id") to "indonesian",
Locale("jv") to "javanese", Locale("jv") to "javanese",
Locale("ca") to "catalan", Locale("ca") to "catalan",
@ -76,13 +77,15 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml()
doc.select(".posts > li").mapNotNull { element -> doc.select(".posts > li").mapNotNull { element ->
val num = element.ownText().let { val num =
element.ownText().let {
Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0 Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0
} }
if (num > 100) { if (num > 100) {
val url = element.selectFirst("a") val url = element.selectFirst("a")
val href = url?.attrAsRelativeUrl("href") val href =
url?.attrAsRelativeUrl("href")
?: return@mapNotNull null ?: return@mapNotNull null
MangaTag( MangaTag(
@ -101,7 +104,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private var cachedSearchIds: List<Int> = emptyList() private var cachedSearchIds: List<Int> = emptyList()
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> { override suspend fun getList(
offset: Int,
filter: MangaListFilter?,
): List<Manga> {
return when (filter) { return when (filter) {
is MangaListFilter.Advanced -> { is MangaListFilter.Advanced -> {
if (filter.tags.isEmpty()) { if (filter.tags.isEmpty()) {
@ -116,7 +122,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
} else { } else {
if (offset == 0) { if (offset == 0) {
val query = filter.tags.joinToString(" ") { it.key }.let { val query =
filter.tags.joinToString(" ") { it.key }.let {
val lang = filter.locale.getSiteLang() val lang = filter.locale.getSiteLang()
if (lang != "all") { if (lang != "all") {
"$it language:$lang" "$it language:$lang"
@ -147,8 +154,13 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return bytes.until(bytes + 100L) return bytes.until(bytes + 100L)
} }
private suspend fun hitomiSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> = coroutineScope { private suspend fun hitomiSearch(
val terms = query query: String,
sortByPopularity: Boolean = false,
): Set<Int> =
coroutineScope {
val terms =
query
.trim() .trim()
.replace(Regex("""^\?"""), "") .replace(Regex("""^\?"""), "")
.lowercase() .lowercase()
@ -161,13 +173,15 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val negativeTerms = LinkedList<String>() val negativeTerms = LinkedList<String>()
for (term in terms) { for (term in terms) {
if (term.startsWith("-")) if (term.startsWith("-")) {
negativeTerms.push(term.removePrefix("-")) negativeTerms.push(term.removePrefix("-"))
else if (term.isNotBlank()) } else if (term.isNotBlank()) {
positiveTerms.push(term) positiveTerms.push(term)
} }
}
val positiveResults = positiveTerms.map { val positiveResults =
positiveTerms.map {
async { async {
runCatching { runCatching {
getGalleryIDsForQuery(it) getGalleryIDsForQuery(it)
@ -175,7 +189,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
} }
val negativeResults = negativeTerms.map { val negativeResults =
negativeTerms.map {
async { async {
runCatching { runCatching {
getGalleryIDsForQuery(it) getGalleryIDsForQuery(it)
@ -183,7 +198,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
} }
val results = when { val results =
when {
sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all") sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all")
positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all") positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all")
else -> emptySet() else -> emptySet()
@ -240,7 +256,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val key = hashTerm(it) val key = hashTerm(it)
val node = getGalleryNodeAtAddress(0) val node = getGalleryNodeAtAddress(0)
val data = bSearch(key, node) val data =
bSearch(key, node)
?: return emptySet() ?: return emptySet()
return getGalleryIDsFromData(data) return getGalleryIDsFromData(data)
@ -250,14 +267,16 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> { private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> {
val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data" val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data"
val (offset, length) = data val (offset, length) = data
if (length > 100000000 || length <= 0) if (length > 100000000 || length <= 0) {
throw Exception("length $length is too long") throw Exception("length $length is too long")
}
val inbuf = getRangedResponse(url, offset.until(offset + length)) val inbuf = getRangedResponse(url, offset.until(offset + length))
val galleryIDs = mutableSetOf<Int>() val galleryIDs = mutableSetOf<Int>()
val buffer = ByteBuffer val buffer =
ByteBuffer
.wrap(inbuf) .wrap(inbuf)
.order(ByteOrder.BIG_ENDIAN) .order(ByteOrder.BIG_ENDIAN)
@ -265,10 +284,11 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val expectedLength = numberOfGalleryIDs * 4 + 4 val expectedLength = numberOfGalleryIDs * 4 + 4
if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0) if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0) {
throw Exception("number_of_galleryids $numberOfGalleryIDs is too long") throw Exception("number_of_galleryids $numberOfGalleryIDs is too long")
else if (inbuf.size != expectedLength) } else if (inbuf.size != expectedLength) {
throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength") throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength")
}
for (i in 0.until(numberOfGalleryIDs)) for (i in 0.until(numberOfGalleryIDs))
galleryIDs.add(buffer.int) galleryIDs.add(buffer.int)
@ -276,54 +296,74 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return galleryIDs return galleryIDs
} }
private suspend fun bSearch(key: UByteArray, node: Node) : Pair<Long, Int>? { private suspend fun bSearch(
fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int { key: UByteArray,
node: Node,
): Pair<Long, Int>? {
fun compareArrayBuffers(
dv1: UByteArray,
dv2: UByteArray,
): Int {
val top = min(dv1.size, dv2.size) val top = min(dv1.size, dv2.size)
for (i in 0.until(top)) { for (i in 0.until(top)) {
if (dv1[i] < dv2[i]) if (dv1[i] < dv2[i]) {
return -1 return -1
else if (dv1[i] > dv2[i]) } else if (dv1[i] > dv2[i]) {
return 1 return 1
} }
}
return 0 return 0
} }
fun locateKey(key: UByteArray, node: Node) : Pair<Boolean, Int> { fun locateKey(
key: UByteArray,
node: Node,
): Pair<Boolean, Int> {
for (i in node.keys.indices) { for (i in node.keys.indices) {
val cmpResult = compareArrayBuffers(key, node.keys[i]) val cmpResult = compareArrayBuffers(key, node.keys[i])
if (cmpResult <= 0) if (cmpResult <= 0) {
return Pair(cmpResult == 0, i) return Pair(cmpResult == 0, i)
} }
}
return Pair(false, node.keys.size) return Pair(false, node.keys.size)
} }
fun isLeaf(node: Node): Boolean { fun isLeaf(node: Node): Boolean {
for (subnode in node.subNodeAddresses) for (subnode in node.subNodeAddresses)
if (subnode != 0L) if (subnode != 0L) {
return false return false
}
return true return true
} }
if (node.keys.isEmpty()) if (node.keys.isEmpty()) {
return null return null
}
val (there, where) = locateKey(key, node) val (there, where) = locateKey(key, node)
if (there) if (there) {
return node.datas[where] return node.datas[where]
else if (isLeaf(node)) } else if (isLeaf(node)) {
return null return null
}
val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where]) val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where])
return bSearch(key, nextNode) return bSearch(key, nextNode)
} }
private suspend fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String, range: LongRange? = null) : Set<Int> { private suspend fun getGalleryIDsFromNozomi(
val nozomiAddress = when(area) { area: String?,
tag: String,
language: String,
range: LongRange? = null,
): Set<Int> {
val nozomiAddress =
when (area) {
null -> "$ltnBaseUrl/$tag-$language.nozomi" null -> "$ltnBaseUrl/$tag-$language.nozomi"
else -> "$ltnBaseUrl/$area/$tag-$language.nozomi" else -> "$ltnBaseUrl/$area/$tag-$language.nozomi"
} }
@ -331,7 +371,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val bytes = getRangedResponse(nozomiAddress, range) val bytes = getRangedResponse(nozomiAddress, range)
val nozomi = mutableSetOf<Int>() val nozomi = mutableSetOf<Int>()
val arrayBuffer = ByteBuffer val arrayBuffer =
ByteBuffer
.wrap(bytes) .wrap(bytes)
.order(ByteOrder.BIG_ENDIAN) .order(ByteOrder.BIG_ENDIAN)
@ -341,7 +382,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return nozomi return nozomi
} }
private val galleriesIndexVersion = SuspendLazy { private val galleriesIndexVersion =
SuspendLazy {
webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw()
} }
@ -352,7 +394,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
) )
private fun decodeNode(data: ByteArray): Node { private fun decodeNode(data: ByteArray): Node {
val buffer = ByteBuffer val buffer =
ByteBuffer
.wrap(data) .wrap(data)
.order(ByteOrder.BIG_ENDIAN) .order(ByteOrder.BIG_ENDIAN)
@ -364,8 +407,9 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
for (i in 0.until(numberOfKeys)) { for (i in 0.until(numberOfKeys)) {
val keySize = buffer.int val keySize = buffer.int
if (keySize == 0 || keySize > 32) if (keySize == 0 || keySize > 32) {
throw Exception("fatal: !keySize || keySize > 32") throw Exception("fatal: !keySize || keySize > 32")
}
keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize))) keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize)))
buffer.position(buffer.position() + keySize) buffer.position(buffer.position() + keySize)
@ -400,8 +444,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return decodeNode(nodedata) return decodeNode(nodedata)
} }
private suspend fun getRangedResponse(url: String, range: LongRange? = null) : ByteArray { private suspend fun getRangedResponse(
val rangeHeaders = when (range) { url: String,
range: LongRange? = null,
): ByteArray {
val rangeHeaders =
when (range) {
null -> Headers.headersOf() null -> Headers.headersOf()
else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}") else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}")
} }
@ -428,10 +476,13 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
id = generateUid(id.toString()), id = generateUid(id.toString()),
title = doc.selectFirstOrThrow("h1").text(), title = doc.selectFirstOrThrow("h1").text(),
url = id.toString(), url = id.toString(),
coverUrl = "https:" + doc.selectFirstOrThrow("picture > source") coverUrl =
"https:" +
doc.selectFirstOrThrow("picture > source")
.attr("data-srcset") .attr("data-srcset")
.substringBefore(" "), .substringBefore(" "),
publicUrl = doc.selectFirstOrThrow("h1 > a") publicUrl =
doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href") .attrAsRelativeUrl("href")
.toAbsoluteUrl(domain), .toAbsoluteUrl(domain),
author = null, author = null,
@ -449,14 +500,16 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js") val json =
webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js")
.parseRaw() .parseRaw()
.substringAfter("var galleryinfo = ") .substringAfter("var galleryinfo = ")
.let(::JSONObject) .let(::JSONObject)
return manga.copy( return manga.copy(
title = json.getString("title"), title = json.getString("title"),
largeCoverUrl = json.getJSONArray("files").getJSONObject(0).let { largeCoverUrl =
json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash") val hash = it.getString("hash")
val commonId = commonImageId() val commonId = commonImageId()
val imageId = imageIdFromHash(hash) val imageId = imageIdFromHash(hash)
@ -464,11 +517,13 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp" "https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
}, },
author = json.optJSONArray("artists") author =
json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() } ?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(), ?.joinToString(),
publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain), publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain),
tags = buildSet { tags =
buildSet {
json.optJSONArray("characters") json.optJSONArray("characters")
?.mapToTags("character") ?.mapToTags("character")
?.let(::addAll) ?.let(::addAll)
@ -485,7 +540,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
?.mapToTags("group") ?.mapToTags("group")
?.let(::addAll) ?.let(::addAll)
}, },
chapters = listOf( chapters =
listOf(
MangaChapter( MangaChapter(
id = generateUid(manga.url), id = generateUid(manga.url),
url = manga.url, url = manga.url,
@ -495,8 +551,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
branch = json.getString("language_localname"), branch = json.getString("language_localname"),
source = source, source = source,
uploadDate = dateFormat.tryParse(json.getString("date").substringBeforeLast("-")), uploadDate = dateFormat.tryParse(json.getString("date").substringBeforeLast("-")),
) ),
) ),
) )
} }
@ -508,7 +564,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val tags = mutableSetOf<MangaTag>() val tags = mutableSetOf<MangaTag>()
mapJSON { mapJSON {
MangaTag( MangaTag(
title = it.getString(key).toCamelCase().let { title -> title =
it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) { if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title ♀" "$title ♀"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
@ -518,7 +575,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
}, },
key = it.getString("url").tagUrlToTag(), key = it.getString("url").tagUrlToTag(),
source = source source = source,
).let(tags::add) ).let(tags::add)
} }
return tags return tags
@ -527,7 +584,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private fun String.tagUrlToTag(): String { private fun String.tagUrlToTag(): String {
val urlContent = this.split("/") val urlContent = this.split("/")
val ns = urlContent[1] val ns = urlContent[1]
val tag = urlContent[2] val tag =
urlContent[2]
.substringBeforeLast("-") .substringBeforeLast("-")
.urlDecode() .urlDecode()
.replace(" ", "_") .replace(" ", "_")
@ -540,7 +598,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
override suspend fun getRelatedManga(seed: Manga): List<Manga> { override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js") val json =
webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js")
.parseRaw() .parseRaw()
.substringAfter("var galleryinfo = ") .substringAfter("var galleryinfo = ")
.let(::JSONObject) .let(::JSONObject)
@ -552,7 +611,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js") val json =
webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js")
.parseRaw() .parseRaw()
.substringAfter("var galleryinfo = ") .substringAfter("var galleryinfo = ")
.let(::JSONObject) .let(::JSONObject)
@ -567,7 +627,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
id = generateUid(hash), id = generateUid(hash),
url = "https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp", url = "https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp",
preview = "https://${getDomain("${subDomain}tn")}/webpsmalltn/${thumbPathFromHash(hash)}/$hash.webp", preview = "https://${getDomain("${subDomain}tn")}/webpsmalltn/${thumbPathFromHash(hash)}/$hash.webp",
source = source source = source,
) )
} }
} }
@ -580,7 +640,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private val subdomainOffsetMap = mutableMapOf<Int, Int>() private val subdomainOffsetMap = mutableMapOf<Int, Int>()
private var commonImageId = "" private var commonImageId = ""
private suspend fun refreshScript() = mutex.withLock { private suspend fun refreshScript() =
mutex.withLock {
if (scriptLastRetrieval == null || (scriptLastRetrieval!! + 60000) < System.currentTimeMillis()) { if (scriptLastRetrieval == null || (scriptLastRetrieval!! + 60000) < System.currentTimeMillis()) {
val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw() val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw()

Write Preview
Loading…
Cancel
Save