KotatsuApp
/
Kotatsu
2
0
Fork 0
Code Issues 124 Pull Requests Packages Projects Releases 163 Wiki Activity

hitomi: formatting

Browse Source
AwkwardPeak7 2 years ago
parent b61c5e8f12
commit 495c9fad33
No known key found for this signature in database
1 changed files with 305 additions and 244 deletions
Show Stats Download Patch File Download Diff File

215
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
Unescape Escape View File

@ -20,17 +20,18 @@ import kotlin.math.min
@OptIn(ExperimentalUnsignedTypes::class)
@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI)
class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.HITOMILA) {
override val configKeyDomain = ConfigKey.Domain("hitomi.la")
private val ltnBaseUrl get() = "https://${getDomain("ltn")}"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
)
private val localeMap: Map<Locale, String> = mapOf(
private val localeMap: Map<Locale, String> =
mapOf(
Locale("id") to "indonesian",
Locale("jv") to "javanese",
Locale("ca") to "catalan",
@ -76,13 +77,15 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml()
doc.select(".posts > li").mapNotNull { element ->
val num = element.ownText().let {
val num =
element.ownText().let {
Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0
}
if (num > 100) {
val url = element.selectFirst("a")
val href = url?.attrAsRelativeUrl("href")
val href =
url?.attrAsRelativeUrl("href")
?: return@mapNotNull null
MangaTag(
@ -101,7 +104,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private var cachedSearchIds: List<Int> = emptyList()
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
override suspend fun getList(
offset: Int,
filter: MangaListFilter?,
): List<Manga> {
return when (filter) {
is MangaListFilter.Advanced -> {
if (filter.tags.isEmpty()) {
@ -116,7 +122,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
} else {
if (offset == 0) {
val query = filter.tags.joinToString(" ") { it.key }.let {
val query =
filter.tags.joinToString(" ") { it.key }.let {
val lang = filter.locale.getSiteLang()
if (lang != "all") {
"$it language:$lang"
@ -125,9 +132,9 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
}
cachedSearchIds = hitomiSearch(query,filter.sortOrder == SortOrder.POPULARITY).toList()
cachedSearchIds = hitomiSearch(query, filter.sortOrder == SortOrder.POPULARITY).toList()
}
cachedSearchIds.subList(offset, min(offset+25, cachedSearchIds.size))
cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size))
}
}
@ -135,7 +142,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
if (offset == 0) {
cachedSearchIds = hitomiSearch(filter.query, filter.sortOrder == SortOrder.POPULARITY).toList()
}
cachedSearchIds.subList(offset, min(offset+25, cachedSearchIds.size))
cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size))
}
else -> getGalleryIDsFromNozomi(null, "popular", "all", offset.nextOffsetRange())
@ -143,12 +150,17 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
private fun Int.nextOffsetRange(): LongRange {
val bytes = this*4L
return bytes.until(bytes+100L)
val bytes = this * 4L
return bytes.until(bytes + 100L)
}
private suspend fun hitomiSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> = coroutineScope {
val terms = query
private suspend fun hitomiSearch(
query: String,
sortByPopularity: Boolean = false,
): Set<Int> =
coroutineScope {
val terms =
query
.trim()
.replace(Regex("""^\?"""), "")
.lowercase()
@ -161,13 +173,15 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val negativeTerms = LinkedList<String>()
for (term in terms) {
if (term.startsWith("-"))
if (term.startsWith("-")) {
negativeTerms.push(term.removePrefix("-"))
else if (term.isNotBlank())
} else if (term.isNotBlank()) {
positiveTerms.push(term)
}
}
val positiveResults = positiveTerms.map {
val positiveResults =
positiveTerms.map {
async {
runCatching {
getGalleryIDsForQuery(it)
@ -175,7 +189,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
}
val negativeResults = negativeTerms.map {
val negativeResults =
negativeTerms.map {
async {
runCatching {
getGalleryIDsForQuery(it)
@ -183,7 +198,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
}
val results = when {
val results =
when {
sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all")
positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all")
else -> emptySet()
@ -200,12 +216,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
results.removeAll(newResults)
}
//positive results
// positive results
positiveResults.forEach {
filterPositive(it.await())
}
//negative results
// negative results
negativeResults.forEach {
filterNegative(it.await())
}
@ -213,15 +229,15 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
results
}
//search.js
private suspend fun getGalleryIDsForQuery(query: String) : Set<Int> {
// search.js
private suspend fun getGalleryIDsForQuery(query: String): Set<Int> {
query.replace("_", " ").let {
if (it.indexOf(':') > -1) {
val sides = it.split(":")
val ns = sides[0]
var tag = sides[1]
var area : String? = ns
var area: String? = ns
var language = "all"
when (ns) {
"female", "male" -> {
@ -240,35 +256,39 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val key = hashTerm(it)
val node = getGalleryNodeAtAddress(0)
val data = bSearch(key, node)
val data =
bSearch(key, node)
?: return emptySet()
return getGalleryIDsFromData(data)
}
}
private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>) : Set<Int> {
private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> {
val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data"
val (offset, length) = data
if (length > 100000000 || length <= 0)
if (length > 100000000 || length <= 0) {
throw Exception("length $length is too long")
}
val inbuf = getRangedResponse(url, offset.until(offset+length))
val inbuf = getRangedResponse(url, offset.until(offset + length))
val galleryIDs = mutableSetOf<Int>()
val buffer = ByteBuffer
val buffer =
ByteBuffer
.wrap(inbuf)
.order(ByteOrder.BIG_ENDIAN)
val numberOfGalleryIDs = buffer.int
val expectedLength = numberOfGalleryIDs*4+4
val expectedLength = numberOfGalleryIDs * 4 + 4
if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0)
if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0) {
throw Exception("number_of_galleryids $numberOfGalleryIDs is too long")
else if (inbuf.size != expectedLength)
} else if (inbuf.size != expectedLength) {
throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength")
}
for (i in 0.until(numberOfGalleryIDs))
galleryIDs.add(buffer.int)
@ -276,54 +296,74 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return galleryIDs
}
private suspend fun bSearch(key: UByteArray, node: Node) : Pair<Long, Int>? {
fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int {
private suspend fun bSearch(
key: UByteArray,
node: Node,
): Pair<Long, Int>? {
fun compareArrayBuffers(
dv1: UByteArray,
dv2: UByteArray,
): Int {
val top = min(dv1.size, dv2.size)
for (i in 0.until(top)) {
if (dv1[i] < dv2[i])
if (dv1[i] < dv2[i]) {
return -1
else if (dv1[i] > dv2[i])
} else if (dv1[i] > dv2[i]) {
return 1
}
}
return 0
}
fun locateKey(key: UByteArray, node: Node) : Pair<Boolean, Int> {
fun locateKey(
key: UByteArray,
node: Node,
): Pair<Boolean, Int> {
for (i in node.keys.indices) {
val cmpResult = compareArrayBuffers(key, node.keys[i])
if (cmpResult <= 0)
return Pair(cmpResult==0, i)
if (cmpResult <= 0) {
return Pair(cmpResult == 0, i)
}
}
return Pair(false, node.keys.size)
}
fun isLeaf(node: Node) : Boolean {
fun isLeaf(node: Node): Boolean {
for (subnode in node.subNodeAddresses)
if (subnode != 0L)
if (subnode != 0L) {
return false
}
return true
}
if (node.keys.isEmpty())
if (node.keys.isEmpty()) {
return null
}
val (there, where) = locateKey(key, node)
if (there)
if (there) {
return node.datas[where]
else if (isLeaf(node))
} else if (isLeaf(node)) {
return null
}
val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where])
return bSearch(key, nextNode)
}
private suspend fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String, range: LongRange? = null) : Set<Int> {
val nozomiAddress = when(area) {
private suspend fun getGalleryIDsFromNozomi(
area: String?,
tag: String,
language: String,
range: LongRange? = null,
): Set<Int> {
val nozomiAddress =
when (area) {
null -> "$ltnBaseUrl/$tag-$language.nozomi"
else -> "$ltnBaseUrl/$area/$tag-$language.nozomi"
}
@ -331,7 +371,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val bytes = getRangedResponse(nozomiAddress, range)
val nozomi = mutableSetOf<Int>()
val arrayBuffer = ByteBuffer
val arrayBuffer =
ByteBuffer
.wrap(bytes)
.order(ByteOrder.BIG_ENDIAN)
@ -341,7 +382,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return nozomi
}
private val galleriesIndexVersion = SuspendLazy {
private val galleriesIndexVersion =
SuspendLazy {
webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw()
}
@ -351,8 +393,9 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val subNodeAddresses: List<Long>,
)
private fun decodeNode(data: ByteArray) : Node {
val buffer = ByteBuffer
private fun decodeNode(data: ByteArray): Node {
val buffer =
ByteBuffer
.wrap(data)
.order(ByteOrder.BIG_ENDIAN)
@ -364,11 +407,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
for (i in 0.until(numberOfKeys)) {
val keySize = buffer.int
if (keySize == 0 || keySize > 32)
if (keySize == 0 || keySize > 32) {
throw Exception("fatal: !keySize || keySize > 32")
}
keys.add(uData.sliceArray(buffer.position().until(buffer.position()+keySize)))
buffer.position(buffer.position()+keySize)
keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize)))
buffer.position(buffer.position() + keySize)
}
val numberOfDatas = buffer.int
@ -392,7 +436,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return Node(keys, datas, subNodeAddresses)
}
private suspend fun getGalleryNodeAtAddress(address: Long) : Node {
private suspend fun getGalleryNodeAtAddress(address: Long): Node {
val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.index"
val nodedata = getRangedResponse(url, address.until(address + 464))
@ -400,8 +444,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return decodeNode(nodedata)
}
private suspend fun getRangedResponse(url: String, range: LongRange? = null) : ByteArray {
val rangeHeaders = when (range) {
private suspend fun getRangedResponse(
url: String,
range: LongRange? = null,
): ByteArray {
val rangeHeaders =
when (range) {
null -> Headers.headersOf()
else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}")
}
@ -409,11 +457,11 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return webClient.httpGet(url, rangeHeaders).parseBytes()
}
private fun hashTerm(term: String) : UByteArray {
private fun hashTerm(term: String): UByteArray {
return sha256(term.toByteArray()).copyOfRange(0, 4).toUByteArray()
}
private fun sha256(data: ByteArray) : ByteArray {
private fun sha256(data: ByteArray): ByteArray {
return MessageDigest.getInstance("SHA-256").digest(data)
}
@ -428,10 +476,13 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
id = generateUid(id.toString()),
title = doc.selectFirstOrThrow("h1").text(),
url = id.toString(),
coverUrl = "https:" + doc.selectFirstOrThrow("picture > source")
coverUrl =
"https:" +
doc.selectFirstOrThrow("picture > source")
.attr("data-srcset")
.substringBefore(" "),
publicUrl = doc.selectFirstOrThrow("h1 > a")
publicUrl =
doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href")
.toAbsoluteUrl(domain),
author = null,
@ -449,14 +500,16 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
override suspend fun getDetails(manga: Manga): Manga {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js")
val json =
webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js")
.parseRaw()
.substringAfter("var galleryinfo = ")
.let(::JSONObject)
return manga.copy(
title = json.getString("title"),
largeCoverUrl = json.getJSONArray("files").getJSONObject(0).let {
largeCoverUrl =
json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash")
val commonId = commonImageId()
val imageId = imageIdFromHash(hash)
@ -464,11 +517,13 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
},
author = json.optJSONArray("artists")
author =
json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(),
publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain),
tags = buildSet {
tags =
buildSet {
json.optJSONArray("characters")
?.mapToTags("character")
?.let(::addAll)
@ -485,7 +540,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
?.mapToTags("group")
?.let(::addAll)
},
chapters = listOf(
chapters =
listOf(
MangaChapter(
id = generateUid(manga.url),
url = manga.url,
@ -495,8 +551,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
branch = json.getString("language_localname"),
source = source,
uploadDate = dateFormat.tryParse(json.getString("date").substringBeforeLast("-")),
)
)
),
),
)
}
@ -508,7 +564,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val tags = mutableSetOf<MangaTag>()
mapJSON {
MangaTag(
title = it.getString(key).toCamelCase().let { title ->
title =
it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title ♀"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
@ -518,7 +575,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
},
key = it.getString("url").tagUrlToTag(),
source = source
source = source,
).let(tags::add)
}
return tags
@ -527,7 +584,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private fun String.tagUrlToTag(): String {
val urlContent = this.split("/")
val ns = urlContent[1]
val tag = urlContent[2]
val tag =
urlContent[2]
.substringBeforeLast("-")
.urlDecode()
.replace(" ", "_")
@ -540,7 +598,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js")
val json =
webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js")
.parseRaw()
.substringAfter("var galleryinfo = ")
.let(::JSONObject)
@ -552,7 +611,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js")
val json =
webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js")
.parseRaw()
.substringAfter("var galleryinfo = ")
.let(::JSONObject)
@ -564,15 +624,15 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val subDomain = 'a' + subdomainOffset(imageId)
MangaPage(
id= generateUid(hash),
id = generateUid(hash),
url = "https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp",
preview = "https://${getDomain("${subDomain}tn")}/webpsmalltn/${thumbPathFromHash(hash)}/$hash.webp",
source = source
source = source,
)
}
}
/// --->
// / --->
private var scriptLastRetrieval: Long? = null
private val mutex = Mutex()
@ -580,7 +640,8 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private val subdomainOffsetMap = mutableMapOf<Int, Int>()
private var commonImageId = ""
private suspend fun refreshScript() = mutex.withLock {
private suspend fun refreshScript() =
mutex.withLock {
if (scriptLastRetrieval == null || (scriptLastRetrieval!! + 60000) < System.currentTimeMillis()) {
val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw()
@ -614,7 +675,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
// s <-- gg.js
private fun imageIdFromHash(hash: String): Int {
val match = Regex("(..)(.)$").find(hash)
return match!!.groupValues.let { it[2]+it[1] }.toInt(16)
return match!!.groupValues.let { it[2] + it[1] }.toInt(16)
}
// real_full_path_from_hash <-- common.js

Write Preview
Loading…
Cancel
Save