KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[HitomiLa] Refactor

Browse Source
pull/428/head
Koitharu 2 years ago
parent 43bdbe5a01
commit 4a0e7221b0
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7
1 changed files with 233 additions and 243 deletions
Show Stats Download Patch File Download Diff File

98
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
Unescape Escape View File

@ -1,10 +1,17 @@
package org.koitharu.kotatsu.parsers.site.all package org.koitharu.kotatsu.parsers.site.all
import kotlinx.coroutines.* import androidx.collection.ArraySet
import kotlinx.coroutines.sync.* import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import okhttp3.Headers import okhttp3.Headers
import org.json.* import org.json.JSONArray
import org.koitharu.kotatsu.parsers.* import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
@ -24,14 +31,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private val ltnBaseUrl get() = "https://${getDomain("ltn")}" private val ltnBaseUrl get() = "https://${getDomain("ltn")}"
override val availableSortOrders: Set<SortOrder> = override val availableSortOrders: Set<SortOrder> = EnumSet.of(
EnumSet.of(
SortOrder.NEWEST, SortOrder.NEWEST,
SortOrder.POPULARITY, SortOrder.POPULARITY,
) )
private val localeMap: Map<Locale, String> = private val localeMap: Map<Locale, String> = mapOf(
mapOf(
Locale("id") to "indonesian", Locale("id") to "indonesian",
Locale("jv") to "javanese", Locale("jv") to "javanese",
Locale("ca") to "catalan", Locale("ca") to "catalan",
@ -59,19 +64,14 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
Locale.JAPANESE to "japanese", Locale.JAPANESE to "japanese",
) )
private fun Locale?.getSiteLang(): String { private fun Locale?.getSiteLang(): String = when (this) {
return when (this) {
null -> "all" null -> "all"
else -> localeMap[this] ?: "all" else -> localeMap[this] ?: "all"
} }
}
override suspend fun getAvailableLocales(): Set<Locale> { override suspend fun getAvailableLocales(): Set<Locale> = localeMap.keys
return localeMap.keys
}
override suspend fun getAvailableTags(): Set<MangaTag> { override suspend fun getAvailableTags(): Set<MangaTag> = coroutineScope {
return coroutineScope {
('a'..'z').map { alphabet -> ('a'..'z').map { alphabet ->
async { async {
val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml()
@ -100,20 +100,23 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
}.awaitAll().flatten().toSet() }.awaitAll().flatten().toSet()
} }
}
private var cachedSearchIds: List<Int> = emptyList() private var cachedSearchIds: List<Int> = emptyList()
override suspend fun getList( override suspend fun getList(
offset: Int, offset: Int,
filter: MangaListFilter?, filter: MangaListFilter?,
): List<Manga> { ): List<Manga> = when (filter) {
return when (filter) {
is MangaListFilter.Advanced -> { is MangaListFilter.Advanced -> {
if (filter.tags.isEmpty()) { if (filter.tags.isEmpty()) {
when (filter.sortOrder) { when (filter.sortOrder) {
SortOrder.POPULARITY -> { SortOrder.POPULARITY -> {
getGalleryIDsFromNozomi("popular", "today", filter.locale.getSiteLang(), offset.nextOffsetRange()) getGalleryIDsFromNozomi(
"popular",
"today",
filter.locale.getSiteLang(),
offset.nextOffsetRange(),
)
} }
else -> { else -> {
@ -142,7 +145,6 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
else -> getGalleryIDsFromNozomi(null, "popular", "all", offset.nextOffsetRange()) else -> getGalleryIDsFromNozomi(null, "popular", "all", offset.nextOffsetRange())
}.toMangaList() }.toMangaList()
}
private fun Int.nextOffsetRange(): LongRange { private fun Int.nextOffsetRange(): LongRange {
val bytes = this * 4L val bytes = this * 4L
@ -243,6 +245,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
area = "tag" area = "tag"
tag = it tag = it
} }
"language" -> { "language" -> {
area = null area = null
lang = tag lang = tag
@ -255,9 +258,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val key = hashTerm(it) val key = hashTerm(it)
val node = getGalleryNodeAtAddress(0) val node = getGalleryNodeAtAddress(0)
val data = val data = bSearch(key, node) ?: return emptySet()
bSearch(key, node)
?: return emptySet()
return getGalleryIDsFromData(data) return getGalleryIDsFromData(data)
} }
@ -266,7 +267,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> { private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> {
val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data" val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data"
val (offset, length) = data val (offset, length) = data
require(length in 0..100000000) { require(length in 1..100000000) {
"Length $length is too long" "Length $length is too long"
} }
@ -283,10 +284,11 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val expectedLength = numberOfGalleryIDs * 4 + 4 val expectedLength = numberOfGalleryIDs * 4 + 4
if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0) { require(numberOfGalleryIDs in 1..10000000) {
throw IllegalArgumentException("number_of_galleryids $numberOfGalleryIDs is too long") "number_of_galleryids $numberOfGalleryIDs is too long"
} else if (inbuf.size != expectedLength) { }
throw IllegalArgumentException("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength") require(inbuf.size == expectedLength) {
"inbuf.byteLength ${inbuf.size} != expected_length $expectedLength"
} }
for (i in 0.until(numberOfGalleryIDs)) for (i in 0.until(numberOfGalleryIDs))
@ -316,7 +318,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return 0 return 0
} }
private fun locateKey( fun locateKey(
key: UByteArray, key: UByteArray,
node: Node, node: Node,
): Pair<Boolean, Int> { ): Pair<Boolean, Int> {
@ -331,7 +333,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return Pair(false, node.keys.size) return Pair(false, node.keys.size)
} }
private fun isLeaf(node: Node): Boolean { fun isLeaf(node: Node): Boolean {
for (subnode in node.subNodeAddresses) for (subnode in node.subNodeAddresses)
if (subnode != 0L) { if (subnode != 0L) {
return false return false
@ -361,8 +363,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
language: String, language: String,
range: LongRange? = null, range: LongRange? = null,
): Set<Int> { ): Set<Int> {
val nozomiAddress = val nozomiAddress = when (area) {
when (area) {
null -> "$ltnBaseUrl/$tag-$language.nozomi" null -> "$ltnBaseUrl/$tag-$language.nozomi"
else -> "$ltnBaseUrl/$area/$tag-$language.nozomi" else -> "$ltnBaseUrl/$area/$tag-$language.nozomi"
} }
@ -370,8 +371,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val bytes = getRangedResponse(nozomiAddress, range) val bytes = getRangedResponse(nozomiAddress, range)
val nozomi = mutableSetOf<Int>() val nozomi = mutableSetOf<Int>()
val arrayBuffer = val arrayBuffer = ByteBuffer
ByteBuffer
.wrap(bytes) .wrap(bytes)
.order(ByteOrder.BIG_ENDIAN) .order(ByteOrder.BIG_ENDIAN)
@ -381,8 +381,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return nozomi return nozomi
} }
private val galleriesIndexVersion = private val galleriesIndexVersion = SuspendLazy {
SuspendLazy {
webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw()
} }
@ -393,8 +392,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
) )
private fun decodeNode(data: ByteArray): Node { private fun decodeNode(data: ByteArray): Node {
val buffer = val buffer = ByteBuffer
ByteBuffer
.wrap(data) .wrap(data)
.order(ByteOrder.BIG_ENDIAN) .order(ByteOrder.BIG_ENDIAN)
@ -447,8 +445,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
url: String, url: String,
range: LongRange? = null, range: LongRange? = null,
): ByteArray { ): ByteArray {
val rangeHeaders = val rangeHeaders = when (range) {
when (range) {
null -> Headers.headersOf() null -> Headers.headersOf()
else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}") else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}")
} }
@ -464,8 +461,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return MessageDigest.getInstance("SHA-256").digest(data) return MessageDigest.getInstance("SHA-256").digest(data)
} }
private suspend fun Collection<Int>.toMangaList(): List<Manga> { private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope {
return coroutineScope {
map { id -> map { id ->
async { async {
runCatching { runCatching {
@ -496,11 +492,9 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
}.awaitAll().filterNotNull() }.awaitAll().filterNotNull()
} }
}
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val json = val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js")
webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js")
.parseRaw() .parseRaw()
.substringAfter("var galleryinfo = ") .substringAfter("var galleryinfo = ")
.let(::JSONObject) .let(::JSONObject)
@ -539,8 +533,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
?.mapToTags("group") ?.mapToTags("group")
?.let(::addAll) ?.let(::addAll)
}, },
chapters = chapters = listOf(
listOf(
MangaChapter( MangaChapter(
id = generateUid(manga.url), id = generateUid(manga.url),
url = manga.url, url = manga.url,
@ -595,8 +588,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
override suspend fun getRelatedManga(seed: Manga): List<Manga> { override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val json = val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js")
webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js")
.parseRaw() .parseRaw()
.substringAfter("var galleryinfo = ") .substringAfter("var galleryinfo = ")
.let(::JSONObject) .let(::JSONObject)
@ -608,8 +600,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val json = val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js")
webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js")
.parseRaw() .parseRaw()
.substringAfter("var galleryinfo = ") .substringAfter("var galleryinfo = ")
.let(::JSONObject) .let(::JSONObject)
@ -637,8 +628,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private val subdomainOffsetMap = mutableMapOf<Int, Int>() private val subdomainOffsetMap = mutableMapOf<Int, Int>()
private var commonImageId = "" private var commonImageId = ""
private suspend fun refreshScript() = private suspend fun refreshScript() = mutex.withLock {
mutex.withLock {
if (scriptLastRetrieval == null || (scriptLastRetrieval!! + 60000) < System.currentTimeMillis()) { if (scriptLastRetrieval == null || (scriptLastRetrieval!! + 60000) < System.currentTimeMillis()) {
val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw() val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw()

Write Preview
Loading…
Cancel
Save