KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[HitomiLa] Fix tag names

Browse Source
pull/429/head
Koitharu 2 years ago
parent 52ac3d2e6c
commit 904e0719eb
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7
2 changed files with 58 additions and 53 deletions
Show Stats Download Patch File Download Diff File

84
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
Unescape Escape View File

@ -90,7 +90,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
?: return@mapNotNull null ?: return@mapNotNull null
MangaTag( MangaTag(
title = url.ownText().toCamelCase(), title = url.ownText().toTagTitle(),
key = href.tagUrlToTag(), key = href.tagUrlToTag(),
source = source, source = source,
) )
@ -158,15 +158,14 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
language: String = "all", language: String = "all",
): Set<Int> = ): Set<Int> =
coroutineScope { coroutineScope {
val terms = val terms = query
query .trim()
.trim() .replace(Regex("""^\?"""), "")
.replace(Regex("""^\?"""), "") .lowercase()
.lowercase() .split(Regex("\\s+"))
.split(Regex("\\s+")) .map {
.map { it.replace('_', ' ')
it.replace('_', ' ') }
}
val positiveTerms = LinkedList<String>() val positiveTerms = LinkedList<String>()
val negativeTerms = LinkedList<String>() val negativeTerms = LinkedList<String>()
@ -179,30 +178,27 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
} }
} }
val positiveResults = val positiveResults = positiveTerms.map {
positiveTerms.map { async {
async { runCatchingCancellable {
runCatching { getGalleryIDsForQuery(it, language)
getGalleryIDsForQuery(it, language) }.getOrDefault(emptySet())
}.getOrDefault(emptySet())
}
} }
}
val negativeResults = val negativeResults = negativeTerms.map {
negativeTerms.map { async {
async { runCatchingCancellable {
runCatching { getGalleryIDsForQuery(it, language)
getGalleryIDsForQuery(it, language) }.getOrDefault(emptySet())
}.getOrDefault(emptySet())
}
} }
}
val results = val results = when {
when { sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language)
sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language) positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language)
positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) else -> emptySet()
else -> emptySet() }.toMutableSet()
}.toMutableSet()
fun filterPositive(newResults: Set<Int>) { fun filterPositive(newResults: Set<Int>) {
when { when {
@ -465,10 +461,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope { private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope {
map { id -> map { id ->
async { async {
runCatching { runCatchingCancellable {
val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html") val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let {
.parseRaw().let { rewriteTnPaths(it) } val baseUri = it.request.url.toString()
.let(Jsoup::parse) val html = it.parseRaw()
Jsoup.parse(rewriteTnPaths(html), baseUri)
}
Manga( Manga(
id = generateUid(id.toString()), id = generateUid(id.toString()),
@ -476,9 +474,9 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
url = id.toString(), url = id.toString(),
coverUrl = coverUrl =
"https:" + "https:" +
doc.selectFirstOrThrow("picture > source") doc.selectFirstOrThrow("picture > source")
.attr("data-srcset") .attr("data-srcset")
.substringBefore(" "), .substringBefore(" "),
publicUrl = publicUrl =
doc.selectFirstOrThrow("h1 > a") doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href") .attrAsRelativeUrl("href")
@ -673,7 +671,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1") return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1")
} }
private suspend fun subdomainFromURL(url: String, base: String? = null) : String { private suspend fun subdomainFromURL(url: String, base: String? = null): String {
var retval = "b" var retval = "b"
if (!base.isNullOrBlank()) if (!base.isNullOrBlank())
@ -681,10 +679,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
val regex = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""") val regex = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""")
val hashMatch = regex.find(url) ?: return "a" val hashMatch = regex.find(url) ?: return "a"
val imageId = hashMatch.groupValues.let { it[2]+it[1] }.toIntOrNull(16) val imageId = hashMatch.groupValues.let { it[2] + it[1] }.toIntOrNull(16)
if (imageId != null) { if (imageId != null) {
retval = ('a'+ subdomainOffset(imageId)).toString() + retval retval = ('a' + subdomainOffset(imageId)).toString() + retval
} }
return retval return retval
@ -699,4 +697,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
return html.replace(tnRegex, newUrl) return html.replace(tnRegex, newUrl)
} }
private fun String.toTagTitle(): String {
return toCamelCase()
.replace("♂", "(male)")
.replace("♀", "(female)")
}
} }

27
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt
Unescape Escape View File

@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers
import kotlinx.coroutines.test.runTest import kotlinx.coroutines.test.runTest
import okhttp3.HttpUrl import okhttp3.HttpUrl
import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.extension.ExtendWith
import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.ParameterizedTest
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaListFilter import org.koitharu.kotatsu.parsers.model.MangaListFilter
@ -16,16 +15,18 @@ import org.koitharu.kotatsu.test_util.isDistinct
import org.koitharu.kotatsu.test_util.isDistinctBy import org.koitharu.kotatsu.test_util.isDistinctBy
import org.koitharu.kotatsu.test_util.isUrlAbsolute import org.koitharu.kotatsu.test_util.isUrlAbsolute
import org.koitharu.kotatsu.test_util.maxDuplicates import org.koitharu.kotatsu.test_util.maxDuplicates
import kotlin.time.Duration.Companion.minutes
@ExtendWith(AuthCheckExtension::class) //@ExtendWith(AuthCheckExtension::class)
internal class MangaParserTest { internal class MangaParserTest {
private val context = MangaLoaderContextMock private val context = MangaLoaderContextMock
private val timeout = 2.minutes
@ParameterizedTest(name = "{index}|list|{0}") @ParameterizedTest(name = "{index}|list|{0}")
@MangaSources @MangaSources
fun list(source: MangaSource) = runTest { fun list(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null)
checkMangaList(list, "list") checkMangaList(list, "list")
@ -34,7 +35,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|pagination|{0}") @ParameterizedTest(name = "{index}|pagination|{0}")
@MangaSources @MangaSources
fun pagination(source: MangaSource) = runTest { fun pagination(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val page1 = parser.getList(0, filter = null) val page1 = parser.getList(0, filter = null)
val page2 = parser.getList(page1.size, filter = null) val page2 = parser.getList(page1.size, filter = null)
@ -54,7 +55,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|search|{0}") @ParameterizedTest(name = "{index}|search|{0}")
@MangaSources @MangaSources
fun search(source: MangaSource) = runTest { fun search(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val subject = parser.getList( val subject = parser.getList(
offset = 0, offset = 0,
@ -78,7 +79,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|tags|{0}") @ParameterizedTest(name = "{index}|tags|{0}")
@MangaSources @MangaSources
fun tags(source: MangaSource) = runTest { fun tags(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val tags = parser.getAvailableTags() val tags = parser.getAvailableTags()
assert(tags.isNotEmpty()) { "No tags found" } assert(tags.isNotEmpty()) { "No tags found" }
@ -98,7 +99,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|tags_multiple|{0}") @ParameterizedTest(name = "{index}|tags_multiple|{0}")
@MangaSources @MangaSources
fun tagsMultiple(source: MangaSource) = runTest { fun tagsMultiple(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val tags = parser.getAvailableTags().shuffled().take(2).toSet() val tags = parser.getAvailableTags().shuffled().take(2).toSet()
@ -117,7 +118,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|locale|{0}") @ParameterizedTest(name = "{index}|locale|{0}")
@MangaSources @MangaSources
fun locale(source: MangaSource) = runTest { fun locale(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val locales = parser.getAvailableLocales() val locales = parser.getAvailableLocales()
if (locales.isEmpty()) { if (locales.isEmpty()) {
@ -137,7 +138,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|details|{0}") @ParameterizedTest(name = "{index}|details|{0}")
@MangaSources @MangaSources
fun details(source: MangaSource) = runTest { fun details(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list[3] val manga = list[3]
@ -166,7 +167,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|pages|{0}") @ParameterizedTest(name = "{index}|pages|{0}")
@MangaSources @MangaSources
fun pages(source: MangaSource) = runTest { fun pages(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val list = parser.getList(0, sortOrder = SortOrder.UPDATED, tags = null) val list = parser.getList(0, sortOrder = SortOrder.UPDATED, tags = null)
val manga = list.first() val manga = list.first()
@ -190,7 +191,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|favicon|{0}") @ParameterizedTest(name = "{index}|favicon|{0}")
@MangaSources @MangaSources
fun favicon(source: MangaSource) = runTest { fun favicon(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val favicons = parser.getFavicons() val favicons = parser.getFavicons()
val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg")
@ -206,7 +207,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|domain|{0}") @ParameterizedTest(name = "{index}|domain|{0}")
@MangaSources @MangaSources
fun domain(source: MangaSource) = runTest { fun domain(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val defaultDomain = parser.domain val defaultDomain = parser.domain
val url = HttpUrl.Builder().host(defaultDomain).scheme("https").toString() val url = HttpUrl.Builder().host(defaultDomain).scheme("https").toString()
@ -222,7 +223,7 @@ internal class MangaParserTest {
@ParameterizedTest(name = "{index}|authorization|{0}") @ParameterizedTest(name = "{index}|authorization|{0}")
@MangaSources @MangaSources
@Disabled @Disabled
fun authorization(source: MangaSource) = runTest { fun authorization(source: MangaSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
if (parser is MangaParserAuthProvider) { if (parser is MangaParserAuthProvider) {
val username = parser.getUsername() val username = parser.getUsername()

Write Preview
Loading…
Cancel
Save