diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt index 27c2ecb2..27ac2306 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt @@ -90,7 +90,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo ?: return@mapNotNull null MangaTag( - title = url.ownText().toCamelCase(), + title = url.ownText().toTagTitle(), key = href.tagUrlToTag(), source = source, ) @@ -158,15 +158,14 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo language: String = "all", ): Set = coroutineScope { - val terms = - query - .trim() - .replace(Regex("""^\?"""), "") - .lowercase() - .split(Regex("\\s+")) - .map { - it.replace('_', ' ') - } + val terms = query + .trim() + .replace(Regex("""^\?"""), "") + .lowercase() + .split(Regex("\\s+")) + .map { + it.replace('_', ' ') + } val positiveTerms = LinkedList() val negativeTerms = LinkedList() @@ -179,30 +178,27 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo } } - val positiveResults = - positiveTerms.map { - async { - runCatching { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } + val positiveResults = positiveTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) } + } - val negativeResults = - negativeTerms.map { - async { - runCatching { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } + val negativeResults = negativeTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) } + } - val results = - when { - sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language) - positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) - else -> emptySet() - }.toMutableSet() + val results = when { + sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language) + positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) + else -> emptySet() + }.toMutableSet() fun filterPositive(newResults: Set) { when { @@ -465,10 +461,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo private suspend fun Collection.toMangaList(): List = coroutineScope { map { id -> async { - runCatching { - val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html") - .parseRaw().let { rewriteTnPaths(it) } - .let(Jsoup::parse) + runCatchingCancellable { + val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let { + val baseUri = it.request.url.toString() + val html = it.parseRaw() + Jsoup.parse(rewriteTnPaths(html), baseUri) + } Manga( id = generateUid(id.toString()), @@ -476,9 +474,9 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo url = id.toString(), coverUrl = "https:" + - doc.selectFirstOrThrow("picture > source") - .attr("data-srcset") - .substringBefore(" "), + doc.selectFirstOrThrow("picture > source") + .attr("data-srcset") + .substringBefore(" "), publicUrl = doc.selectFirstOrThrow("h1 > a") .attrAsRelativeUrl("href") @@ -673,7 +671,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1") } - private suspend fun subdomainFromURL(url: String, base: String? = null) : String { + private suspend fun subdomainFromURL(url: String, base: String? = null): String { var retval = "b" if (!base.isNullOrBlank()) @@ -681,10 +679,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo val regex = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""") val hashMatch = regex.find(url) ?: return "a" - val imageId = hashMatch.groupValues.let { it[2]+it[1] }.toIntOrNull(16) + val imageId = hashMatch.groupValues.let { it[2] + it[1] }.toIntOrNull(16) if (imageId != null) { - retval = ('a'+ subdomainOffset(imageId)).toString() + retval + retval = ('a' + subdomainOffset(imageId)).toString() + retval } return retval @@ -699,4 +697,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo return html.replace(tnRegex, newUrl) } + + private fun String.toTagTitle(): String { + return toCamelCase() + .replace("♂", "(male)") + .replace("♀", "(female)") + } } diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index b3cf9130..b086e735 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers import kotlinx.coroutines.test.runTest import okhttp3.HttpUrl import org.junit.jupiter.api.Disabled -import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.params.ParameterizedTest import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaListFilter @@ -16,16 +15,18 @@ import org.koitharu.kotatsu.test_util.isDistinct import org.koitharu.kotatsu.test_util.isDistinctBy import org.koitharu.kotatsu.test_util.isUrlAbsolute import org.koitharu.kotatsu.test_util.maxDuplicates +import kotlin.time.Duration.Companion.minutes -@ExtendWith(AuthCheckExtension::class) +//@ExtendWith(AuthCheckExtension::class) internal class MangaParserTest { private val context = MangaLoaderContextMock + private val timeout = 2.minutes @ParameterizedTest(name = "{index}|list|{0}") @MangaSources - fun list(source: MangaSource) = runTest { + fun list(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) checkMangaList(list, "list") @@ -34,7 +35,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|pagination|{0}") @MangaSources - fun pagination(source: MangaSource) = runTest { + fun pagination(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val page1 = parser.getList(0, filter = null) val page2 = parser.getList(page1.size, filter = null) @@ -54,7 +55,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|search|{0}") @MangaSources - fun search(source: MangaSource) = runTest { + fun search(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val subject = parser.getList( offset = 0, @@ -78,7 +79,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|tags|{0}") @MangaSources - fun tags(source: MangaSource) = runTest { + fun tags(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val tags = parser.getAvailableTags() assert(tags.isNotEmpty()) { "No tags found" } @@ -98,7 +99,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|tags_multiple|{0}") @MangaSources - fun tagsMultiple(source: MangaSource) = runTest { + fun tagsMultiple(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val tags = parser.getAvailableTags().shuffled().take(2).toSet() @@ -117,7 +118,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|locale|{0}") @MangaSources - fun locale(source: MangaSource) = runTest { + fun locale(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val locales = parser.getAvailableLocales() if (locales.isEmpty()) { @@ -137,7 +138,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|details|{0}") @MangaSources - fun details(source: MangaSource) = runTest { + fun details(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) val manga = list[3] @@ -166,7 +167,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|pages|{0}") @MangaSources - fun pages(source: MangaSource) = runTest { + fun pages(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val list = parser.getList(0, sortOrder = SortOrder.UPDATED, tags = null) val manga = list.first() @@ -190,7 +191,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|favicon|{0}") @MangaSources - fun favicon(source: MangaSource) = runTest { + fun favicon(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val favicons = parser.getFavicons() val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") @@ -206,7 +207,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|domain|{0}") @MangaSources - fun domain(source: MangaSource) = runTest { + fun domain(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) val defaultDomain = parser.domain val url = HttpUrl.Builder().host(defaultDomain).scheme("https").toString() @@ -222,7 +223,7 @@ internal class MangaParserTest { @ParameterizedTest(name = "{index}|authorization|{0}") @MangaSources @Disabled - fun authorization(source: MangaSource) = runTest { + fun authorization(source: MangaSource) = runTest(timeout = timeout) { val parser = context.newParserInstance(source) if (parser is MangaParserAuthProvider) { val username = parser.getUsername()