[HitomiLa] Fix tag names

2023-12-25 19:48:33 +02:00
parent 52ac3d2e6c
commit 904e0719eb
2 changed files with 58 additions and 53 deletions
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
@@ -90,7 +90,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
 								?: return@mapNotNull null

 						MangaTag(
-							title = url.ownText().toCamelCase(),
+							title = url.ownText().toTagTitle(),
 							key = href.tagUrlToTag(),
 							source = source,
 						)
@@ -158,15 +158,14 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
 		language: String = "all",
 	): Set<Int> =
 		coroutineScope {
-			val terms =
-				query
-					.trim()
-					.replace(Regex("""^\?"""), "")
-					.lowercase()
-					.split(Regex("\\s+"))
-					.map {
-						it.replace('_', ' ')
-					}
+			val terms = query
+				.trim()
+				.replace(Regex("""^\?"""), "")
+				.lowercase()
+				.split(Regex("\\s+"))
+				.map {
+					it.replace('_', ' ')
+				}

 			val positiveTerms = LinkedList<String>()
 			val negativeTerms = LinkedList<String>()
@@ -179,30 +178,27 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
 				}
 			}

-			val positiveResults =
-				positiveTerms.map {
-					async {
-						runCatching {
-							getGalleryIDsForQuery(it, language)
-						}.getOrDefault(emptySet())
-					}
+			val positiveResults = positiveTerms.map {
+				async {
+					runCatchingCancellable {
+						getGalleryIDsForQuery(it, language)
+					}.getOrDefault(emptySet())
 				}
+			}

-			val negativeResults =
-				negativeTerms.map {
-					async {
-						runCatching {
-							getGalleryIDsForQuery(it, language)
-						}.getOrDefault(emptySet())
-					}
+			val negativeResults = negativeTerms.map {
+				async {
+					runCatchingCancellable {
+						getGalleryIDsForQuery(it, language)
+					}.getOrDefault(emptySet())
 				}
+			}

-			val results =
-				when {
-					sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language)
-					positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language)
-					else -> emptySet()
-				}.toMutableSet()
+			val results = when {
+				sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language)
+				positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language)
+				else -> emptySet()
+			}.toMutableSet()

 			fun filterPositive(newResults: Set<Int>) {
 				when {
@@ -465,10 +461,12 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
 	private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope {
 		map { id ->
 			async {
-				runCatching {
-					val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html")
-						.parseRaw().let { rewriteTnPaths(it) }
-						.let(Jsoup::parse)
+				runCatchingCancellable {
+					val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let {
+						val baseUri = it.request.url.toString()
+						val html = it.parseRaw()
+						Jsoup.parse(rewriteTnPaths(html), baseUri)
+					}

 					Manga(
 						id = generateUid(id.toString()),
@@ -476,9 +474,9 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
 						url = id.toString(),
 						coverUrl =
 						"https:" +
-								doc.selectFirstOrThrow("picture > source")
-									.attr("data-srcset")
-									.substringBefore(" "),
+							doc.selectFirstOrThrow("picture > source")
+								.attr("data-srcset")
+								.substringBefore(" "),
 						publicUrl =
 						doc.selectFirstOrThrow("h1 > a")
 							.attrAsRelativeUrl("href")
@@ -673,7 +671,7 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo
 		return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1")
 	}

-	private suspend fun subdomainFromURL(url: String, base: String? = null) : String {
+	private suspend fun subdomainFromURL(url: String, base: String? = null): String {
 		var retval = "b"

 		if (!base.isNullOrBlank())
@@ -681,10 +679,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo

 		val regex = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""")
 		val hashMatch = regex.find(url) ?: return "a"
-		val imageId = hashMatch.groupValues.let { it[2]+it[1] }.toIntOrNull(16)
+		val imageId = hashMatch.groupValues.let { it[2] + it[1] }.toIntOrNull(16)

 		if (imageId != null) {
-			retval = ('a'+ subdomainOffset(imageId)).toString() + retval
+			retval = ('a' + subdomainOffset(imageId)).toString() + retval
 		}

 		return retval
@@ -699,4 +697,10 @@ class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSo

 		return html.replace(tnRegex, newUrl)
 	}
+
+	private fun String.toTagTitle(): String {
+		return toCamelCase()
+			.replace("♂", "(male)")
+			.replace("♀", "(female)")
+	}
 }
--- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt
+++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt
@@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers
 import kotlinx.coroutines.test.runTest
 import okhttp3.HttpUrl
 import org.junit.jupiter.api.Disabled
-import org.junit.jupiter.api.extension.ExtendWith
 import org.junit.jupiter.params.ParameterizedTest
 import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaListFilter
@@ -16,16 +15,18 @@ import org.koitharu.kotatsu.test_util.isDistinct
 import org.koitharu.kotatsu.test_util.isDistinctBy
 import org.koitharu.kotatsu.test_util.isUrlAbsolute
 import org.koitharu.kotatsu.test_util.maxDuplicates
+import kotlin.time.Duration.Companion.minutes


-@ExtendWith(AuthCheckExtension::class)
+//@ExtendWith(AuthCheckExtension::class)
 internal class MangaParserTest {

 	private val context = MangaLoaderContextMock
+	private val timeout = 2.minutes

 	@ParameterizedTest(name = "{index}|list|{0}")
 	@MangaSources
-	fun list(source: MangaSource) = runTest {
+	fun list(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null)
 		checkMangaList(list, "list")
@@ -34,7 +35,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|pagination|{0}")
 	@MangaSources
-	fun pagination(source: MangaSource) = runTest {
+	fun pagination(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val page1 = parser.getList(0, filter = null)
 		val page2 = parser.getList(page1.size, filter = null)
@@ -54,7 +55,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|search|{0}")
 	@MangaSources
-	fun search(source: MangaSource) = runTest {
+	fun search(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val subject = parser.getList(
 			offset = 0,
@@ -78,7 +79,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|tags|{0}")
 	@MangaSources
-	fun tags(source: MangaSource) = runTest {
+	fun tags(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val tags = parser.getAvailableTags()
 		assert(tags.isNotEmpty()) { "No tags found" }
@@ -98,7 +99,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|tags_multiple|{0}")
 	@MangaSources
-	fun tagsMultiple(source: MangaSource) = runTest {
+	fun tagsMultiple(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val tags = parser.getAvailableTags().shuffled().take(2).toSet()

@@ -117,7 +118,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|locale|{0}")
 	@MangaSources
-	fun locale(source: MangaSource) = runTest {
+	fun locale(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val locales = parser.getAvailableLocales()
 		if (locales.isEmpty()) {
@@ -137,7 +138,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|details|{0}")
 	@MangaSources
-	fun details(source: MangaSource) = runTest {
+	fun details(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null)
 		val manga = list[3]
@@ -166,7 +167,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|pages|{0}")
 	@MangaSources
-	fun pages(source: MangaSource) = runTest {
+	fun pages(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val list = parser.getList(0, sortOrder = SortOrder.UPDATED, tags = null)
 		val manga = list.first()
@@ -190,7 +191,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|favicon|{0}")
 	@MangaSources
-	fun favicon(source: MangaSource) = runTest {
+	fun favicon(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val favicons = parser.getFavicons()
 		val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg")
@@ -206,7 +207,7 @@ internal class MangaParserTest {

 	@ParameterizedTest(name = "{index}|domain|{0}")
 	@MangaSources
-	fun domain(source: MangaSource) = runTest {
+	fun domain(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		val defaultDomain = parser.domain
 		val url = HttpUrl.Builder().host(defaultDomain).scheme("https").toString()
@@ -222,7 +223,7 @@ internal class MangaParserTest {
 	@ParameterizedTest(name = "{index}|authorization|{0}")
 	@MangaSources
 	@Disabled
-	fun authorization(source: MangaSource) = runTest {
+	fun authorization(source: MangaSource) = runTest(timeout = timeout) {
 		val parser = context.newParserInstance(source)
 		if (parser is MangaParserAuthProvider) {
 			val username = parser.getUsername()