Improve unit tests

.gitignore

@@ -73,4 +73,6 @@ build/
 .idea/**/misc.xml
 .idea/**/vcs.xml
 .idea/**/ktlint.xml
-.idea/codeStyles/
+.idea/codeStyles/
+
+src/test/resources/cookies.txt

build.gradle

@@ -20,13 +20,17 @@ compileKotlin {
 		jvmTarget = '1.8'
 		freeCompilerArgs += [
 				'-Xopt-in=kotlin.contracts.ExperimentalContracts',
-				'-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
 		]
 	}
 }
 
 compileTestKotlin {
-	kotlinOptions.jvmTarget = '1.8'
+	kotlinOptions {
+		jvmTarget = '1.8'
+		freeCompilerArgs += [
+				'-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
+		]
+	}
 }
 
 dependencies {

src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt

@@ -4,6 +4,7 @@ import okhttp3.*
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.RequestBody.Companion.toRequestBody
 import org.json.JSONObject
+import org.jsoup.HttpStatusException
 import org.koitharu.kotatsu.parsers.exception.GraphQLException
 import org.koitharu.kotatsu.parsers.model.MangaSource
 import org.koitharu.kotatsu.parsers.util.await
@@ -23,7 +24,7 @@ abstract class MangaLoaderContext {
 		if (headers != null) {
 			request.headers(headers)
 		}
-		return httpClient.newCall(request.build()).await()
+		return httpClient.newCall(request.build()).await().ensureSuccess()
 	}
 
 	suspend fun httpPost(
@@ -37,7 +38,7 @@ abstract class MangaLoaderContext {
 		val request = Request.Builder()
 			.post(body.build())
 			.url(url)
-		return httpClient.newCall(request.build()).await()
+		return httpClient.newCall(request.build()).await().ensureSuccess()
 	}
 
 	suspend fun httpPost(
@@ -56,7 +57,7 @@ abstract class MangaLoaderContext {
 		val request = Request.Builder()
 			.post(body.build())
 			.url(url)
-		return httpClient.newCall(request.build()).await()
+		return httpClient.newCall(request.build()).await().ensureSuccess()
 	}
 
 	suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
@@ -69,7 +70,7 @@ abstract class MangaLoaderContext {
 		val request = Request.Builder()
 			.post(requestBody)
 			.url(endpoint)
-		val json = httpClient.newCall(request.build()).await().parseJson()
+		val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson()
 		json.optJSONArray("errors")?.let {
 			if (it.length() != 0) {
 				throw GraphQLException(it)
@@ -87,4 +88,16 @@ abstract class MangaLoaderContext {
 	abstract suspend fun evaluateJs(script: String): String?
 
 	abstract fun getConfig(source: MangaSource): MangaSourceConfig
+
+	private fun Response.ensureSuccess() = apply {
+		if (!isSuccessful) {
+			val exception = HttpStatusException(message, code, request.url.toString())
+			runCatching {
+				close()
+			}.onFailure {
+				exception.addSuppressed(it)
+			}
+			throw exception
+		}
+	}
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt

@@ -213,7 +213,7 @@ internal class ExHentaiParser(override val context: MangaLoaderContext) : MangaP
 	override suspend fun getUsername(): String {
 		val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
 		val username = doc.getElementById("userlinks")
-			?.getElementsByAttributeValueContaining("href", "?showuser=")
+			?.getElementsByAttributeValueContaining("href", "showuser=")
 			?.firstOrNull()
 			?.ownText()
 			?: if (doc.getElementById("userlinksguest") != null) {

src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaLibParser.kt

@@ -236,7 +236,9 @@ internal open class MangaLibParser(override val context: MangaLoaderContext) : M
 	}
 
 	protected open fun isNsfw(doc: Document): Boolean {
-		val sidebar = doc.body().selectFirst(".media-sidebar") ?: parseFailed("Sidebar not found")
+		val sidebar = doc.body().run {
+			selectFirst(".media-sidebar") ?: selectFirst(".media-info")
+		} ?: parseFailed("Sidebar not found")
 		return sidebar.getElementsContainingOwnText("18+").isNotEmpty()
 	}
 
@@ -257,12 +259,13 @@ internal open class MangaLibParser(override val context: MangaLoaderContext) : M
 			val slug = jo.getString("slug")
 			val url = "/$slug"
 			val covers = jo.getJSONObject("covers")
+			val title = jo.getString("rus_name").ifEmpty { jo.getString("name") }
 			Manga(
 				id = generateUid(url),
 				url = url,
 				publicUrl = "https://$domain/$slug",
-				title = jo.getString("rus_name"),
-				altTitle = jo.getString("name"),
+				title = title,
+				altTitle = jo.getString("name").takeUnless { it == title },
 				author = null,
 				tags = emptySet(),
 				rating = jo.getString("rate_avg")

src/main/kotlin/org/koitharu/kotatsu/parsers/util/CookieJarExt.kt

@@ -19,6 +19,14 @@ fun CookieJar.insertCookies(domain: String, vararg cookies: String) {
 	)
 }
 
+fun CookieJar.insertCookie(domain: String, cookie: Cookie) {
+	val url = HttpUrl.Builder()
+		.scheme(SCHEME_HTTPS)
+		.host(domain)
+		.build()
+	saveFromResponse(url, listOf(cookie))
+}
+
 fun CookieJar.getCookies(domain: String): List<Cookie> {
 	val url = HttpUrl.Builder()
 		.scheme(SCHEME_HTTPS)

src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt

@@ -0,0 +1,33 @@
+package org.koitharu.kotatsu.parsers
+
+import kotlinx.coroutines.test.runTest
+import org.junit.jupiter.api.extension.BeforeAllCallback
+import org.junit.jupiter.api.extension.ExtensionContext
+import org.koitharu.kotatsu.parsers.model.MangaSource
+
+class AuthCheckExtension : BeforeAllCallback {
+
+	private val loaderContext: MangaLoaderContext = MangaLoaderContextMock()
+
+	override fun beforeAll(context: ExtensionContext) {
+		for (source in MangaSource.values()) {
+			if (source == MangaSource.LOCAL) {
+				continue
+			}
+			val parser = source.newParser(loaderContext)
+			if (parser is MangaParserAuthProvider) {
+				checkAuthorization(source, parser)
+			}
+		}
+	}
+
+	private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest {
+		runCatching {
+			parser.getUsername()
+		}.onSuccess { username ->
+			println("Signed in to ${source.name} as $username")
+		}.onFailure { error ->
+			System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})")
+		}
+	}
+}

src/test/kotlin/org/koitharu/kotatsu/parsers/InMemoryCookieJar.kt

@@ -3,6 +3,12 @@ package org.koitharu.kotatsu.parsers
 import okhttp3.Cookie
 import okhttp3.CookieJar
 import okhttp3.HttpUrl
+import org.koitharu.kotatsu.parsers.util.insertCookie
+import org.koitharu.kotatsu.test_util.component6
+import org.koitharu.kotatsu.test_util.component7
+import java.io.InputStream
+import java.util.*
+import java.util.concurrent.TimeUnit
 
 class InMemoryCookieJar : CookieJar {
 
@@ -20,6 +26,27 @@ class InMemoryCookieJar : CookieJar {
 		}
 	}
 
+	fun loadFromStream(stream: InputStream) {
+		val reader = stream.bufferedReader()
+		for (line in reader.lineSequence()) {
+			if (line.isBlank() || line.startsWith("# ")) {
+				continue
+			}
+			val (host, includeSubdomains, path, secure, expire, name, value) = line.split(Regex("\\s+"))
+			val domain = host.removePrefix("#HttpOnly_").trimStart('.')
+			val httpOnly = host.startsWith("#HttpOnly_")
+			val cookie = Cookie.Builder()
+			cookie.domain(domain)
+			if (httpOnly) cookie.httpOnly()
+			cookie.path(path)
+			if (secure.lowercase(Locale.ROOT).toBooleanStrict()) cookie.secure()
+			cookie.expiresAt(TimeUnit.SECONDS.toMillis(expire.toLong()))
+			cookie.name(name)
+			cookie.value(value)
+			insertCookie(domain, cookie.build())
+		}
+	}
+
 	private data class CookieKey(
 		val host: String,
 		val name: String,

src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt

@@ -1,7 +1,6 @@
 package org.koitharu.kotatsu.parsers
 
 import com.koushikdutta.quack.QuackContext
-import okhttp3.CookieJar
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
@@ -17,10 +16,10 @@ internal class MangaLoaderContextMock : MangaLoaderContext() {
 		/*Build.MODEL*/ "",
 		/*Build.BRAND*/ "",
 		/*Build.DEVICE*/ "",
-		/*Locale.getDefault().language*/"en",
+		/*Locale.getDefault().language*/ "en",
 	)
 
-	override val cookieJar: CookieJar = InMemoryCookieJar()
+	override val cookieJar = InMemoryCookieJar()
 
 	override val httpClient: OkHttpClient = OkHttpClient.Builder()
 		.cookieJar(cookieJar)
@@ -30,6 +29,10 @@ internal class MangaLoaderContextMock : MangaLoaderContext() {
 		.writeTimeout(20, TimeUnit.SECONDS)
 		.build()
 
+	init {
+		loadTestCookies()
+	}
+
 	override suspend fun evaluateJs(script: String): String? {
 		return QuackContext.create().use {
 			it.evaluate(script)?.toString()
@@ -40,12 +43,20 @@ internal class MangaLoaderContextMock : MangaLoaderContext() {
 		return SourceConfigMock()
 	}
 
-	suspend fun doRequest(url: String, builder: Request.Builder.() -> Unit): Response {
+	suspend fun doRequest(url: String, referer: String? = null): Response {
 		val request = Request.Builder()
 			.get()
 			.url(url)
-			.apply(builder)
-			.build()
-		return httpClient.newCall(request).await()
+		if (referer != null) {
+			request.header("Referrer", referer)
+		}
+		return httpClient.newCall(request.build()).await()
+	}
+
+	private fun loadTestCookies() {
+		// https://addons.mozilla.org/ru/firefox/addon/cookies-txt/
+		javaClass.getResourceAsStream("/cookies.txt")?.use {
+			cookieJar.loadFromStream(it)
+		} ?: println("No cookies loaded!")
 	}
 }

src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt

@@ -1,6 +1,8 @@
 package org.koitharu.kotatsu.parsers
 
 import kotlinx.coroutines.test.runTest
+import org.junit.jupiter.api.Disabled
+import org.junit.jupiter.api.extension.ExtendWith
 import org.junit.jupiter.params.ParameterizedTest
 import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaSource
@@ -10,7 +12,9 @@ import org.koitharu.kotatsu.parsers.util.mimeType
 import org.koitharu.kotatsu.test_util.isDistinct
 import org.koitharu.kotatsu.test_util.isDistinctBy
 import org.koitharu.kotatsu.test_util.isUrlAbsoulte
+import org.koitharu.kotatsu.test_util.maxDuplicates
 
+@ExtendWith(AuthCheckExtension::class)
 internal class MangaParserTest {
 
 	private val context = MangaLoaderContextMock()
@@ -20,7 +24,7 @@ internal class MangaParserTest {
 	fun list(source: MangaSource) = runTest {
 		val parser = source.newParser(context)
 		val list = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
-		checkMangaList(list)
+		checkMangaList(list, "list")
 		assert(list.all { it.source == source })
 	}
 
@@ -31,11 +35,13 @@ internal class MangaParserTest {
 		val subject = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull {
 			it.title.length
 		} ?: error("No manga found")
-		val list = parser.getList(offset = 0, query = subject.title, sortOrder = null, tags = null)
+		val query = subject.title
+		check(query.isNotBlank()) { "Manga title '$query' is blank" }
+		val list = parser.getList(offset = 0, query, sortOrder = null, tags = null)
 		assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) {
-			"Single subject ${subject.title} not found in search results"
+			"Single subject '${subject.title} (${subject.publicUrl})' not found in search results"
 		}
-		checkMangaList(list)
+		checkMangaList(list, "search('$query')")
 		assert(list.all { it.source == source })
 	}
 
@@ -53,8 +59,9 @@ internal class MangaParserTest {
 		assert("" !in titles)
 		assert(tags.all { it.source == source })
 
-		val list = parser.getList(offset = 0, tags = setOf(tags.last()), query = null, sortOrder = null)
-		checkMangaList(list)
+		val tag = tags.last()
+		val list = parser.getList(offset = 0, tags = setOf(tag), query = null, sortOrder = null)
+		checkMangaList(list, "${tag.title} (${tag.key})")
 		assert(list.all { it.source == source })
 	}
 
@@ -65,16 +72,28 @@ internal class MangaParserTest {
 		val list = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
 		val manga = list[3]
 		parser.getDetails(manga).apply {
-			assert(!chapters.isNullOrEmpty())
-			assert(publicUrl.isUrlAbsoulte())
-			assert(description != null)
-			assert(title.startsWith(manga.title))
+			assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" }
+			assert(publicUrl.isUrlAbsoulte()) { "Manga public url is not absolute: '$publicUrl'" }
+			assert(description != null) { "Detailed description is null" }
+			assert(title.startsWith(manga.title)) {
+				"Titles are mismatch: '$title' and '${manga.title}' for $publicUrl"
+			}
 			assert(this.source == source)
 			val c = checkNotNull(chapters)
-			assert(c.isDistinctBy { it.id })
-			assert(c.isDistinctBy { it.number })
-			assert(c.isDistinctBy { it.name })
+			assert(c.isDistinctBy { it.id }) {
+				"Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl"
+			}
+			assert(c.isDistinctBy { it.number to it.branch }) {
+				"Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl"
+			}
+			assert(c.isDistinctBy { it.name to it.branch }) {
+				"Chapters are not distinct by name: ${c.maxDuplicates { it.name to it.branch }} for $publicUrl"
+			}
 			assert(c.all { it.source == source })
+			checkImageRequest(coverUrl, publicUrl)
+			largeCoverUrl?.let {
+				checkImageRequest(it, publicUrl)
+			}
 		}
 	}
 
@@ -95,22 +114,41 @@ internal class MangaParserTest {
 		val pageUrl = parser.getPageUrl(page)
 		assert(pageUrl.isNotEmpty())
 		assert(pageUrl.isUrlAbsoulte())
-		val pageResponse = context.doRequest(pageUrl) {
-			header("Referrer", page.referer)
-		}
-		assert(pageResponse.isSuccessful)
-		assert(pageResponse.mimeType?.startsWith("image/") == true)
+		checkImageRequest(pageUrl, page.referer)
 	}
 
-	private fun checkMangaList(list: List<Manga>) {
-		assert(list.isNotEmpty()) { "Manga list is empty" }
-		assert(list.isDistinctBy { it.id }) { "Manga list contains duplicated ids" }
+	@ParameterizedTest
+	@MangaSources
+	@Disabled
+	fun authorization(source: MangaSource) = runTest {
+		val parser = source.newParser(context)
+		if (parser is MangaParserAuthProvider) {
+			val username = parser.getUsername()
+			assert(username.isNotBlank()) { "Username is blank" }
+			println("Signed in to ${source.name} as $username")
+		}
+	}
+
+	private suspend fun checkMangaList(list: List<Manga>, cause: String) {
+		assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" }
+		assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" }
 		for (item in list) {
 			assert(item.url.isNotEmpty())
 			assert(!item.url.isUrlAbsoulte())
 			assert(item.coverUrl.isUrlAbsoulte())
-			assert(item.title.isNotEmpty())
+			assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" }
 			assert(item.publicUrl.isUrlAbsoulte())
 		}
+		val testItem = list.random()
+		checkImageRequest(testItem.coverUrl, testItem.publicUrl)
+	}
+
+	private suspend fun checkImageRequest(url: String, referer: String?) {
+		context.doRequest(url, referer).use {
+			assert(it.isSuccessful) { "Request failed: ${it.code}: ${it.message}" }
+			assert(it.mimeType?.startsWith("image/") == true) {
+				"Wrong response mime type: ${it.mimeType}"
+			}
+		}
 	}
 }

src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt

@@ -26,4 +26,18 @@ internal fun <T, K> Collection<T>.isDistinctBy(selector: (T) -> K): Boolean {
 }
 
 internal fun String.isUrlRelative() = matches(PATTERN_URL_RELATIVE)
-internal fun String.isUrlAbsoulte() = matches(PATTERN_URL_ABSOLUTE)
+internal fun String.isUrlAbsoulte() = matches(PATTERN_URL_ABSOLUTE)
+
+internal inline fun <T, K> Collection<T>.maxDuplicates(selector: (T) -> K): K? {
+	return groupBy(selector).maxByOrNull { it.value.size }?.key
+}
+
+@Suppress("NOTHING_TO_INLINE")
+inline operator fun <T> List<T>.component6(): T {
+	return get(5)
+}
+
+@Suppress("NOTHING_TO_INLINE")
+inline operator fun <T> List<T>.component7(): T {
+	return get(6)
+}