diff --git a/.gitignore b/.gitignore index b0ef1211..49187094 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,6 @@ build/ .idea/**/misc.xml .idea/**/vcs.xml .idea/**/ktlint.xml -.idea/codeStyles/ \ No newline at end of file +.idea/codeStyles/ + +src/test/resources/cookies.txt \ No newline at end of file diff --git a/build.gradle b/build.gradle index b014e95c..873728db 100644 --- a/build.gradle +++ b/build.gradle @@ -20,13 +20,17 @@ compileKotlin { jvmTarget = '1.8' freeCompilerArgs += [ '-Xopt-in=kotlin.contracts.ExperimentalContracts', - '-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi', ] } } compileTestKotlin { - kotlinOptions.jvmTarget = '1.8' + kotlinOptions { + jvmTarget = '1.8' + freeCompilerArgs += [ + '-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi', + ] + } } dependencies { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt index e627e343..ccb9e931 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt @@ -4,6 +4,7 @@ import okhttp3.* import okhttp3.MediaType.Companion.toMediaType import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject +import org.jsoup.HttpStatusException import org.koitharu.kotatsu.parsers.exception.GraphQLException import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.util.await @@ -23,7 +24,7 @@ abstract class MangaLoaderContext { if (headers != null) { request.headers(headers) } - return httpClient.newCall(request.build()).await() + return httpClient.newCall(request.build()).await().ensureSuccess() } suspend fun httpPost( @@ -37,7 +38,7 @@ abstract class MangaLoaderContext { val request = Request.Builder() .post(body.build()) .url(url) - return httpClient.newCall(request.build()).await() + return httpClient.newCall(request.build()).await().ensureSuccess() } suspend fun httpPost( @@ -56,7 +57,7 @@ abstract class MangaLoaderContext { val request = Request.Builder() .post(body.build()) .url(url) - return httpClient.newCall(request.build()).await() + return httpClient.newCall(request.build()).await().ensureSuccess() } suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { @@ -69,7 +70,7 @@ abstract class MangaLoaderContext { val request = Request.Builder() .post(requestBody) .url(endpoint) - val json = httpClient.newCall(request.build()).await().parseJson() + val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson() json.optJSONArray("errors")?.let { if (it.length() != 0) { throw GraphQLException(it) @@ -87,4 +88,16 @@ abstract class MangaLoaderContext { abstract suspend fun evaluateJs(script: String): String? abstract fun getConfig(source: MangaSource): MangaSourceConfig + + private fun Response.ensureSuccess() = apply { + if (!isSuccessful) { + val exception = HttpStatusException(message, code, request.url.toString()) + runCatching { + close() + }.onFailure { + exception.addSuppressed(it) + } + throw exception + } + } } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 6c8ed79b..abfb9bd7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -213,7 +213,7 @@ internal class ExHentaiParser(override val context: MangaLoaderContext) : MangaP override suspend fun getUsername(): String { val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() val username = doc.getElementById("userlinks") - ?.getElementsByAttributeValueContaining("href", "?showuser=") + ?.getElementsByAttributeValueContaining("href", "showuser=") ?.firstOrNull() ?.ownText() ?: if (doc.getElementById("userlinksguest") != null) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaLibParser.kt index bd2414eb..a7acf379 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaLibParser.kt @@ -236,7 +236,9 @@ internal open class MangaLibParser(override val context: MangaLoaderContext) : M } protected open fun isNsfw(doc: Document): Boolean { - val sidebar = doc.body().selectFirst(".media-sidebar") ?: parseFailed("Sidebar not found") + val sidebar = doc.body().run { + selectFirst(".media-sidebar") ?: selectFirst(".media-info") + } ?: parseFailed("Sidebar not found") return sidebar.getElementsContainingOwnText("18+").isNotEmpty() } @@ -257,12 +259,13 @@ internal open class MangaLibParser(override val context: MangaLoaderContext) : M val slug = jo.getString("slug") val url = "/$slug" val covers = jo.getJSONObject("covers") + val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } Manga( id = generateUid(url), url = url, publicUrl = "https://$domain/$slug", - title = jo.getString("rus_name"), - altTitle = jo.getString("name"), + title = title, + altTitle = jo.getString("name").takeUnless { it == title }, author = null, tags = emptySet(), rating = jo.getString("rate_avg") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CookieJarExt.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CookieJarExt.kt index c4635fdb..7ff3cc82 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CookieJarExt.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CookieJarExt.kt @@ -19,6 +19,14 @@ fun CookieJar.insertCookies(domain: String, vararg cookies: String) { ) } +fun CookieJar.insertCookie(domain: String, cookie: Cookie) { + val url = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(domain) + .build() + saveFromResponse(url, listOf(cookie)) +} + fun CookieJar.getCookies(domain: String): List { val url = HttpUrl.Builder() .scheme(SCHEME_HTTPS) diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt new file mode 100644 index 00000000..4a10d2be --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt @@ -0,0 +1,33 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.extension.BeforeAllCallback +import org.junit.jupiter.api.extension.ExtensionContext +import org.koitharu.kotatsu.parsers.model.MangaSource + +class AuthCheckExtension : BeforeAllCallback { + + private val loaderContext: MangaLoaderContext = MangaLoaderContextMock() + + override fun beforeAll(context: ExtensionContext) { + for (source in MangaSource.values()) { + if (source == MangaSource.LOCAL) { + continue + } + val parser = source.newParser(loaderContext) + if (parser is MangaParserAuthProvider) { + checkAuthorization(source, parser) + } + } + } + + private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { + runCatching { + parser.getUsername() + }.onSuccess { username -> + println("Signed in to ${source.name} as $username") + }.onFailure { error -> + System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})") + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/InMemoryCookieJar.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/InMemoryCookieJar.kt index 45a232d7..34c16f54 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/InMemoryCookieJar.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/InMemoryCookieJar.kt @@ -3,6 +3,12 @@ package org.koitharu.kotatsu.parsers import okhttp3.Cookie import okhttp3.CookieJar import okhttp3.HttpUrl +import org.koitharu.kotatsu.parsers.util.insertCookie +import org.koitharu.kotatsu.test_util.component6 +import org.koitharu.kotatsu.test_util.component7 +import java.io.InputStream +import java.util.* +import java.util.concurrent.TimeUnit class InMemoryCookieJar : CookieJar { @@ -20,6 +26,27 @@ class InMemoryCookieJar : CookieJar { } } + fun loadFromStream(stream: InputStream) { + val reader = stream.bufferedReader() + for (line in reader.lineSequence()) { + if (line.isBlank() || line.startsWith("# ")) { + continue + } + val (host, includeSubdomains, path, secure, expire, name, value) = line.split(Regex("\\s+")) + val domain = host.removePrefix("#HttpOnly_").trimStart('.') + val httpOnly = host.startsWith("#HttpOnly_") + val cookie = Cookie.Builder() + cookie.domain(domain) + if (httpOnly) cookie.httpOnly() + cookie.path(path) + if (secure.lowercase(Locale.ROOT).toBooleanStrict()) cookie.secure() + cookie.expiresAt(TimeUnit.SECONDS.toMillis(expire.toLong())) + cookie.name(name) + cookie.value(value) + insertCookie(domain, cookie.build()) + } + } + private data class CookieKey( val host: String, val name: String, diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt index e45e751c..d2696b22 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt @@ -1,7 +1,6 @@ package org.koitharu.kotatsu.parsers import com.koushikdutta.quack.QuackContext -import okhttp3.CookieJar import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -17,10 +16,10 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { /*Build.MODEL*/ "", /*Build.BRAND*/ "", /*Build.DEVICE*/ "", - /*Locale.getDefault().language*/"en", + /*Locale.getDefault().language*/ "en", ) - override val cookieJar: CookieJar = InMemoryCookieJar() + override val cookieJar = InMemoryCookieJar() override val httpClient: OkHttpClient = OkHttpClient.Builder() .cookieJar(cookieJar) @@ -30,6 +29,10 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { .writeTimeout(20, TimeUnit.SECONDS) .build() + init { + loadTestCookies() + } + override suspend fun evaluateJs(script: String): String? { return QuackContext.create().use { it.evaluate(script)?.toString() @@ -40,12 +43,20 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { return SourceConfigMock() } - suspend fun doRequest(url: String, builder: Request.Builder.() -> Unit): Response { + suspend fun doRequest(url: String, referer: String? = null): Response { val request = Request.Builder() .get() .url(url) - .apply(builder) - .build() - return httpClient.newCall(request).await() + if (referer != null) { + request.header("Referrer", referer) + } + return httpClient.newCall(request.build()).await() + } + + private fun loadTestCookies() { + // https://addons.mozilla.org/ru/firefox/addon/cookies-txt/ + javaClass.getResourceAsStream("/cookies.txt")?.use { + cookieJar.loadFromStream(it) + } ?: println("No cookies loaded!") } } \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 4df2417c..7fa41b5e 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -1,6 +1,8 @@ package org.koitharu.kotatsu.parsers import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.params.ParameterizedTest import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaSource @@ -10,7 +12,9 @@ import org.koitharu.kotatsu.parsers.util.mimeType import org.koitharu.kotatsu.test_util.isDistinct import org.koitharu.kotatsu.test_util.isDistinctBy import org.koitharu.kotatsu.test_util.isUrlAbsoulte +import org.koitharu.kotatsu.test_util.maxDuplicates +@ExtendWith(AuthCheckExtension::class) internal class MangaParserTest { private val context = MangaLoaderContextMock() @@ -20,7 +24,7 @@ internal class MangaParserTest { fun list(source: MangaSource) = runTest { val parser = source.newParser(context) val list = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null) - checkMangaList(list) + checkMangaList(list, "list") assert(list.all { it.source == source }) } @@ -31,11 +35,13 @@ internal class MangaParserTest { val subject = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { it.title.length } ?: error("No manga found") - val list = parser.getList(offset = 0, query = subject.title, sortOrder = null, tags = null) + val query = subject.title + check(query.isNotBlank()) { "Manga title '$query' is blank" } + val list = parser.getList(offset = 0, query, sortOrder = null, tags = null) assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { - "Single subject ${subject.title} not found in search results" + "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" } - checkMangaList(list) + checkMangaList(list, "search('$query')") assert(list.all { it.source == source }) } @@ -53,8 +59,9 @@ internal class MangaParserTest { assert("" !in titles) assert(tags.all { it.source == source }) - val list = parser.getList(offset = 0, tags = setOf(tags.last()), query = null, sortOrder = null) - checkMangaList(list) + val tag = tags.last() + val list = parser.getList(offset = 0, tags = setOf(tag), query = null, sortOrder = null) + checkMangaList(list, "${tag.title} (${tag.key})") assert(list.all { it.source == source }) } @@ -65,16 +72,28 @@ internal class MangaParserTest { val list = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null) val manga = list[3] parser.getDetails(manga).apply { - assert(!chapters.isNullOrEmpty()) - assert(publicUrl.isUrlAbsoulte()) - assert(description != null) - assert(title.startsWith(manga.title)) + assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } + assert(publicUrl.isUrlAbsoulte()) { "Manga public url is not absolute: '$publicUrl'" } + assert(description != null) { "Detailed description is null" } + assert(title.startsWith(manga.title)) { + "Titles are mismatch: '$title' and '${manga.title}' for $publicUrl" + } assert(this.source == source) val c = checkNotNull(chapters) - assert(c.isDistinctBy { it.id }) - assert(c.isDistinctBy { it.number }) - assert(c.isDistinctBy { it.name }) + assert(c.isDistinctBy { it.id }) { + "Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl" + } + assert(c.isDistinctBy { it.number to it.branch }) { + "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" + } + assert(c.isDistinctBy { it.name to it.branch }) { + "Chapters are not distinct by name: ${c.maxDuplicates { it.name to it.branch }} for $publicUrl" + } assert(c.all { it.source == source }) + checkImageRequest(coverUrl, publicUrl) + largeCoverUrl?.let { + checkImageRequest(it, publicUrl) + } } } @@ -95,22 +114,41 @@ internal class MangaParserTest { val pageUrl = parser.getPageUrl(page) assert(pageUrl.isNotEmpty()) assert(pageUrl.isUrlAbsoulte()) - val pageResponse = context.doRequest(pageUrl) { - header("Referrer", page.referer) + checkImageRequest(pageUrl, page.referer) + } + + @ParameterizedTest + @MangaSources + @Disabled + fun authorization(source: MangaSource) = runTest { + val parser = source.newParser(context) + if (parser is MangaParserAuthProvider) { + val username = parser.getUsername() + assert(username.isNotBlank()) { "Username is blank" } + println("Signed in to ${source.name} as $username") } - assert(pageResponse.isSuccessful) - assert(pageResponse.mimeType?.startsWith("image/") == true) } - private fun checkMangaList(list: List) { - assert(list.isNotEmpty()) { "Manga list is empty" } - assert(list.isDistinctBy { it.id }) { "Manga list contains duplicated ids" } + private suspend fun checkMangaList(list: List, cause: String) { + assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" } + assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" } for (item in list) { assert(item.url.isNotEmpty()) assert(!item.url.isUrlAbsoulte()) assert(item.coverUrl.isUrlAbsoulte()) - assert(item.title.isNotEmpty()) + assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } assert(item.publicUrl.isUrlAbsoulte()) } + val testItem = list.random() + checkImageRequest(testItem.coverUrl, testItem.publicUrl) + } + + private suspend fun checkImageRequest(url: String, referer: String?) { + context.doRequest(url, referer).use { + assert(it.isSuccessful) { "Request failed: ${it.code}: ${it.message}" } + assert(it.mimeType?.startsWith("image/") == true) { + "Wrong response mime type: ${it.mimeType}" + } + } } } \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt b/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt index abaa8bcf..095357ba 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt @@ -26,4 +26,18 @@ internal fun Collection.isDistinctBy(selector: (T) -> K): Boolean { } internal fun String.isUrlRelative() = matches(PATTERN_URL_RELATIVE) -internal fun String.isUrlAbsoulte() = matches(PATTERN_URL_ABSOLUTE) \ No newline at end of file +internal fun String.isUrlAbsoulte() = matches(PATTERN_URL_ABSOLUTE) + +internal inline fun Collection.maxDuplicates(selector: (T) -> K): K? { + return groupBy(selector).maxByOrNull { it.value.size }?.key +} + +@Suppress("NOTHING_TO_INLINE") +inline operator fun List.component6(): T { + return get(5) +} + +@Suppress("NOTHING_TO_INLINE") +inline operator fun List.component7(): T { + return get(6) +} \ No newline at end of file