diff --git a/.idea/runConfigurations/TestsAndReport.xml b/.idea/runConfigurations/TestsAndReport.xml new file mode 100644 index 00000000..614cfdf3 --- /dev/null +++ b/.idea/runConfigurations/TestsAndReport.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 7ae0228c..c229f4f8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -1,9 +1,12 @@ package org.koitharu.kotatsu.parsers import androidx.annotation.CallSuper +import androidx.annotation.VisibleForTesting +import okhttp3.Headers import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.FaviconParser import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* @@ -30,6 +33,9 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour */ protected abstract val configKeyDomain: ConfigKey.Domain + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) + internal open val headers: Headers? = null + /** * Used as fallback if value of `sortOrder` passed to [getList] is null */ @@ -105,8 +111,19 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour /** * Returns direct link to the website favicon */ + @Deprecated( + message = "Use parseFavicons() to get multiple favicons with different size", + replaceWith = ReplaceWith("parseFavicons()"), + ) open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico" + /** + * Parse favicons from the main page of the source`s website + */ + open suspend fun getFavicons(): Favicons { + return FaviconParser(context, getDomain(), headers).parseFavicons() + } + @CallSuper open fun onCreateConfig(keys: MutableCollection>) { keys.add(configKeyDomain) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt new file mode 100644 index 00000000..7256524e --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt @@ -0,0 +1,51 @@ +package org.koitharu.kotatsu.parsers.model + +import okhttp3.HttpUrl.Companion.toHttpUrl + +class Favicon internal constructor( + val url: String, + val size: Int, + internal val rel: String?, +) : Comparable { + + val type: String = url.toHttpUrl().pathSegments.last() + .substringAfterLast('.', "").lowercase() + + override fun compareTo(other: Favicon): Int { + val res = size.compareTo(other.size) + if (res != 0) { + return res + } + return relWeightOf(rel).compareTo(relWeightOf(other.rel)) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Favicon + + if (url != other.url) return false + if (size != other.size) return false + if (rel != other.rel) return false + + return true + } + + override fun hashCode(): Int { + var result = url.hashCode() + result = 31 * result + size + result = 31 * result + rel.hashCode() + return result + } + + override fun toString(): String { + return "Favicon(size=$size, type='$type', rel='$rel', url='$url')" + } + + private fun relWeightOf(rel: String?) = when (rel) { + "apple-touch-icon" -> 1 // Prefer apple-touch-icon because it has a better quality + "mask-icon" -> -1 + else -> 0 + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt new file mode 100644 index 00000000..f9c9aab7 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt @@ -0,0 +1,45 @@ +package org.koitharu.kotatsu.parsers.model + +class Favicons internal constructor( + favicons: Collection, + val referer: String, +) : Collection { + + private val icons = favicons.sortedDescending() + + override val size: Int + get() = icons.size + + override fun contains(element: Favicon): Boolean = icons.contains(element) + + override fun containsAll(elements: Collection): Boolean = icons.containsAll(elements) + + override fun isEmpty(): Boolean = icons.isEmpty() + + override fun iterator(): Iterator = icons.iterator() + + /** + * Finds a favicon whose size in pixels is greater than or equal to the specified size. + * If such icon is not available returns the largest icon + * @param size in pixels + * @param types supported file types, e.g. png, svg, ico. May be null but not empty + */ + @JvmOverloads + fun find(size: Int, types: Set? = null): Favicon? { + if (icons.isEmpty()) { + return null + } + var result: Favicon? = null + for (icon in icons) { + if (types != null && icon.type !in types) { + continue + } + if (result == null || icon.size >= size) { + result = icon + } else { + break + } + } + return result + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index af5e2a55..26eee91b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -26,7 +26,7 @@ internal abstract class NineMangaParser( context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes") } - private val headers = Headers.Builder() + override val headers = Headers.Builder() .add("Accept-Language", "en-US;q=0.7,en;q=0.3") .build() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 073b68bf..f7425c1d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -24,7 +24,7 @@ internal abstract class GroupleParser( private val siteId: Int, ) : MangaParser(source), MangaParserAuthProvider { - private val headers = Headers.Builder() + override val headers = Headers.Builder() .add("User-Agent", userAgent) .build() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt new file mode 100644 index 00000000..fec7f706 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt @@ -0,0 +1,94 @@ +package org.koitharu.kotatsu.parsers.util + +import okhttp3.Headers +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.model.Favicon +import org.koitharu.kotatsu.parsers.model.Favicons +import org.koitharu.kotatsu.parsers.util.json.mapJSON + +class FaviconParser( + private val context: MangaLoaderContext, + private val domain: String, + private val headers: Headers?, +) { + + suspend fun parseFavicons(): Favicons { + val url = "https://$domain" + val doc = context.httpGet(url, headers).parseHtml() + val result = HashSet() + val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull() + ?.attrAsAbsoluteUrlOrNull("href") + if (manifestLink != null) { + result += parseManifest(manifestLink) + } + val links = doc.getElementsByAttributeValueContaining("rel", "icon") + links.mapNotNullTo(result) { link -> + parseLink(link) + } + if (result.isEmpty()) { + result.add(createFallback()) + } + return Favicons(result, url) + } + + private fun parseLink(link: Element): Favicon? { + val href = link.attrAsAbsoluteUrlOrNull("href") + if (href == null || href.endsWith('/')) { + return null + } + val sizes = link.attr("sizes") + return Favicon( + url = href, + size = parseSize(sizes), + rel = link.attrOrNull("rel"), + ) + } + + private fun parseSize(sizes: String): Int { + if (sizes.isEmpty() || sizes == "any") { + return 0 + } + return sizes.substringBefore(' ') + .split('x', 'X', '*') + .firstNotNullOfOrNull { it.toIntOrNull() } + ?: 0 + } + + private suspend fun parseManifest(url: String): List { + val json = context.httpGet(url, headers).parseJson() + val icons = json.getJSONArray("icons") + return icons.mapJSON { jo -> + Favicon( + url = jo.getString("src").resolveLink(), + size = parseSize(jo.getString("sizes")), + rel = null, + ) + } + } + + private fun createFallback(): Favicon { + val href = "https://$domain/favicon.ico" + return Favicon( + url = href, + size = 0, + rel = null, + ) + } + + private fun String.resolveLink(): String { + return when { + startsWith("http:") || startsWith("https:") -> { + this + } + + startsWith('/') -> { + "https://$domain$this" + } + + else -> { + "https://$domain/$this" + } + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt index 50a3e9d4..5f9a64c3 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt @@ -17,7 +17,7 @@ class CloudFlareInterceptor : Interceptor { if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) { if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { response.closeQuietly() - throw CloudFlareProtectedException(chain.request().url.toString()) + throw CloudFlareProtectedException(response.request.url.toString()) } } return response diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt index 60357aa4..95ebb59c 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt @@ -1,6 +1,7 @@ package org.koitharu.kotatsu.parsers import com.koushikdutta.quack.QuackContext +import okhttp3.Headers import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -45,10 +46,13 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { return SourceConfigMock() } - suspend fun doRequest(url: String, referer: String? = null): Response { + suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response { val request = Request.Builder() .get() .url(url) + if (extraHeaders != null) { + request.headers(extraHeaders) + } if (referer != null) { request.header("Referer", referer) } diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 66333666..bb7a1725 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -136,9 +136,16 @@ internal class MangaParserTest { @MangaSources fun favicon(source: MangaSource) = runTest { val parser = source.newParser(context) - val faviconUrl = parser.getFaviconUrl() - assert(faviconUrl.isUrlAbsolute()) - checkImageRequest(faviconUrl, null) + val favicons = parser.getFavicons() + val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") + assert(favicons.isNotEmpty()) + favicons.forEach { + assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" } + assert(it.type in types) { "Unknown icon type: ${it.type}" } + } + val favicon = favicons.find(24) + checkNotNull(favicon) + checkImageRequest(favicon.url, favicons.referer) } @ParameterizedTest(name = "{index}|domain|{0}") @@ -150,7 +157,7 @@ internal class MangaParserTest { .host(defaultDomain) .scheme("https") .toString() - val response = context.doRequest(url) + val response = context.doRequest(url, extraHeaders = parser.headers) val realUrl = response.request.url val realDomain = realUrl.topPrivateDomain() val realHost = realUrl.host