Merge branch 'feature/favicons'

.idea/runConfigurations/TestsAndReport.xml

@@ -0,0 +1,7 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="TestsAndReport" type="CompoundRunConfigurationType">
+    <toRun name="MangaParserTest" type="GradleRunConfiguration" />
+    <toRun name="kotatsu-parsers [generateTestsReport]" type="GradleRunConfiguration" />
+    <method v="2" />
+  </configuration>
+</component>

src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt

@@ -1,9 +1,12 @@
 package org.koitharu.kotatsu.parsers
 
 import androidx.annotation.CallSuper
+import androidx.annotation.VisibleForTesting
+import okhttp3.Headers
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.util.FaviconParser
 import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
 import java.util.*
 
@@ -30,6 +33,9 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
 	 */
 	protected abstract val configKeyDomain: ConfigKey.Domain
 
+	@VisibleForTesting(otherwise = VisibleForTesting.PROTECTED)
+	internal open val headers: Headers? = null
+
 	/**
 	 * Used as fallback if value of `sortOrder` passed to [getList] is null
 	 */
@@ -105,8 +111,19 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
 	/**
 	 * Returns direct link to the website favicon
 	 */
+	@Deprecated(
+		message = "Use parseFavicons() to get multiple favicons with different size",
+		replaceWith = ReplaceWith("parseFavicons()"),
+	)
 	open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
 
+	/**
+	 * Parse favicons from the main page of the source`s website
+	 */
+	open suspend fun getFavicons(): Favicons {
+		return FaviconParser(context, getDomain(), headers).parseFavicons()
+	}
+
 	@CallSuper
 	open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
 		keys.add(configKeyDomain)

src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt

@@ -0,0 +1,51 @@
+package org.koitharu.kotatsu.parsers.model
+
+import okhttp3.HttpUrl.Companion.toHttpUrl
+
+class Favicon internal constructor(
+	val url: String,
+	val size: Int,
+	internal val rel: String?,
+) : Comparable<Favicon> {
+
+	val type: String = url.toHttpUrl().pathSegments.last()
+		.substringAfterLast('.', "").lowercase()
+
+	override fun compareTo(other: Favicon): Int {
+		val res = size.compareTo(other.size)
+		if (res != 0) {
+			return res
+		}
+		return relWeightOf(rel).compareTo(relWeightOf(other.rel))
+	}
+
+	override fun equals(other: Any?): Boolean {
+		if (this === other) return true
+		if (javaClass != other?.javaClass) return false
+
+		other as Favicon
+
+		if (url != other.url) return false
+		if (size != other.size) return false
+		if (rel != other.rel) return false
+
+		return true
+	}
+
+	override fun hashCode(): Int {
+		var result = url.hashCode()
+		result = 31 * result + size
+		result = 31 * result + rel.hashCode()
+		return result
+	}
+
+	override fun toString(): String {
+		return "Favicon(size=$size, type='$type', rel='$rel', url='$url')"
+	}
+
+	private fun relWeightOf(rel: String?) = when (rel) {
+		"apple-touch-icon" -> 1 // Prefer apple-touch-icon because it has a better quality
+		"mask-icon" -> -1
+		else -> 0
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt

@@ -0,0 +1,45 @@
+package org.koitharu.kotatsu.parsers.model
+
+class Favicons internal constructor(
+	favicons: Collection<Favicon>,
+	val referer: String,
+) : Collection<Favicon> {
+
+	private val icons = favicons.sortedDescending()
+
+	override val size: Int
+		get() = icons.size
+
+	override fun contains(element: Favicon): Boolean = icons.contains(element)
+
+	override fun containsAll(elements: Collection<Favicon>): Boolean = icons.containsAll(elements)
+
+	override fun isEmpty(): Boolean = icons.isEmpty()
+
+	override fun iterator(): Iterator<Favicon> = icons.iterator()
+
+	/**
+	 * Finds a favicon whose size in pixels is greater than or equal to the specified size.
+	 * If such icon is not available returns the largest icon
+	 * @param size in pixels
+	 * @param types supported file types, e.g. png, svg, ico. May be null but not empty
+	 */
+	@JvmOverloads
+	fun find(size: Int, types: Set<String>? = null): Favicon? {
+		if (icons.isEmpty()) {
+			return null
+		}
+		var result: Favicon? = null
+		for (icon in icons) {
+			if (types != null && icon.type !in types) {
+				continue
+			}
+			if (result == null || icon.size >= size) {
+				result = icon
+			} else {
+				break
+			}
+		}
+		return result
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt

@@ -26,7 +26,7 @@ internal abstract class NineMangaParser(
 		context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes")
 	}
 
-	private val headers = Headers.Builder()
+	override val headers = Headers.Builder()
 		.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
 		.build()
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt

@@ -24,7 +24,7 @@ internal abstract class GroupleParser(
 	private val siteId: Int,
 ) : MangaParser(source), MangaParserAuthProvider {
 
-	private val headers = Headers.Builder()
+	override val headers = Headers.Builder()
 		.add("User-Agent", userAgent)
 		.build()
 

src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt

@@ -0,0 +1,94 @@
+package org.koitharu.kotatsu.parsers.util
+
+import okhttp3.Headers
+import org.jsoup.nodes.Element
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.model.Favicon
+import org.koitharu.kotatsu.parsers.model.Favicons
+import org.koitharu.kotatsu.parsers.util.json.mapJSON
+
+class FaviconParser(
+	private val context: MangaLoaderContext,
+	private val domain: String,
+	private val headers: Headers?,
+) {
+
+	suspend fun parseFavicons(): Favicons {
+		val url = "https://$domain"
+		val doc = context.httpGet(url, headers).parseHtml()
+		val result = HashSet<Favicon>()
+		val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull()
+			?.attrAsAbsoluteUrlOrNull("href")
+		if (manifestLink != null) {
+			result += parseManifest(manifestLink)
+		}
+		val links = doc.getElementsByAttributeValueContaining("rel", "icon")
+		links.mapNotNullTo(result) { link ->
+			parseLink(link)
+		}
+		if (result.isEmpty()) {
+			result.add(createFallback())
+		}
+		return Favicons(result, url)
+	}
+
+	private fun parseLink(link: Element): Favicon? {
+		val href = link.attrAsAbsoluteUrlOrNull("href")
+		if (href == null || href.endsWith('/')) {
+			return null
+		}
+		val sizes = link.attr("sizes")
+		return Favicon(
+			url = href,
+			size = parseSize(sizes),
+			rel = link.attrOrNull("rel"),
+		)
+	}
+
+	private fun parseSize(sizes: String): Int {
+		if (sizes.isEmpty() || sizes == "any") {
+			return 0
+		}
+		return sizes.substringBefore(' ')
+			.split('x', 'X', '*')
+			.firstNotNullOfOrNull { it.toIntOrNull() }
+			?: 0
+	}
+
+	private suspend fun parseManifest(url: String): List<Favicon> {
+		val json = context.httpGet(url, headers).parseJson()
+		val icons = json.getJSONArray("icons")
+		return icons.mapJSON { jo ->
+			Favicon(
+				url = jo.getString("src").resolveLink(),
+				size = parseSize(jo.getString("sizes")),
+				rel = null,
+			)
+		}
+	}
+
+	private fun createFallback(): Favicon {
+		val href = "https://$domain/favicon.ico"
+		return Favicon(
+			url = href,
+			size = 0,
+			rel = null,
+		)
+	}
+
+	private fun String.resolveLink(): String {
+		return when {
+			startsWith("http:") || startsWith("https:") -> {
+				this
+			}
+
+			startsWith('/') -> {
+				"https://$domain$this"
+			}
+
+			else -> {
+				"https://$domain/$this"
+			}
+		}
+	}
+}

src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt

@@ -17,7 +17,7 @@ class CloudFlareInterceptor : Interceptor {
 		if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) {
 			if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) {
 				response.closeQuietly()
-				throw CloudFlareProtectedException(chain.request().url.toString())
+				throw CloudFlareProtectedException(response.request.url.toString())
 			}
 		}
 		return response

src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt

@@ -1,6 +1,7 @@
 package org.koitharu.kotatsu.parsers
 
 import com.koushikdutta.quack.QuackContext
+import okhttp3.Headers
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
@@ -45,10 +46,13 @@ internal class MangaLoaderContextMock : MangaLoaderContext() {
 		return SourceConfigMock()
 	}
 
-	suspend fun doRequest(url: String, referer: String? = null): Response {
+	suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response {
 		val request = Request.Builder()
 			.get()
 			.url(url)
+		if (extraHeaders != null) {
+			request.headers(extraHeaders)
+		}
 		if (referer != null) {
 			request.header("Referer", referer)
 		}

src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt

@@ -136,9 +136,16 @@ internal class MangaParserTest {
 	@MangaSources
 	fun favicon(source: MangaSource) = runTest {
 		val parser = source.newParser(context)
-		val faviconUrl = parser.getFaviconUrl()
+		val favicons = parser.getFavicons()
-		assert(faviconUrl.isUrlAbsolute())
+		val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg")
-		checkImageRequest(faviconUrl, null)
+		assert(favicons.isNotEmpty())
+		favicons.forEach {
+			assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" }
+			assert(it.type in types) { "Unknown icon type: ${it.type}" }
+		}
+		val favicon = favicons.find(24)
+		checkNotNull(favicon)
+		checkImageRequest(favicon.url, favicons.referer)
 	}
 
 	@ParameterizedTest(name = "{index}|domain|{0}")
@@ -150,7 +157,7 @@ internal class MangaParserTest {
 			.host(defaultDomain)
 			.scheme("https")
 			.toString()
-		val response = context.doRequest(url)
+		val response = context.doRequest(url, extraHeaders = parser.headers)
 		val realUrl = response.request.url
 		val realDomain = realUrl.topPrivateDomain()
 		val realHost = realUrl.host