Support custom headers for page requests

app/src/main/java/org/koitharu/kotatsu/base/domain/MangaUtils.kt

@@ -5,7 +5,6 @@ import android.net.Uri
 import android.util.Size
 import androidx.annotation.WorkerThread
 import okhttp3.OkHttpClient
-import okhttp3.Request
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.get
 import org.koitharu.kotatsu.BuildConfig
@@ -26,8 +25,8 @@ object MangaUtils : KoinComponent {
 	suspend fun determineMangaIsWebtoon(pages: List<MangaPage>): Boolean? {
 		try {
 			val page = pages.medianOrNull() ?: return null
-			val url = page.source.repository.getPageFullUrl(page)
+			val pageRequest = page.source.repository.getPageRequest(page)
-			val uri = Uri.parse(url)
+			val uri = Uri.parse(pageRequest.url)
 			val size = if (uri.scheme == "cbz") {
 				val zip = ZipFile(uri.schemeSpecificPart)
 				val entry = zip.getEntry(uri.fragment)
@@ -36,9 +35,7 @@ object MangaUtils : KoinComponent {
 				}
 			} else {
 				val client = get<OkHttpClient>()
-				val request = Request.Builder()
+				val request = pageRequest.newBuilder()
-					.url(url)
-					.get()
 					.build()
 				client.newCall(request).await().use {
 					getBitmapSize(it.body?.byteStream())

app/src/main/java/org/koitharu/kotatsu/core/model/RequestDraft.kt

@@ -0,0 +1,18 @@
+package org.koitharu.kotatsu.core.model
+
+import okhttp3.Headers
+import okhttp3.Request
+
+data class RequestDraft(
+	val url: String,
+	val headers: Headers
+) {
+
+	val isValid: Boolean
+		get() = url.isNotEmpty()
+
+	fun newBuilder(): Request.Builder = Request.Builder()
+		.url(url)
+		.get()
+		.headers(headers)
+}

app/src/main/java/org/koitharu/kotatsu/core/network/CommonHeaders.kt

@@ -0,0 +1,9 @@
+package org.koitharu.kotatsu.core.network
+
+object CommonHeaders {
+
+	const val REFERER = "Referer"
+	const val USER_AGENT = "User-Agent"
+	const val ACCEPT = "Accept"
+	const val CONTENT_DISPOSITION = "Content-Disposition"
+}

app/src/main/java/org/koitharu/kotatsu/core/network/UserAgentInterceptor.kt

@@ -11,9 +11,9 @@ class UserAgentInterceptor : Interceptor {
 	override fun intercept(chain: Interceptor.Chain): Response {
 		val request = chain.request()
 		return chain.proceed(
-			if (request.header(HEADER_USER_AGENT) == null) {
+			if (request.header(CommonHeaders.USER_AGENT) == null) {
 				request.newBuilder()
-					.header(HEADER_USER_AGENT, userAgent)
+					.addHeader(CommonHeaders.USER_AGENT, userAgent)
 					.build()
 			} else request
 		)
@@ -21,8 +21,6 @@ class UserAgentInterceptor : Interceptor {
 
 	companion object {
 
-		private const val HEADER_USER_AGENT = "User-Agent"
-
 		val userAgent
 			get() = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
 				BuildConfig.VERSION_NAME,

app/src/main/java/org/koitharu/kotatsu/core/parser/MangaRepository.kt

@@ -17,7 +17,7 @@ interface MangaRepository {
 
 	suspend fun getPages(chapter: MangaChapter): List<MangaPage>
 
-	suspend fun getPageFullUrl(page: MangaPage): String
+	suspend fun getPageRequest(page: MangaPage): RequestDraft
 
 	suspend fun getTags(): Set<MangaTag>
 }

app/src/main/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepository.kt

@@ -1,10 +1,9 @@
 package org.koitharu.kotatsu.core.parser
 
+import okhttp3.Headers
 import org.koitharu.kotatsu.base.domain.MangaLoaderContext
-import org.koitharu.kotatsu.core.model.MangaPage
+import org.koitharu.kotatsu.core.model.*
-import org.koitharu.kotatsu.core.model.MangaSource
+import org.koitharu.kotatsu.core.network.CommonHeaders
-import org.koitharu.kotatsu.core.model.MangaTag
-import org.koitharu.kotatsu.core.model.SortOrder
 
 abstract class RemoteMangaRepository(
 	protected val loaderContext: MangaLoaderContext
@@ -18,7 +17,12 @@ abstract class RemoteMangaRepository(
 
 	override val sortOrders: Set<SortOrder> get() = emptySet()
 
-	override suspend fun getPageFullUrl(page: MangaPage): String = page.url
+	override suspend fun getPageRequest(page: MangaPage): RequestDraft {
+		return RequestDraft(
+			url = page.url,
+			headers = Headers.headersOf(CommonHeaders.REFERER, page.referer)
+		)
+	}
 
 	override suspend fun getTags(): Set<MangaTag> = emptySet()
 

app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaTownRepository.kt

@@ -1,10 +1,12 @@
 package org.koitharu.kotatsu.core.parser.site
 
 import androidx.collection.arraySetOf
+import okhttp3.Headers
 import org.intellij.lang.annotations.Language
 import org.koitharu.kotatsu.base.domain.MangaLoaderContext
 import org.koitharu.kotatsu.core.exceptions.ParseException
 import org.koitharu.kotatsu.core.model.*
+import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
 import org.koitharu.kotatsu.core.prefs.SourceSettings
 import org.koitharu.kotatsu.utils.ext.*
@@ -140,11 +142,12 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
 		}
 	}
 
-	override suspend fun getPageFullUrl(page: MangaPage): String {
+	override suspend fun getPageRequest(page: MangaPage): RequestDraft {
 		val domain = conf.getDomain(DOMAIN)
 		val ssl = conf.isUseSsl(false)
 		val doc = loaderContext.httpGet(page.url).parseHtml()
-		return doc.getElementById("image").attr("src").withDomain(domain, ssl)
+		val url = doc.getElementById("image").attr("src").withDomain(domain, ssl)
+		return RequestDraft(url, Headers.headersOf(CommonHeaders.REFERER, page.referer))
 	}
 
 	override suspend fun getTags(): Set<MangaTag> {

app/src/main/java/org/koitharu/kotatsu/core/parser/site/NudeMoonRepository.kt

@@ -1,9 +1,11 @@
 package org.koitharu.kotatsu.core.parser.site
 
 import androidx.collection.arraySetOf
+import okhttp3.Headers
 import org.koitharu.kotatsu.base.domain.MangaLoaderContext
 import org.koitharu.kotatsu.core.exceptions.ParseException
 import org.koitharu.kotatsu.core.model.*
+import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
 import org.koitharu.kotatsu.core.prefs.SourceSettings
 import org.koitharu.kotatsu.utils.ext.*
@@ -116,9 +118,10 @@ class NudeMoonRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposit
 		}
 	}
 
-	override suspend fun getPageFullUrl(page: MangaPage): String {
+	override suspend fun getPageRequest(page: MangaPage): RequestDraft {
 		val doc = loaderContext.httpGet(page.url).parseHtml()
-		return doc.body().getElementById("gallery").attr("src").inContextOf(doc)
+		val url = doc.body().getElementById("gallery").attr("src").inContextOf(doc)
+		return RequestDraft(url, Headers.headersOf(CommonHeaders.REFERER, page.referer))
 	}
 
 	override suspend fun getTags(): Set<MangaTag> {

app/src/main/java/org/koitharu/kotatsu/download/DownloadService.kt

@@ -12,8 +12,8 @@ import coil.ImageLoader
 import coil.request.ImageRequest
 import kotlinx.coroutines.*
 import kotlinx.coroutines.sync.Mutex
+import okhttp3.Headers
 import okhttp3.OkHttpClient
-import okhttp3.Request
 import okio.IOException
 import org.koin.android.ext.android.get
 import org.koin.android.ext.android.inject
@@ -23,6 +23,8 @@ import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.base.ui.BaseService
 import org.koitharu.kotatsu.base.ui.dialog.CheckBoxAlertDialog
 import org.koitharu.kotatsu.core.model.Manga
+import org.koitharu.kotatsu.core.model.RequestDraft
+import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.core.prefs.AppSettings
 import org.koitharu.kotatsu.local.data.MangaZip
 import org.koitharu.kotatsu.local.data.PagesCache
@@ -105,7 +107,12 @@ class DownloadService : BaseService() {
 				output = MangaZip.findInDir(destination, data)
 				output.prepare(data)
 				val coverUrl = data.largeCoverUrl ?: data.coverUrl
-				downloadPage(coverUrl, destination).let { file ->
+				downloadPage(
+					RequestDraft(
+						coverUrl,
+						Headers.headersOf(CommonHeaders.REFERER, data.url)
+					), destination
+				).let { file ->
 					output.addCover(file, MimeTypeMap.getFileExtensionFromUrl(coverUrl))
 				}
 				val chapters = if (chaptersIds == null) {
@@ -119,13 +126,14 @@ class DownloadService : BaseService() {
 						for ((pageIndex, page) in pages.withIndex()) {
 							failsafe@ do {
 								try {
-									val url = repo.getPageFullUrl(page)
+									val request = repo.getPageRequest(page)
-									val file = cache[url] ?: downloadPage(url, destination)
+									val file =
+										cache[request.url] ?: downloadPage(request, destination)
 									output.addPage(
 										chapter,
 										file,
 										pageIndex,
-										MimeTypeMap.getFileExtensionFromUrl(url)
+										MimeTypeMap.getFileExtensionFromUrl(request.url)
 									)
 								} catch (e: IOException) {
 									notification.setWaitingForNetwork()
@@ -187,9 +195,8 @@ class DownloadService : BaseService() {
 		}
 	}
 
-	private suspend fun downloadPage(url: String, destination: File): File {
+	private suspend fun downloadPage(requestDraft: RequestDraft, destination: File): File {
-		val request = Request.Builder()
+		val request = requestDraft.newBuilder()
-			.url(url)
 			.cacheControl(CacheUtils.CONTROL_DISABLED)
 			.get()
 			.build()

app/src/main/java/org/koitharu/kotatsu/local/domain/LocalMangaRepository.kt

@@ -7,6 +7,7 @@ import android.webkit.MimeTypeMap
 import androidx.collection.ArraySet
 import androidx.core.net.toFile
 import androidx.core.net.toUri
+import okhttp3.internal.EMPTY_HEADERS
 import org.koitharu.kotatsu.core.model.*
 import org.koitharu.kotatsu.core.parser.MangaRepository
 import org.koitharu.kotatsu.local.data.CbzFilter
@@ -156,7 +157,9 @@ class LocalMangaRepository(private val context: Context) : MangaRepository {
 
 	override val sortOrders = emptySet<SortOrder>()
 
-	override suspend fun getPageFullUrl(page: MangaPage) = page.url
+	override suspend fun getPageRequest(page: MangaPage): RequestDraft {
+		return RequestDraft(page.url, EMPTY_HEADERS)
+	}
 
 	override suspend fun getTags() = emptySet<MangaTag>()
 

app/src/main/java/org/koitharu/kotatsu/reader/domain/PageLoader.kt

@@ -8,7 +8,8 @@ import kotlinx.coroutines.*
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import okhttp3.OkHttpClient
-import okhttp3.Request
+import org.koitharu.kotatsu.core.model.RequestDraft
+import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.local.data.PagesCache
 import org.koitharu.kotatsu.utils.CacheUtils
 import org.koitharu.kotatsu.utils.ext.await
@@ -24,30 +25,28 @@ class PageLoader(
 	private val tasks = ArrayMap<String, Deferred<File>>()
 	private val convertLock = Mutex()
 
-	suspend fun loadFile(url: String, referer: String, force: Boolean): File {
+	suspend fun loadFile(requestDraft: RequestDraft, force: Boolean): File {
 		if (!force) {
-			cache[url]?.let {
+			cache[requestDraft.url]?.let {
 				return it
 			}
 		}
-		val task = tasks[url]?.takeUnless { it.isCancelled || (force && it.isCompleted) }
+		val task =
-		return (task ?: loadAsync(url, referer).also { tasks[url] = it }).await()
+			tasks[requestDraft.url]?.takeUnless { it.isCancelled || (force && it.isCompleted) }
+		return (task ?: loadAsync(requestDraft).also { tasks[requestDraft.url] = it }).await()
 	}
 
-	private fun loadAsync(url: String, referer: String) = async(Dispatchers.IO) {
+	private fun loadAsync(requestDraft: RequestDraft) = async(Dispatchers.IO) {
-		val uri = Uri.parse(url)
+		val uri = Uri.parse(requestDraft.url)
 		if (uri.scheme == "cbz") {
 			val zip = ZipFile(uri.schemeSpecificPart)
 			val entry = zip.getEntry(uri.fragment)
 			zip.getInputStream(entry).use {
-				cache.put(url, it)
+				cache.put(requestDraft.url, it)
 			}
 		} else {
-			val request = Request.Builder()
+			val request = requestDraft.newBuilder()
-				.url(url)
+				.header(CommonHeaders.ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8")
-				.get()
-				.header("Accept", "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8")
-				.header("Referer", referer)
 				.cacheControl(CacheUtils.CONTROL_DISABLED)
 				.build()
 			okHttp.newCall(request).await().use { response ->
@@ -58,7 +57,7 @@ class PageLoader(
 					"Null response"
 				}
 				body.byteStream().use {
-					cache.put(url, it)
+					cache.put(requestDraft.url, it)
 				}
 			}
 		}

app/src/main/java/org/koitharu/kotatsu/reader/ui/ReaderViewModel.kt

@@ -9,7 +9,6 @@ import androidx.lifecycle.viewModelScope
 import kotlinx.coroutines.*
 import kotlinx.coroutines.flow.*
 import okhttp3.OkHttpClient
-import okhttp3.Request
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.get
 import org.koitharu.kotatsu.base.domain.MangaDataRepository
@@ -154,15 +153,14 @@ class ReaderViewModel(
 					it.chapterId == state.chapterId && it.index == state.page
 				}?.toMangaPage() ?: error("Page not found")
 				val repo = page.source.repository
-				val url = repo.getPageFullUrl(page)
+				val pageRequest = repo.getPageRequest(page)
-				val request = Request.Builder()
+				val request = pageRequest.newBuilder()
-					.url(url)
 					.get()
 					.build()
 				val uri = get<OkHttpClient>().newCall(request).await().use { response ->
 					val fileName =
 						URLUtil.guessFileName(
-							url,
+							pageRequest.url,
 							response.contentDisposition,
 							response.mimeType
 						)

app/src/main/java/org/koitharu/kotatsu/reader/ui/pager/PageHolderDelegate.kt

@@ -89,9 +89,9 @@ class PageHolderDelegate(
 		callback.onLoadingStarted()
 		try {
 			val file = withContext(Dispatchers.IO) {
-				val pageUrl = data.source.repository.getPageFullUrl(data)
+				val pageRequest = data.source.repository.getPageRequest(data)
-				check(pageUrl.isNotEmpty()) { "Cannot obtain full image url" }
+				check(pageRequest.isValid) { "Cannot obtain full image url" }
-				loader.loadFile(pageUrl, data.referer, force)
+				loader.loadFile(pageRequest, force)
 			}
 			this@PageHolderDelegate.file = file
 			state = State.LOADED

app/src/main/java/org/koitharu/kotatsu/reader/ui/thumbnails/adapter/PageThumbnailAD.kt

@@ -6,9 +6,12 @@ import coil.request.ImageRequest
 import coil.size.PixelSize
 import com.hannesdorfmann.adapterdelegates4.dsl.adapterDelegateViewBinding
 import kotlinx.coroutines.*
+import okhttp3.Headers
 import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.base.ui.list.OnListItemClickListener
 import org.koitharu.kotatsu.core.model.MangaPage
+import org.koitharu.kotatsu.core.model.RequestDraft
+import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.databinding.ItemPageThumbBinding
 import org.koitharu.kotatsu.local.data.PagesCache
 import org.koitharu.kotatsu.reader.ui.thumbnails.PageThumbnail
@@ -42,13 +45,18 @@ fun pageThumbnailAD(
 			text = (item.number).toString()
 		}
 		job = scope.launch(Dispatchers.Default + IgnoreErrors) {
-			val url = item.page.preview ?: item.page.url.let {
+			val pageRequest = item.page.preview?.let {
-				val pageUrl = item.repository.getPageFullUrl(item.page)
+				RequestDraft(it, Headers.headersOf(CommonHeaders.REFERER, item.page.referer))
-				cache[pageUrl]?.toUri()?.toString() ?: pageUrl
+			} ?: item.page.url.let {
+				val pageRequest = item.repository.getPageRequest(item.page)
+				cache[pageRequest.url]?.toUri()?.toString()?.let {
+					pageRequest.copy(url = it)
+				} ?: pageRequest
 			}
 			val drawable = coil.execute(
 				ImageRequest.Builder(context)
-					.data(url)
+					.data(pageRequest.url)
+					.headers(pageRequest.headers)
 					.size(thumbSize)
 					.allowRgb565(true)
 					.build()

app/src/main/java/org/koitharu/kotatsu/utils/ext/CoilExt.kt

@@ -7,6 +7,7 @@ import coil.request.ErrorResult
 import coil.request.ImageRequest
 import coil.request.ImageResult
 import coil.request.SuccessResult
+import org.koitharu.kotatsu.core.network.CommonHeaders
 
 @Suppress("NOTHING_TO_INLINE")
 inline fun ImageView.newImageRequest(url: String) = ImageRequest.Builder(context)
@@ -31,4 +32,6 @@ fun ImageResult.toBitmapOrNull() = when (this) {
 }
 
 @Suppress("NOTHING_TO_INLINE")
-inline fun ImageRequest.Builder.referer(referer: String) = this.setHeader("Referer", referer)
+inline fun ImageRequest.Builder.referer(referer: String): ImageRequest.Builder {
+	return setHeader(CommonHeaders.REFERER, referer)
+}

app/src/main/java/org/koitharu/kotatsu/utils/ext/HttpExt.kt

@@ -1,9 +1,10 @@
 package org.koitharu.kotatsu.utils.ext
 
 import okhttp3.Response
+import org.koitharu.kotatsu.core.network.CommonHeaders
 
 val Response.mimeType: String?
 	get() = body?.contentType()?.run { "$type/$subtype" }
 
 val Response.contentDisposition: String?
-	get() = header("Content-Disposition")
+	get() = header(CommonHeaders.CONTENT_DISPOSITION)

app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt

@@ -82,7 +82,7 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
 		val pages = runBlocking { repo.getPages(details.chapters!!.random()) }
 		Assert.assertFalse(pages.isEmpty())
 		val page = pages.random()
-		val fullUrl = runBlocking { repo.getPageFullUrl(page) }
+		val fullUrl = runBlocking { repo.getPageRequest(page) }
 		AssertX.assertContentType(fullUrl, "image/*")
 	}