Support Referer header for image requests

pull/26/head
Koitharu 5 years ago
parent 22e7bab879
commit 1a0986212b

@ -9,14 +9,14 @@ def gitCommits = 'git rev-list --count HEAD'.execute([], rootDir).text.trim().to
android { android {
compileSdkVersion 30 compileSdkVersion 30
buildToolsVersion '30.0.2' buildToolsVersion '30.0.3'
defaultConfig { defaultConfig {
applicationId 'org.koitharu.kotatsu' applicationId 'org.koitharu.kotatsu'
minSdkVersion 21 minSdkVersion 21
targetSdkVersion 30 targetSdkVersion 30
versionCode gitCommits versionCode gitCommits
versionName '1.0-b1' versionName '1.0-b2'
kapt { kapt {
arguments { arguments {
@ -79,7 +79,7 @@ dependencies {
implementation 'androidx.recyclerview:recyclerview:1.2.0-beta01' implementation 'androidx.recyclerview:recyclerview:1.2.0-beta01'
implementation 'androidx.viewpager2:viewpager2:1.1.0-alpha01' implementation 'androidx.viewpager2:viewpager2:1.1.0-alpha01'
implementation 'androidx.preference:preference-ktx:1.1.1' implementation 'androidx.preference:preference-ktx:1.1.1'
implementation 'androidx.work:work-runtime-ktx:2.4.0' implementation 'androidx.work:work-runtime-ktx:2.5.0-beta02'
implementation 'com.google.android.material:material:1.3.0-beta01' implementation 'com.google.android.material:material:1.3.0-beta01'
//noinspection LifecycleAnnotationProcessorWithJava8 //noinspection LifecycleAnnotationProcessorWithJava8
kapt 'androidx.lifecycle:lifecycle-compiler:2.3.0-rc01' kapt 'androidx.lifecycle:lifecycle-compiler:2.3.0-rc01'

@ -7,6 +7,7 @@ import kotlinx.parcelize.Parcelize
data class MangaPage( data class MangaPage(
val id: Long, val id: Long,
val url: String, val url: String,
val referer: String,
val preview: String? = null, val preview: String? = null,
val source: MangaSource val source: MangaSource
) : Parcelable ) : Parcelable

@ -2,6 +2,7 @@ package org.koitharu.kotatsu.core.network
import okhttp3.Interceptor import okhttp3.Interceptor
import okhttp3.Response import okhttp3.Response
import okhttp3.internal.closeQuietly
import org.koitharu.kotatsu.core.exceptions.CloudFlareProtectedException import org.koitharu.kotatsu.core.exceptions.CloudFlareProtectedException
import java.net.HttpURLConnection.HTTP_FORBIDDEN import java.net.HttpURLConnection.HTTP_FORBIDDEN
import java.net.HttpURLConnection.HTTP_UNAVAILABLE import java.net.HttpURLConnection.HTTP_UNAVAILABLE
@ -12,6 +13,7 @@ class CloudFlareInterceptor : Interceptor {
val response = chain.proceed(chain.request()) val response = chain.proceed(chain.request())
if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) { if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) {
if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) {
response.closeQuietly()
throw CloudFlareProtectedException(chain.request().url.toString()) throw CloudFlareProtectedException(chain.request().url.toString())
} }
} }

@ -9,12 +9,14 @@ class UserAgentInterceptor : Interceptor {
override fun intercept(chain: Interceptor.Chain) = chain.proceed( override fun intercept(chain: Interceptor.Chain) = chain.proceed(
chain.request().newBuilder() chain.request().newBuilder()
.header("User-Agent", userAgent) .header(HEADER_USER_AGENT, userAgent)
.build() .build()
) )
companion object { companion object {
private const val HEADER_USER_AGENT = "User-Agent"
val userAgent val userAgent
get() = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( get() = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
BuildConfig.VERSION_NAME, BuildConfig.VERSION_NAME,

@ -112,6 +112,7 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
MangaPage( MangaPage(
id = url.longHashCode(), id = url.longHashCode(),
url = url, url = url,
referer = chapter.url,
source = source source = source
) )
} }

@ -103,6 +103,7 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
return json.getJSONObject("pages").getJSONArray("list").map { return json.getJSONObject("pages").getJSONArray("list").map {
MangaPage( MangaPage(
id = it.getLong("id"), id = it.getLong("id"),
referer = chapter.url,
source = chapter.source, source = chapter.source,
url = it.getString("img") url = it.getString("img")
) )

@ -154,6 +154,7 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
MangaPage( MangaPage(
id = url.longHashCode(), id = url.longHashCode(),
url = url, url = url,
referer = chapter.url,
source = source source = source
) )
} }

@ -168,8 +168,9 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
val pageUrl = "$domain$url${x.getString("u")}" val pageUrl = "$domain$url${x.getString("u")}"
MangaPage( MangaPage(
id = pageUrl.longHashCode(), id = pageUrl.longHashCode(),
source = source, url = pageUrl,
url = pageUrl referer = chapter.url,
source = source
) )
} }
} }

@ -134,6 +134,7 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
MangaPage( MangaPage(
id = href.longHashCode(), id = href.longHashCode(),
url = href, url = href,
referer = chapter.url,
source = MangaSource.MANGATOWN source = MangaSource.MANGATOWN
) )
} }

@ -146,6 +146,7 @@ class MangareadRepository(
MangaPage( MangaPage(
id = url.longHashCode(), id = url.longHashCode(),
url = url, url = url,
referer = chapter.url,
source = MangaSource.MANGAREAD source = MangaSource.MANGAREAD
) )
} }

@ -109,6 +109,7 @@ class NudeMoonRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposit
MangaPage( MangaPage(
id = url.longHashCode(), id = url.longHashCode(),
url = url, url = url,
referer = chapter.url,
preview = a.selectFirst("img")?.absUrl("src"), preview = a.selectFirst("img")?.absUrl("src"),
source = source source = source
) )

@ -51,6 +51,7 @@ class DetailsFragment : BaseFragment<FragmentDetailsBinding>(), View.OnClickList
private fun onMangaUpdated(manga: Manga) { private fun onMangaUpdated(manga: Manga) {
with(binding) { with(binding) {
imageViewCover.newImageRequest(manga.largeCoverUrl ?: manga.coverUrl) imageViewCover.newImageRequest(manga.largeCoverUrl ?: manga.coverUrl)
.referer(manga.url)
.fallback(R.drawable.ic_placeholder) .fallback(R.drawable.ic_placeholder)
.placeholderMemoryCacheKey(CoilUtils.metadata(imageViewCover)?.memoryCacheKey) .placeholderMemoryCacheKey(CoilUtils.metadata(imageViewCover)?.memoryCacheKey)
.lifecycle(viewLifecycleOwner) .lifecycle(viewLifecycleOwner)

@ -13,6 +13,7 @@ import org.koitharu.kotatsu.list.ui.model.ListModel
import org.koitharu.kotatsu.list.ui.model.MangaGridModel import org.koitharu.kotatsu.list.ui.model.MangaGridModel
import org.koitharu.kotatsu.utils.ext.enqueueWith import org.koitharu.kotatsu.utils.ext.enqueueWith
import org.koitharu.kotatsu.utils.ext.newImageRequest import org.koitharu.kotatsu.utils.ext.newImageRequest
import org.koitharu.kotatsu.utils.ext.referer
fun mangaGridItemAD( fun mangaGridItemAD(
coil: ImageLoader, coil: ImageLoader,
@ -35,6 +36,7 @@ fun mangaGridItemAD(
binding.textViewTitle.text = item.title binding.textViewTitle.text = item.title
imageRequest?.dispose() imageRequest?.dispose()
imageRequest = binding.imageViewCover.newImageRequest(item.coverUrl) imageRequest = binding.imageViewCover.newImageRequest(item.coverUrl)
.referer(item.manga.url)
.placeholder(R.drawable.ic_placeholder) .placeholder(R.drawable.ic_placeholder)
.fallback(R.drawable.ic_placeholder) .fallback(R.drawable.ic_placeholder)
.error(R.drawable.ic_placeholder) .error(R.drawable.ic_placeholder)

@ -13,6 +13,7 @@ import org.koitharu.kotatsu.list.ui.model.ListModel
import org.koitharu.kotatsu.list.ui.model.MangaListDetailedModel import org.koitharu.kotatsu.list.ui.model.MangaListDetailedModel
import org.koitharu.kotatsu.utils.ext.enqueueWith import org.koitharu.kotatsu.utils.ext.enqueueWith
import org.koitharu.kotatsu.utils.ext.newImageRequest import org.koitharu.kotatsu.utils.ext.newImageRequest
import org.koitharu.kotatsu.utils.ext.referer
import org.koitharu.kotatsu.utils.ext.textAndVisible import org.koitharu.kotatsu.utils.ext.textAndVisible
fun mangaListDetailedItemAD( fun mangaListDetailedItemAD(
@ -37,6 +38,7 @@ fun mangaListDetailedItemAD(
binding.textViewTitle.text = item.title binding.textViewTitle.text = item.title
binding.textViewSubtitle.textAndVisible = item.subtitle binding.textViewSubtitle.textAndVisible = item.subtitle
imageRequest = binding.imageViewCover.newImageRequest(item.coverUrl) imageRequest = binding.imageViewCover.newImageRequest(item.coverUrl)
.referer(item.manga.url)
.placeholder(R.drawable.ic_placeholder) .placeholder(R.drawable.ic_placeholder)
.fallback(R.drawable.ic_placeholder) .fallback(R.drawable.ic_placeholder)
.error(R.drawable.ic_placeholder) .error(R.drawable.ic_placeholder)

@ -13,6 +13,7 @@ import org.koitharu.kotatsu.list.ui.model.ListModel
import org.koitharu.kotatsu.list.ui.model.MangaListModel import org.koitharu.kotatsu.list.ui.model.MangaListModel
import org.koitharu.kotatsu.utils.ext.enqueueWith import org.koitharu.kotatsu.utils.ext.enqueueWith
import org.koitharu.kotatsu.utils.ext.newImageRequest import org.koitharu.kotatsu.utils.ext.newImageRequest
import org.koitharu.kotatsu.utils.ext.referer
import org.koitharu.kotatsu.utils.ext.textAndVisible import org.koitharu.kotatsu.utils.ext.textAndVisible
fun mangaListItemAD( fun mangaListItemAD(
@ -37,6 +38,7 @@ fun mangaListItemAD(
binding.textViewTitle.text = item.title binding.textViewTitle.text = item.title
binding.textViewSubtitle.textAndVisible = item.subtitle binding.textViewSubtitle.textAndVisible = item.subtitle
imageRequest = binding.imageViewCover.newImageRequest(item.coverUrl) imageRequest = binding.imageViewCover.newImageRequest(item.coverUrl)
.referer(item.manga.url)
.placeholder(R.drawable.ic_placeholder) .placeholder(R.drawable.ic_placeholder)
.fallback(R.drawable.ic_placeholder) .fallback(R.drawable.ic_placeholder)
.error(R.drawable.ic_placeholder) .error(R.drawable.ic_placeholder)

@ -7,6 +7,7 @@ import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.sub import org.koitharu.kotatsu.utils.ext.sub
import org.koitharu.kotatsu.utils.ext.takeIfReadable import org.koitharu.kotatsu.utils.ext.takeIfReadable
import java.io.File import java.io.File
import java.io.InputStream
import java.io.OutputStream import java.io.OutputStream
class PagesCache(context: Context) { class PagesCache(context: Context) {
@ -19,6 +20,7 @@ class PagesCache(context: Context) {
return lruCache.get(url)?.takeIfReadable() return lruCache.get(url)?.takeIfReadable()
} }
@Deprecated("Useless lambda")
fun put(url: String, writer: (OutputStream) -> Unit): File { fun put(url: String, writer: (OutputStream) -> Unit): File {
val file = cacheDir.sub(url.longHashCode().toString()) val file = cacheDir.sub(url.longHashCode().toString())
file.outputStream().use(writer) file.outputStream().use(writer)
@ -26,4 +28,14 @@ class PagesCache(context: Context) {
file.delete() file.delete()
return res return res
} }
fun put(url: String, inputStream: InputStream): File {
val file = cacheDir.sub(url.longHashCode().toString())
file.outputStream().use { out ->
inputStream.copyTo(out)
}
val res = lruCache.put(url, file)
file.delete()
return res
}
} }

@ -71,6 +71,7 @@ class LocalMangaRepository(private val context: Context) : MangaRepository {
MangaPage( MangaPage(
id = entryUri.longHashCode(), id = entryUri.longHashCode(),
url = entryUri, url = entryUri,
referer = chapter.url,
source = MangaSource.LOCAL source = MangaSource.LOCAL
) )
} }

@ -24,43 +24,41 @@ class PageLoader(
private val tasks = ArrayMap<String, Deferred<File>>() private val tasks = ArrayMap<String, Deferred<File>>()
private val convertLock = Mutex() private val convertLock = Mutex()
suspend fun loadFile(url: String, force: Boolean): File { suspend fun loadFile(url: String, referer: String, force: Boolean): File {
if (!force) { if (!force) {
cache[url]?.let { cache[url]?.let {
return it return it
} }
} }
val task = tasks[url]?.takeUnless { it.isCancelled || (force && it.isCompleted) } val task = tasks[url]?.takeUnless { it.isCancelled || (force && it.isCompleted) }
return (task ?: loadAsync(url).also { tasks[url] = it }).await() return (task ?: loadAsync(url, referer).also { tasks[url] = it }).await()
} }
private fun loadAsync(url: String) = async(Dispatchers.IO) { private fun loadAsync(url: String, referer: String) = async(Dispatchers.IO) {
val uri = Uri.parse(url) val uri = Uri.parse(url)
if (uri.scheme == "cbz") { if (uri.scheme == "cbz") {
val zip = ZipFile(uri.schemeSpecificPart) val zip = ZipFile(uri.schemeSpecificPart)
val entry = zip.getEntry(uri.fragment) val entry = zip.getEntry(uri.fragment)
zip.getInputStream(entry).use { zip.getInputStream(entry).use {
cache.put(url) { out -> cache.put(url, it)
it.copyTo(out)
}
} }
} else { } else {
val request = Request.Builder() val request = Request.Builder()
.url(url) .url(url)
.get() .get()
.header("Accept", "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8") .header("Accept", "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8")
.header("Referer", referer)
.cacheControl(CacheUtils.CONTROL_DISABLED) .cacheControl(CacheUtils.CONTROL_DISABLED)
.build() .build()
okHttp.newCall(request).await().use { response -> okHttp.newCall(request).await().use { response ->
val body = response.body
check(response.isSuccessful) { check(response.isSuccessful) {
"Invalid response: ${response.code} ${response.message}" "Invalid response: ${response.code} ${response.message}"
} }
checkNotNull(body) { val body = checkNotNull(response.body) {
"Null response" "Null response"
} }
cache.put(url) { out -> body.byteStream().use {
body.byteStream().use { it.copyTo(out) } cache.put(url, it)
} }
} }
} }

@ -91,7 +91,7 @@ class PageHolderDelegate(
val file = withContext(Dispatchers.IO) { val file = withContext(Dispatchers.IO) {
val pageUrl = data.source.repository.getPageFullUrl(data) val pageUrl = data.source.repository.getPageFullUrl(data)
check(pageUrl.isNotEmpty()) { "Cannot obtain full image url" } check(pageUrl.isNotEmpty()) { "Cannot obtain full image url" }
loader.loadFile(pageUrl, force) loader.loadFile(pageUrl, data.referer, force)
} }
this@PageHolderDelegate.file = file this@PageHolderDelegate.file = file
state = State.LOADED state = State.LOADED

@ -9,6 +9,7 @@ import org.koitharu.kotatsu.core.model.MangaSource
data class ReaderPage( data class ReaderPage(
val id: Long, val id: Long,
val url: String, val url: String,
val referer: String,
val preview: String?, val preview: String?,
val chapterId: Long, val chapterId: Long,
val index: Int, val index: Int,
@ -18,6 +19,7 @@ data class ReaderPage(
fun toMangaPage() = MangaPage( fun toMangaPage() = MangaPage(
id = id, id = id,
url = url, url = url,
referer = referer,
preview = preview, preview = preview,
source = source source = source
) )
@ -27,6 +29,7 @@ data class ReaderPage(
fun from(page: MangaPage, index: Int, chapterId: Long) = ReaderPage( fun from(page: MangaPage, index: Int, chapterId: Long) = ReaderPage(
id = page.id, id = page.id,
url = page.url, url = page.url,
referer = page.referer,
preview = page.preview, preview = page.preview,
chapterId = chapterId, chapterId = chapterId,
index = index, index = index,

@ -28,4 +28,7 @@ fun ImageResult.toBitmapOrNull() = when (this) {
null null
} }
is ErrorResult -> null is ErrorResult -> null
} }
@Suppress("NOTHING_TO_INLINE")
inline fun ImageRequest.Builder.referer(referer: String) = this.setHeader("Referer", referer)

@ -24,7 +24,7 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
private val repo = try { private val repo = try {
source.cls.getDeclaredConstructor(MangaLoaderContext::class.java) source.cls.getDeclaredConstructor(MangaLoaderContext::class.java)
.newInstance(get()) .newInstance(get<MangaLoaderContext>())
} catch (e: NoSuchMethodException) { } catch (e: NoSuchMethodException) {
source.cls.newInstance() source.cls.newInstance()
} }

Loading…
Cancel
Save