Support for storing local manga in directories with multiple chapters cbz

pull/311/head
Koitharu 3 years ago
parent b1a94c0f34
commit 4744a0a162
No known key found for this signature in database
GPG Key ID: 8E861F8CE6E7CE27

@ -15,8 +15,8 @@ android {
applicationId 'org.koitharu.kotatsu' applicationId 'org.koitharu.kotatsu'
minSdkVersion 21 minSdkVersion 21
targetSdkVersion 33 targetSdkVersion 33
versionCode 521 versionCode 530
versionName '4.4.5' versionName '5.0-a1'
generatedDensities = [] generatedDensities = []
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"

@ -128,7 +128,7 @@ class MangaDataRepository @Inject constructor(
.url(url) .url(url)
.get() .get()
.tag(MangaSource::class.java, page.source) .tag(MangaSource::class.java, page.source)
.cacheControl(CommonHeaders.CACHE_CONTROL_DISABLED) .cacheControl(CommonHeaders.CACHE_CONTROL_NO_STORE)
.build() .build()
okHttpClient.newCall(request).await().use { okHttpClient.newCall(request).await().use {
runInterruptible(Dispatchers.IO) { runInterruptible(Dispatchers.IO) {

@ -14,6 +14,6 @@ object CommonHeaders {
const val ACCEPT_ENCODING = "Accept-Encoding" const val ACCEPT_ENCODING = "Accept-Encoding"
const val AUTHORIZATION = "Authorization" const val AUTHORIZATION = "Authorization"
val CACHE_CONTROL_DISABLED: CacheControl val CACHE_CONTROL_NO_STORE: CacheControl
get() = CacheControl.Builder().noStore().build() get() = CacheControl.Builder().noStore().build()
} }

@ -187,7 +187,7 @@ class DetailsViewModel @Inject constructor(
return return
} }
launchLoadingJob(Dispatchers.Default) { launchLoadingJob(Dispatchers.Default) {
val manga = if (m.source == MangaSource.LOCAL) m else localMangaRepository.findSavedManga(m) val manga = if (m.source == MangaSource.LOCAL) m else localMangaRepository.findSavedManga(m)?.manga
checkNotNull(manga) { "Cannot find saved manga for ${m.title}" } checkNotNull(manga) { "Cannot find saved manga for ${m.title}" }
val original = localMangaRepository.getRemoteManga(manga) val original = localMangaRepository.getRemoteManga(manga)
localMangaRepository.delete(manga) || throw IOException("Unable to delete file") localMangaRepository.delete(manga) || throw IOException("Unable to delete file")

@ -53,7 +53,7 @@ class MangaDetailsDelegate @Inject constructor(
val m = localMangaRepository.getRemoteManga(manga) ?: return@runCatchingCancellable null val m = localMangaRepository.getRemoteManga(manga) ?: return@runCatchingCancellable null
mangaRepositoryFactory.create(m.source).getDetails(m) mangaRepositoryFactory.create(m.source).getDetails(m)
} else { } else {
localMangaRepository.findSavedManga(manga) localMangaRepository.findSavedManga(manga)?.manga
} }
}.onFailure { error -> }.onFailure { error ->
error.printStackTraceDebug() error.printStackTraceDebug()

@ -8,7 +8,6 @@ import androidx.lifecycle.lifecycleScope
import coil.ImageLoader import coil.ImageLoader
import coil.request.ImageRequest import coil.request.ImageRequest
import coil.size.Scale import coil.size.Scale
import dagger.hilt.android.lifecycle.RetainedLifecycle
import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.android.qualifiers.ApplicationContext
import dagger.hilt.android.scopes.ServiceScoped import dagger.hilt.android.scopes.ServiceScoped
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
@ -31,12 +30,12 @@ import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.core.prefs.AppSettings import org.koitharu.kotatsu.core.prefs.AppSettings
import org.koitharu.kotatsu.download.ui.service.PausingHandle import org.koitharu.kotatsu.download.ui.service.PausingHandle
import org.koitharu.kotatsu.local.data.PagesCache import org.koitharu.kotatsu.local.data.PagesCache
import org.koitharu.kotatsu.local.domain.CbzMangaOutput import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.local.domain.LocalMangaRepository import org.koitharu.kotatsu.local.domain.LocalMangaRepository
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.utils.RetainedLifecycleCoroutineScope
import org.koitharu.kotatsu.utils.ext.copyToSuspending import org.koitharu.kotatsu.utils.ext.copyToSuspending
import org.koitharu.kotatsu.utils.ext.deleteAwait import org.koitharu.kotatsu.utils.ext.deleteAwait
import org.koitharu.kotatsu.utils.ext.printStackTraceDebug import org.koitharu.kotatsu.utils.ext.printStackTraceDebug
@ -111,7 +110,7 @@ class DownloadManager @Inject constructor(
val destination = localMangaRepository.getOutputDir() val destination = localMangaRepository.getOutputDir()
checkNotNull(destination) { context.getString(R.string.cannot_find_available_storage) } checkNotNull(destination) { context.getString(R.string.cannot_find_available_storage) }
val tempFileName = "${manga.id}_$startId.tmp" val tempFileName = "${manga.id}_$startId.tmp"
var output: CbzMangaOutput? = null var output: LocalMangaOutput? = null
try { try {
if (manga.source == MangaSource.LOCAL) { if (manga.source == MangaSource.LOCAL) {
manga = localMangaRepository.getRemoteManga(manga) manga = localMangaRepository.getRemoteManga(manga)
@ -120,9 +119,9 @@ class DownloadManager @Inject constructor(
val repo = mangaRepositoryFactory.create(manga.source) val repo = mangaRepositoryFactory.create(manga.source)
outState.value = DownloadState.Preparing(startId, manga, cover) outState.value = DownloadState.Preparing(startId, manga, cover)
val data = if (manga.chapters.isNullOrEmpty()) repo.getDetails(manga) else manga val data = if (manga.chapters.isNullOrEmpty()) repo.getDetails(manga) else manga
output = CbzMangaOutput.get(destination, data) output = LocalMangaOutput.getOrCreate(destination, data)
val coverUrl = data.largeCoverUrl ?: data.coverUrl val coverUrl = data.largeCoverUrl ?: data.coverUrl
downloadFile(coverUrl, data.publicUrl, destination, tempFileName, repo.source).let { file -> downloadFile(coverUrl, destination, tempFileName, repo.source).let { file ->
output.addCover(file, MimeTypeMap.getFileExtensionFromUrl(coverUrl)) output.addCover(file, MimeTypeMap.getFileExtensionFromUrl(coverUrl))
} }
val chapters = checkNotNull( val chapters = checkNotNull(
@ -144,7 +143,7 @@ class DownloadManager @Inject constructor(
runFailsafe(outState, pausingHandle) { runFailsafe(outState, pausingHandle) {
val url = repo.getPageUrl(page) val url = repo.getPageUrl(page)
val file = cache.get(url) val file = cache.get(url)
?: downloadFile(url, page.referer, destination, tempFileName, repo.source) ?: downloadFile(url, destination, tempFileName, repo.source)
output.addPage( output.addPage(
chapter = chapter, chapter = chapter,
file = file, file = file,
@ -166,11 +165,12 @@ class DownloadManager @Inject constructor(
delay(SLOWDOWN_DELAY) delay(SLOWDOWN_DELAY)
} }
} }
output.flushChapter(chapter)
} }
outState.value = DownloadState.PostProcessing(startId, data, cover) outState.value = DownloadState.PostProcessing(startId, data, cover)
output.mergeWithExisting() output.mergeWithExisting()
output.finish() output.finish()
val localManga = localMangaRepository.getFromFile(output.file) val localManga = LocalMangaInput.of(output.rootFile).getManga().manga
outState.value = DownloadState.Done(startId, data, cover, localManga) outState.value = DownloadState.Done(startId, data, cover, localManga)
} catch (e: CancellationException) { } catch (e: CancellationException) {
outState.value = DownloadState.Cancelled(startId, manga, cover) outState.value = DownloadState.Cancelled(startId, manga, cover)
@ -216,16 +216,14 @@ class DownloadManager @Inject constructor(
private suspend fun downloadFile( private suspend fun downloadFile(
url: String, url: String,
referer: String,
destination: File, destination: File,
tempFileName: String, tempFileName: String,
source: MangaSource, source: MangaSource,
): File { ): File {
val request = Request.Builder() val request = Request.Builder()
.url(url) .url(url)
.header(CommonHeaders.REFERER, referer)
.tag(MangaSource::class.java, source) .tag(MangaSource::class.java, source)
.cacheControl(CommonHeaders.CACHE_CONTROL_DISABLED) .cacheControl(CommonHeaders.CACHE_CONTROL_NO_STORE)
.get() .get()
.build() .build()
val call = okHttp.newCall(request) val call = okHttp.newCall(request)

@ -12,6 +12,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.runInterruptible
import okio.buffer import okio.buffer
import okio.source import okio.source
import org.koitharu.kotatsu.local.data.util.withExtraCloseable
import java.util.zip.ZipFile import java.util.zip.ZipFile
class CbzFetcher( class CbzFetcher(

@ -1,15 +1,20 @@
package org.koitharu.kotatsu.local.data package org.koitharu.kotatsu.local.data
import java.io.File import java.io.File
import java.io.FileFilter
import java.io.FilenameFilter import java.io.FilenameFilter
import java.util.* import java.util.Locale
class CbzFilter : FilenameFilter { class CbzFilter : FileFilter, FilenameFilter {
override fun accept(dir: File, name: String): Boolean { override fun accept(dir: File, name: String): Boolean {
return isFileSupported(name) return isFileSupported(name)
} }
override fun accept(pathname: File?): Boolean {
return isFileSupported(pathname?.name ?: return false)
}
companion object { companion object {
fun isFileSupported(name: String): Boolean { fun isFileSupported(name: String): Boolean {

@ -1,27 +0,0 @@
package org.koitharu.kotatsu.local.data
import android.os.FileObserver
import java.io.File
import kotlinx.coroutines.channels.ProducerScope
import kotlinx.coroutines.channels.awaitClose
import kotlinx.coroutines.channels.trySendBlocking
import kotlinx.coroutines.flow.callbackFlow
@Suppress("DEPRECATION")
class FlowFileObserver(
private val producerScope: ProducerScope<File>,
private val file: File,
) : FileObserver(file.absolutePath, CREATE or DELETE or CLOSE_WRITE) {
override fun onEvent(event: Int, path: String?) {
producerScope.trySendBlocking(
if (path == null) file else file.resolve(path),
)
}
}
fun File.observe() = callbackFlow {
val observer = FlowFileObserver(this, this@observe)
observer.startWatching()
awaitClose { observer.stopWatching() }
}

@ -0,0 +1,29 @@
package org.koitharu.kotatsu.local.data
import java.io.File
import java.io.FileFilter
import java.io.FilenameFilter
import java.util.Locale
import java.util.zip.ZipEntry
class ImageFileFilter : FilenameFilter, FileFilter {
override fun accept(dir: File, name: String): Boolean {
val ext = name.substringAfterLast('.', "").lowercase(Locale.ROOT)
return isExtensionValid(ext)
}
override fun accept(pathname: File?): Boolean {
val ext = pathname?.extension?.lowercase(Locale.ROOT) ?: return false
return isExtensionValid(ext)
}
fun accept(entry: ZipEntry): Boolean {
val ext = entry.name.substringAfterLast('.', "").lowercase(Locale.ROOT)
return isExtensionValid(ext)
}
private fun isExtensionValid(ext: String): Boolean {
return ext == "png" || ext == "jpg" || ext == "jpeg" || ext == "webp"
}
}

@ -1,15 +1,16 @@
package org.koitharu.kotatsu.local.domain package org.koitharu.kotatsu.local.data
import java.io.File
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.model.MangaTag
import java.io.File
class LocalManga( class LocalManga(
val manga: Manga,
val file: File, val file: File,
val manga: Manga,
) { ) {
var createdAt: Long = -1L var createdAt: Long = -1L
private set
get() { get() {
if (field == -1L) { if (field == -1L) {
field = file.lastModified() field = file.lastModified()
@ -17,6 +18,15 @@ class LocalManga(
return field return field
} }
fun isMatchesQuery(query: String): Boolean {
return manga.title.contains(query, ignoreCase = true) ||
manga.altTitle?.contains(query, ignoreCase = true) == true
}
fun containsTags(tags: Set<MangaTag>): Boolean {
return manga.tags.containsAll(tags)
}
override fun equals(other: Any?): Boolean { override fun equals(other: Any?): Boolean {
if (this === other) return true if (this === other) return true
if (javaClass != other?.javaClass) return false if (javaClass != other?.javaClass) return false
@ -34,15 +44,8 @@ class LocalManga(
result = 31 * result + file.hashCode() result = 31 * result + file.hashCode()
return result return result
} }
}
fun Collection<LocalManga>.unwrap(): List<Manga> = map { it.manga }
fun LocalManga.isMatchesQuery(query: String): Boolean { override fun toString(): String {
return manga.title.contains(query, ignoreCase = true) || return "LocalManga(${file.path}: ${manga.title})"
manga.altTitle?.contains(query, ignoreCase = true) == true }
}
fun LocalManga.containsTags(tags: Set<MangaTag>): Boolean {
return manga.tags.containsAll(tags)
} }

@ -5,9 +5,6 @@ import android.content.Context
import android.os.StatFs import android.os.StatFs
import androidx.annotation.WorkerThread import androidx.annotation.WorkerThread
import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.android.qualifiers.ApplicationContext
import java.io.File
import javax.inject.Inject
import javax.inject.Singleton
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.asFlow
@ -17,9 +14,13 @@ import kotlinx.coroutines.runInterruptible
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import okhttp3.Cache import okhttp3.Cache
import org.koitharu.kotatsu.core.prefs.AppSettings import org.koitharu.kotatsu.core.prefs.AppSettings
import org.koitharu.kotatsu.local.data.util.observe
import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.utils.ext.computeSize import org.koitharu.kotatsu.utils.ext.computeSize
import org.koitharu.kotatsu.utils.ext.getStorageName import org.koitharu.kotatsu.utils.ext.getStorageName
import java.io.File
import javax.inject.Inject
import javax.inject.Singleton
private const val DIR_NAME = "manga" private const val DIR_NAME = "manga"
private const val CACHE_DISK_PERCENTAGE = 0.02 private const val CACHE_DISK_PERCENTAGE = 0.02

@ -1,15 +1,21 @@
package org.koitharu.kotatsu.local.data package org.koitharu.kotatsu.local.data
import androidx.annotation.WorkerThread
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.koitharu.kotatsu.BuildConfig import org.koitharu.kotatsu.BuildConfig
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.util.json.getBooleanOrDefault import org.koitharu.kotatsu.parsers.util.json.getBooleanOrDefault
import org.koitharu.kotatsu.parsers.util.json.getLongOrDefault import org.koitharu.kotatsu.parsers.util.json.getLongOrDefault
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import org.koitharu.kotatsu.parsers.util.toTitleCase import org.koitharu.kotatsu.parsers.util.toTitleCase
import org.koitharu.kotatsu.utils.AlphanumComparator import org.koitharu.kotatsu.utils.AlphanumComparator
import java.io.File
class MangaIndex(source: String?) { class MangaIndex(source: String?) {
@ -151,4 +157,18 @@ class MangaIndex(source: String?) {
} else { } else {
json.toString() json.toString()
} }
companion object {
@WorkerThread
fun read(file: File): MangaIndex? {
if (file.exists() && file.canRead()) {
val text = file.readText()
if (text.length > 2) {
return MangaIndex(text)
}
}
return null
}
}
} }

@ -0,0 +1,146 @@
package org.koitharu.kotatsu.local.data.input
import androidx.core.net.toFile
import androidx.core.net.toUri
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible
import org.koitharu.kotatsu.local.data.CbzFilter
import org.koitharu.kotatsu.local.data.ImageFileFilter
import org.koitharu.kotatsu.local.data.LocalManga
import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.toCamelCase
import org.koitharu.kotatsu.utils.AlphanumComparator
import org.koitharu.kotatsu.utils.ext.listFilesRecursive
import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.toListSorted
import java.io.File
import java.util.zip.ZipFile
/**
* Manga {Folder}
* |--- index.json (optional)
* |--- Chapter 1.cbz
* |--- Page 1.png
* :
* L--- Page x.png
* |--- Chapter 2.cbz
* :
* L--- Chapter x.cbz
*/
class LocalMangaDirInput(root: File) : LocalMangaInput(root) {
override suspend fun getManga(): LocalManga = runInterruptible(Dispatchers.IO) {
val index = MangaIndex.read(File(root, LocalMangaOutput.ENTRY_NAME_INDEX))
val mangaUri = root.toUri().toString()
val chapterFiles = getChaptersFiles()
val info = index?.getMangaInfo()
val manga = info?.copy2(
source = MangaSource.LOCAL,
url = mangaUri,
coverUrl = fileUri(
root,
index.getCoverEntry() ?: findFirstImageEntry().orEmpty(),
),
chapters = info.chapters?.mapIndexed { i, c ->
c.copy(url = chapterFiles[i].toUri().toString(), source = MangaSource.LOCAL)
},
) ?: Manga(
id = root.absolutePath.longHashCode(),
title = root.name.toHumanReadable(),
url = mangaUri,
publicUrl = mangaUri,
source = MangaSource.LOCAL,
coverUrl = findFirstImageEntry().orEmpty(),
chapters = chapterFiles.mapIndexed { i, f ->
MangaChapter(
id = "$i${f.name}".longHashCode(),
name = f.nameWithoutExtension.toHumanReadable(),
number = i + 1,
source = MangaSource.LOCAL,
uploadDate = f.lastModified(),
url = f.toUri().toString(),
scanlator = null,
branch = null,
)
},
altTitle = null,
rating = -1f,
isNsfw = false,
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
)
LocalManga(root, manga)
}
override suspend fun getMangaInfo(): Manga? = runInterruptible(Dispatchers.IO) {
val index = MangaIndex.read(File(root, LocalMangaOutput.ENTRY_NAME_INDEX))
index?.getMangaInfo()
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = runInterruptible(Dispatchers.IO) {
val file = chapter.url.toUri().toFile()
if (file.isDirectory) {
file.listFilesRecursive(ImageFileFilter())
.toListSorted(compareBy(AlphanumComparator()) { x -> x.name })
.map {
val pageUri = it.toUri().toString()
MangaPage(
id = pageUri.longHashCode(),
url = pageUri,
preview = null,
referer = chapter.url,
source = MangaSource.LOCAL,
)
}
} else {
ZipFile(file).use { zip ->
zip.entries()
.asSequence()
.filter { x -> !x.isDirectory }
.map { it.name }
.toListSorted(AlphanumComparator())
.map {
val pageUri = zipUri(file, it)
MangaPage(
id = pageUri.longHashCode(),
url = pageUri,
preview = null,
referer = chapter.url,
source = MangaSource.LOCAL,
)
}
}
}
}
private fun String.toHumanReadable() = replace("_", " ").toCamelCase()
private fun getChaptersFiles(): List<File> = root.listFilesRecursive(CbzFilter())
.toListSorted(compareBy(AlphanumComparator()) { x -> x.name })
private fun findFirstImageEntry(): String? {
val filter = ImageFileFilter()
root.listFilesRecursive(filter).firstOrNull()?.let {
return it.toUri().toString()
}
val cbz = root.listFilesRecursive(CbzFilter()).firstOrNull() ?: return null
return ZipFile(cbz).use { zip ->
val filter = ImageFileFilter()
zip.entries().asSequence()
.firstOrNull { x -> !x.isDirectory && filter.accept(x) }
?.let { entry -> zipUri(cbz, entry.name) }
}
}
private fun fileUri(base: File, name: String): String {
return File(base, name).toUri().toString()
}
}

@ -0,0 +1,75 @@
package org.koitharu.kotatsu.local.data.input
import android.net.Uri
import androidx.core.net.toFile
import org.koitharu.kotatsu.local.data.LocalManga
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import java.io.File
abstract class LocalMangaInput(
protected val root: File,
) {
abstract suspend fun getManga(): LocalManga
abstract suspend fun getMangaInfo(): Manga?
abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
companion object {
fun of(manga: Manga): LocalMangaInput = of(Uri.parse(manga.url).toFile())
fun of(chapter: MangaChapter): LocalMangaInput = of(Uri.parse(chapter.url).toFile())
fun of(file: File): LocalMangaInput = when {
file.isDirectory -> LocalMangaDirInput(file)
else -> LocalMangaZipInput(file)
}
@JvmStatic
protected fun zipUri(file: File, entryName: String) = "cbz://${file.path}#$entryName"
@JvmStatic
protected fun Manga.copy2(
url: String = this.url,
coverUrl: String = this.coverUrl,
chapters: List<MangaChapter>? = this.chapters,
source: MangaSource = this.source,
) = Manga(
id = id,
title = title,
altTitle = altTitle,
url = url,
publicUrl = publicUrl,
rating = rating,
isNsfw = isNsfw,
coverUrl = coverUrl,
tags = tags,
state = state,
author = author,
largeCoverUrl = largeCoverUrl,
description = description,
chapters = chapters,
source = source,
)
@JvmStatic
protected fun MangaChapter.copy(
url: String = this.url,
source: MangaSource = this.source,
) = MangaChapter(
id = id,
name = name,
number = number,
url = url,
scanlator = scanlator,
uploadDate = uploadDate,
branch = branch,
source = source,
)
}
}

@ -0,0 +1,152 @@
package org.koitharu.kotatsu.local.data.input
import android.net.Uri
import android.webkit.MimeTypeMap
import androidx.collection.ArraySet
import androidx.core.net.toFile
import androidx.core.net.toUri
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible
import org.koitharu.kotatsu.local.data.LocalManga
import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.toCamelCase
import org.koitharu.kotatsu.utils.AlphanumComparator
import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.readText
import org.koitharu.kotatsu.utils.ext.toListSorted
import java.io.File
import java.util.Enumeration
import java.util.zip.ZipEntry
import java.util.zip.ZipFile
/**
* Manga archive {.cbz or .zip file}
* |--- index.json (optional)
* |--- Page 1.png
* |--- Page 2.png
* :
* L--- Page x.png
*/
class LocalMangaZipInput(root: File) : LocalMangaInput(root) {
override suspend fun getManga(): LocalManga {
val manga = runInterruptible(Dispatchers.IO) {
ZipFile(root).use { zip ->
val fileUri = root.toUri().toString()
val entry = zip.getEntry(LocalMangaOutput.ENTRY_NAME_INDEX)
val index = entry?.let(zip::readText)?.let(::MangaIndex)
val info = index?.getMangaInfo()
if (info != null) {
return@use info.copy2(
source = MangaSource.LOCAL,
url = fileUri,
coverUrl = zipUri(
root,
entryName = index.getCoverEntry()
?: findFirstImageEntry(zip.entries())?.name.orEmpty(),
),
chapters = info.chapters?.map { c ->
c.copy(url = fileUri, source = MangaSource.LOCAL)
},
)
}
// fallback
val title = root.nameWithoutExtension.replace("_", " ").toCamelCase()
val chapters = ArraySet<String>()
for (x in zip.entries()) {
if (!x.isDirectory) {
chapters += x.name.substringBeforeLast(File.separatorChar, "")
}
}
val uriBuilder = root.toUri().buildUpon()
Manga(
id = root.absolutePath.longHashCode(),
title = title,
url = fileUri,
publicUrl = fileUri,
source = MangaSource.LOCAL,
coverUrl = zipUri(root, findFirstImageEntry(zip.entries())?.name.orEmpty()),
chapters = chapters.sortedWith(AlphanumComparator()).mapIndexed { i, s ->
MangaChapter(
id = "$i$s".longHashCode(),
name = s.ifEmpty { title },
number = i + 1,
source = MangaSource.LOCAL,
uploadDate = 0L,
url = uriBuilder.fragment(s).build().toString(),
scanlator = null,
branch = null,
)
},
altTitle = null,
rating = -1f,
isNsfw = false,
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
)
}
}
return LocalManga(root, manga)
}
override suspend fun getMangaInfo(): Manga? = runInterruptible(Dispatchers.IO) {
ZipFile(root).use { zip ->
val entry = zip.getEntry(LocalMangaOutput.ENTRY_NAME_INDEX)
val index = entry?.let(zip::readText)?.let(::MangaIndex)
index?.getMangaInfo()
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
return runInterruptible(Dispatchers.IO) {
val uri = Uri.parse(chapter.url)
val file = uri.toFile()
val zip = ZipFile(file)
val index = zip.getEntry(LocalMangaOutput.ENTRY_NAME_INDEX)?.let(zip::readText)?.let(::MangaIndex)
var entries = zip.entries().asSequence()
entries = if (index != null) {
val pattern = index.getChapterNamesPattern(chapter)
entries.filter { x -> !x.isDirectory && x.name.substringBefore('.').matches(pattern) }
} else {
val parent = uri.fragment.orEmpty()
entries.filter { x ->
!x.isDirectory && x.name.substringBeforeLast(
File.separatorChar,
"",
) == parent
}
}
entries
.toListSorted(compareBy(AlphanumComparator()) { x -> x.name })
.map { x ->
val entryUri = zipUri(file, x.name)
MangaPage(
id = entryUri.longHashCode(),
url = entryUri,
preview = null,
referer = chapter.url,
source = MangaSource.LOCAL,
)
}
}
}
private fun findFirstImageEntry(entries: Enumeration<out ZipEntry>): ZipEntry? {
val list = entries.toList()
.filterNot { it.isDirectory }
.sortedWith(compareBy(AlphanumComparator()) { x -> x.name })
val map = MimeTypeMap.getSingleton()
return list.firstOrNull {
map.getMimeTypeFromExtension(it.name.substringAfterLast('.'))
?.startsWith("image/") == true
}
}
}

@ -0,0 +1,114 @@
package org.koitharu.kotatsu.local.data.output
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible
import org.koitharu.kotatsu.core.zip.ZipOutput
import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.util.toFileNameSafe
import org.koitharu.kotatsu.utils.ext.deleteAwait
import org.koitharu.kotatsu.utils.ext.takeIfReadable
import java.io.File
class LocalMangaDirOutput(
rootFile: File,
manga: Manga,
) : LocalMangaOutput(rootFile) {
private val chaptersOutput = HashMap<MangaChapter, ZipOutput>()
private val index = MangaIndex(File(rootFile, ENTRY_NAME_INDEX).takeIfReadable()?.readText())
init {
index.setMangaInfo(manga, append = true)
}
override suspend fun mergeWithExisting() = Unit
override suspend fun addCover(file: File, ext: String) {
val name = buildString {
append("cover")
if (ext.isNotEmpty() && ext.length <= 4) {
append('.')
append(ext)
}
}
runInterruptible(Dispatchers.IO) {
file.copyTo(File(rootFile, name))
}
index.setCoverEntry(name)
}
override suspend fun addPage(chapter: MangaChapter, file: File, pageNumber: Int, ext: String) {
val output = chaptersOutput.getOrPut(chapter) {
ZipOutput(File(rootFile, chapterFileName(chapter) + SUFFIX_TMP))
}
val name = buildString {
append(FILENAME_PATTERN.format(chapter.branch.hashCode(), chapter.number, pageNumber))
if (ext.isNotEmpty() && ext.length <= 4) {
append('.')
append(ext)
}
}
runInterruptible(Dispatchers.IO) {
output.put(name, file)
}
index.addChapter(chapter)
}
override suspend fun flushChapter(chapter: MangaChapter) {
val output = chaptersOutput.remove(chapter) ?: return
output.flushAndFinish()
}
override suspend fun finish() {
runInterruptible(Dispatchers.IO) {
File(rootFile, ENTRY_NAME_INDEX).writeText(index.toString())
}
for (output in chaptersOutput.values) {
output.flushAndFinish()
}
chaptersOutput.clear()
}
override suspend fun cleanup() {
for (output in chaptersOutput.values) {
output.file.deleteAwait()
}
}
override fun close() {
for (output in chaptersOutput.values) {
output.close()
}
}
override fun sortChaptersByName() {
index.sortChaptersByName()
}
suspend fun deleteChapter(chapterId: Long) {
val chapter = checkNotNull(index.getMangaInfo()?.chapters) {
"No chapters found"
}.first { it.id == chapterId }
val chapterDir = File(rootFile, chapterFileName(chapter))
chapterDir.deleteAwait()
index.removeChapter(chapterId)
}
private suspend fun ZipOutput.flushAndFinish() = runInterruptible(Dispatchers.IO) {
finish()
close()
val resFile = File(file.absolutePath.removeSuffix(SUFFIX_TMP))
file.renameTo(resFile)
}
private fun chapterFileName(chapter: MangaChapter): String {
return "${chapter.number}_${chapter.name.toFileNameSafe()}".take(18) + ".cbz"
}
companion object {
private const val FILENAME_PATTERN = "%08d_%03d%03d"
}
}

@ -0,0 +1,58 @@
package org.koitharu.kotatsu.local.data.output
import okio.Closeable
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.util.toFileNameSafe
import java.io.File
abstract class LocalMangaOutput(
val rootFile: File,
) : Closeable {
abstract suspend fun mergeWithExisting()
abstract suspend fun addCover(file: File, ext: String)
abstract suspend fun addPage(chapter: MangaChapter, file: File, pageNumber: Int, ext: String)
abstract suspend fun flushChapter(chapter: MangaChapter)
abstract suspend fun finish()
abstract suspend fun cleanup()
abstract fun sortChaptersByName()
companion object {
const val ENTRY_NAME_INDEX = "index.json"
const val SUFFIX_TMP = ".tmp"
fun getOrCreate(root: File, manga: Manga): LocalMangaOutput {
return checkNotNull(getImpl(root, manga, onlyIfExists = false))
}
fun get(root: File, manga: Manga): LocalMangaOutput? {
return getImpl(root, manga, onlyIfExists = true)
}
private fun getImpl(root: File, manga: Manga, onlyIfExists: Boolean): LocalMangaOutput? {
val name = manga.title.toFileNameSafe()
val file = File(root, name)
return if (file.exists()) {
if (file.isDirectory) {
LocalMangaDirOutput(file, manga)
} else {
LocalMangaZipOutput(file, manga)
}
} else {
if (onlyIfExists) {
null
} else {
LocalMangaDirOutput(file, manga)
}
}
}
}
}

@ -1,40 +1,38 @@
package org.koitharu.kotatsu.local.domain package org.koitharu.kotatsu.local.data.output
import androidx.annotation.WorkerThread import androidx.annotation.WorkerThread
import java.io.File
import java.util.zip.ZipFile
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.runInterruptible
import okio.Closeable
import org.koitharu.kotatsu.core.zip.ZipOutput import org.koitharu.kotatsu.core.zip.ZipOutput
import org.koitharu.kotatsu.local.data.MangaIndex import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.util.toFileNameSafe
import org.koitharu.kotatsu.utils.ext.deleteAwait import org.koitharu.kotatsu.utils.ext.deleteAwait
import org.koitharu.kotatsu.utils.ext.readText import org.koitharu.kotatsu.utils.ext.readText
import java.io.File
import java.util.zip.ZipFile
class CbzMangaOutput( class LocalMangaZipOutput(
val file: File, rootFile: File,
manga: Manga, manga: Manga,
) : Closeable { ) : LocalMangaOutput(rootFile) {
private val output = ZipOutput(File(file.path + ".tmp")) private val output = ZipOutput(File(rootFile.path + ".tmp"))
private val index = MangaIndex(null) private val index = MangaIndex(null)
init { init {
index.setMangaInfo(manga, false) index.setMangaInfo(manga, false)
} }
suspend fun mergeWithExisting() { override suspend fun mergeWithExisting() {
if (file.exists()) { if (rootFile.exists()) {
runInterruptible(Dispatchers.IO) { runInterruptible(Dispatchers.IO) {
mergeWith(file) mergeWith(rootFile)
} }
} }
} }
suspend fun addCover(file: File, ext: String) { override suspend fun addCover(file: File, ext: String) {
val name = buildString { val name = buildString {
append(FILENAME_PATTERN.format(0, 0, 0)) append(FILENAME_PATTERN.format(0, 0, 0))
if (ext.isNotEmpty() && ext.length <= 4) { if (ext.isNotEmpty() && ext.length <= 4) {
@ -48,7 +46,7 @@ class CbzMangaOutput(
index.setCoverEntry(name) index.setCoverEntry(name)
} }
suspend fun addPage(chapter: MangaChapter, file: File, pageNumber: Int, ext: String) { override suspend fun addPage(chapter: MangaChapter, file: File, pageNumber: Int, ext: String) {
val name = buildString { val name = buildString {
append(FILENAME_PATTERN.format(chapter.branch.hashCode(), chapter.number, pageNumber)) append(FILENAME_PATTERN.format(chapter.branch.hashCode(), chapter.number, pageNumber))
if (ext.isNotEmpty() && ext.length <= 4) { if (ext.isNotEmpty() && ext.length <= 4) {
@ -62,17 +60,19 @@ class CbzMangaOutput(
index.addChapter(chapter) index.addChapter(chapter)
} }
suspend fun finish() { override suspend fun flushChapter(chapter: MangaChapter) = Unit
override suspend fun finish() {
runInterruptible(Dispatchers.IO) { runInterruptible(Dispatchers.IO) {
output.put(ENTRY_NAME_INDEX, index.toString()) output.put(ENTRY_NAME_INDEX, index.toString())
output.finish() output.finish()
output.close() output.close()
} }
file.deleteAwait() rootFile.deleteAwait()
output.file.renameTo(file) output.file.renameTo(rootFile)
} }
suspend fun cleanup() { override suspend fun cleanup() {
output.file.deleteAwait() output.file.deleteAwait()
} }
@ -80,7 +80,7 @@ class CbzMangaOutput(
output.close() output.close()
} }
fun sortChaptersByName() { override fun sortChaptersByName() {
index.sortChaptersByName() index.sortChaptersByName()
} }
@ -111,17 +111,9 @@ class CbzMangaOutput(
private const val FILENAME_PATTERN = "%08d_%03d%03d" private const val FILENAME_PATTERN = "%08d_%03d%03d"
const val ENTRY_NAME_INDEX = "index.json"
fun get(root: File, manga: Manga): CbzMangaOutput {
val name = manga.title.toFileNameSafe() + ".cbz"
val file = File(root, name)
return CbzMangaOutput(file, manga)
}
@WorkerThread @WorkerThread
fun filterChapters(subject: CbzMangaOutput, idsToRemove: Set<Long>) { fun filterChapters(subject: LocalMangaZipOutput, idsToRemove: Set<Long>) {
ZipFile(subject.file).use { zip -> ZipFile(subject.rootFile).use { zip ->
val index = MangaIndex(zip.readText(zip.getEntry(ENTRY_NAME_INDEX))) val index = MangaIndex(zip.readText(zip.getEntry(ENTRY_NAME_INDEX)))
idsToRemove.forEach { id -> index.removeChapter(id) } idsToRemove.forEach { id -> index.removeChapter(id) }
val patterns = requireNotNull(index.getMangaInfo()?.chapters).map { val patterns = requireNotNull(index.getMangaInfo()?.chapters).map {
@ -133,12 +125,15 @@ class CbzMangaOutput(
entry.name == ENTRY_NAME_INDEX -> { entry.name == ENTRY_NAME_INDEX -> {
subject.output.put(ENTRY_NAME_INDEX, index.toString()) subject.output.put(ENTRY_NAME_INDEX, index.toString())
} }
entry.isDirectory -> { entry.isDirectory -> {
subject.output.addDirectory(entry.name) subject.output.addDirectory(entry.name)
} }
entry.name == coverEntryName -> { entry.name == coverEntryName -> {
subject.output.copyEntryFrom(zip, entry) subject.output.copyEntryFrom(zip, entry)
} }
else -> { else -> {
val name = entry.name.substringBefore('.') val name = entry.name.substringBefore('.')
if (patterns.any { it.matches(name) }) { if (patterns.any { it.matches(name) }) {
@ -149,8 +144,8 @@ class CbzMangaOutput(
} }
subject.output.finish() subject.output.finish()
subject.output.close() subject.output.close()
subject.file.delete() subject.rootFile.delete()
subject.output.file.renameTo(subject.file) subject.output.file.renameTo(subject.rootFile)
} }
} }
} }

@ -1,4 +1,4 @@
package org.koitharu.kotatsu.local.data package org.koitharu.kotatsu.local.data.util
import okhttp3.internal.closeQuietly import okhttp3.internal.closeQuietly
import okio.Closeable import okio.Closeable

@ -0,0 +1,48 @@
package org.koitharu.kotatsu.local.data.util
import android.os.Build
import android.os.FileObserver
import androidx.annotation.RequiresApi
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.channels.ProducerScope
import kotlinx.coroutines.channels.awaitClose
import kotlinx.coroutines.channels.trySendBlocking
import kotlinx.coroutines.flow.callbackFlow
import kotlinx.coroutines.flow.flowOn
import java.io.File
fun File.observe() = callbackFlow {
val observer = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
FlowFileObserverQ(this, this@observe)
} else {
FlowFileObserver(this, this@observe)
}
observer.startWatching()
awaitClose { observer.stopWatching() }
}.flowOn(Dispatchers.IO)
@RequiresApi(Build.VERSION_CODES.Q)
private class FlowFileObserverQ(
private val producerScope: ProducerScope<File>,
private val file: File,
) : FileObserver(file, CREATE or DELETE or CLOSE_WRITE) {
override fun onEvent(event: Int, path: String?) {
producerScope.trySendBlocking(
if (path == null) file else file.resolve(path),
)
}
}
@Suppress("DEPRECATION")
private class FlowFileObserver(
private val producerScope: ProducerScope<File>,
private val file: File,
) : FileObserver(file.absolutePath, CREATE or DELETE or CLOSE_WRITE) {
override fun onEvent(event: Int, path: String?) {
producerScope.trySendBlocking(
if (path == null) file else file.resolve(path),
)
}
}

@ -1,14 +1,7 @@
package org.koitharu.kotatsu.local.domain package org.koitharu.kotatsu.local.domain
import android.annotation.SuppressLint
import android.net.Uri import android.net.Uri
import android.webkit.MimeTypeMap
import androidx.annotation.WorkerThread
import androidx.collection.ArraySet
import androidx.core.net.toFile import androidx.core.net.toFile
import androidx.core.net.toUri
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll import kotlinx.coroutines.awaitAll
@ -17,30 +10,29 @@ import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.filterNot import kotlinx.coroutines.flow.filterNot
import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.runInterruptible
import org.koitharu.kotatsu.core.parser.MangaRepository import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.local.data.CbzFilter import org.koitharu.kotatsu.local.data.LocalManga
import org.koitharu.kotatsu.local.data.LocalStorageManager import org.koitharu.kotatsu.local.data.LocalStorageManager
import org.koitharu.kotatsu.local.data.MangaIndex import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.local.data.TempFileFilter import org.koitharu.kotatsu.local.data.TempFileFilter
import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.data.output.LocalMangaDirOutput
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.local.data.output.LocalMangaZipOutput
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.util.toCamelCase
import org.koitharu.kotatsu.utils.AlphanumComparator import org.koitharu.kotatsu.utils.AlphanumComparator
import org.koitharu.kotatsu.utils.CompositeMutex import org.koitharu.kotatsu.utils.CompositeMutex
import org.koitharu.kotatsu.utils.ext.deleteAwait import org.koitharu.kotatsu.utils.ext.deleteAwait
import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.readText import org.koitharu.kotatsu.utils.ext.readText
import org.koitharu.kotatsu.utils.ext.runCatchingCancellable import org.koitharu.kotatsu.utils.ext.runCatchingCancellable
import java.io.File import java.io.File
import java.util.Enumeration
import java.util.zip.ZipEntry
import java.util.zip.ZipFile import java.util.zip.ZipFile
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
import kotlin.coroutines.CoroutineContext
private const val MAX_PARALLELISM = 4 private const val MAX_PARALLELISM = 4
@ -48,7 +40,6 @@ private const val MAX_PARALLELISM = 4
class LocalMangaRepository @Inject constructor(private val storageManager: LocalStorageManager) : MangaRepository { class LocalMangaRepository @Inject constructor(private val storageManager: LocalStorageManager) : MangaRepository {
override val source = MangaSource.LOCAL override val source = MangaSource.LOCAL
private val filenameFilter = CbzFilter()
private val locks = CompositeMutex<Long>() private val locks = CompositeMutex<Long>()
override suspend fun getList(offset: Int, query: String): List<Manga> { override suspend fun getList(offset: Int, query: String): List<Manga> {
@ -82,47 +73,16 @@ class LocalMangaRepository @Inject constructor(private val storageManager: Local
return list.unwrap() return list.unwrap()
} }
override suspend fun getDetails(manga: Manga) = when { override suspend fun getDetails(manga: Manga): Manga = when {
manga.source != MangaSource.LOCAL -> requireNotNull(findSavedManga(manga)) { manga.source != MangaSource.LOCAL -> requireNotNull(findSavedManga(manga)?.manga) {
"Manga is not local or saved" "Manga is not local or saved"
} }
else -> getFromFile(Uri.parse(manga.url).toFile()) else -> LocalMangaInput.of(manga).getManga().manga
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
return runInterruptible(Dispatchers.IO) { return LocalMangaInput.of(chapter).getPages(chapter)
val uri = Uri.parse(chapter.url)
val file = uri.toFile()
val zip = ZipFile(file)
val index = zip.getEntry(CbzMangaOutput.ENTRY_NAME_INDEX)?.let(zip::readText)?.let(::MangaIndex)
var entries = zip.entries().asSequence()
entries = if (index != null) {
val pattern = index.getChapterNamesPattern(chapter)
entries.filter { x -> !x.isDirectory && x.name.substringBefore('.').matches(pattern) }
} else {
val parent = uri.fragment.orEmpty()
entries.filter { x ->
!x.isDirectory && x.name.substringBeforeLast(
File.separatorChar,
"",
) == parent
}
}
entries
.toList()
.sortedWith(compareBy(AlphanumComparator()) { x -> x.name })
.map { x ->
val entryUri = zipUri(file, x.name)
MangaPage(
id = entryUri.longHashCode(),
url = entryUri,
preview = null,
referer = chapter.url,
source = MangaSource.LOCAL,
)
}
}
} }
suspend fun delete(manga: Manga): Boolean { suspend fun delete(manga: Manga): Boolean {
@ -133,109 +93,49 @@ class LocalMangaRepository @Inject constructor(private val storageManager: Local
suspend fun deleteChapters(manga: Manga, ids: Set<Long>) { suspend fun deleteChapters(manga: Manga, ids: Set<Long>) {
lockManga(manga.id) lockManga(manga.id)
try { try {
runInterruptible(Dispatchers.IO) { val uri = Uri.parse(manga.url)
val uri = Uri.parse(manga.url) val file = uri.toFile()
val file = uri.toFile() if (file.isDirectory) {
val cbz = CbzMangaOutput(file, manga) LocalMangaDirOutput(file, manga).use { output ->
CbzMangaOutput.filterChapters(cbz, ids) for (id in ids) {
output.deleteChapter(id)
}
output.finish()
}
} else {
runInterruptible(Dispatchers.IO) {
val cbz = LocalMangaZipOutput(file, manga)
LocalMangaZipOutput.filterChapters(cbz, ids)
}
} }
} finally { } finally {
unlockManga(manga.id) unlockManga(manga.id)
} }
} }
@WorkerThread
@SuppressLint("DefaultLocale")
fun getFromFile(file: File): Manga = ZipFile(file).use { zip ->
val fileUri = file.toUri().toString()
val entry = zip.getEntry(CbzMangaOutput.ENTRY_NAME_INDEX)
val index = entry?.let(zip::readText)?.let(::MangaIndex)
val info = index?.getMangaInfo()
if (index != null && info != null) {
return info.copy2(
source = MangaSource.LOCAL,
url = fileUri,
coverUrl = zipUri(
file,
entryName = index.getCoverEntry() ?: findFirstImageEntry(zip.entries())?.name.orEmpty(),
),
chapters = info.chapters?.map { c ->
c.copy(url = fileUri, source = MangaSource.LOCAL)
},
)
}
// fallback
val title = file.nameWithoutExtension.replace("_", " ").toCamelCase()
val chapters = ArraySet<String>()
for (x in zip.entries()) {
if (!x.isDirectory) {
chapters += x.name.substringBeforeLast(File.separatorChar, "")
}
}
val uriBuilder = file.toUri().buildUpon()
Manga(
id = file.absolutePath.longHashCode(),
title = title,
url = fileUri,
publicUrl = fileUri,
source = MangaSource.LOCAL,
coverUrl = zipUri(file, findFirstImageEntry(zip.entries())?.name.orEmpty()),
chapters = chapters.sortedWith(AlphanumComparator()).mapIndexed { i, s ->
MangaChapter(
id = "$i$s".longHashCode(),
name = s.ifEmpty { title },
number = i + 1,
source = MangaSource.LOCAL,
uploadDate = 0L,
url = uriBuilder.fragment(s).build().toString(),
scanlator = null,
branch = null,
)
},
altTitle = null,
rating = -1f,
isNsfw = false,
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
)
}
suspend fun getRemoteManga(localManga: Manga): Manga? { suspend fun getRemoteManga(localManga: Manga): Manga? {
val file = runCatching { val file = runCatching {
Uri.parse(localManga.url).toFile() Uri.parse(localManga.url).toFile()
}.getOrNull() ?: return null }.getOrNull() ?: return null
return runInterruptible(Dispatchers.IO) { return runInterruptible(Dispatchers.IO) {
ZipFile(file).use { zip -> ZipFile(file).use { zip ->
val entry = zip.getEntry(CbzMangaOutput.ENTRY_NAME_INDEX) val entry = zip.getEntry(LocalMangaOutput.ENTRY_NAME_INDEX)
val index = entry?.let(zip::readText)?.let(::MangaIndex) val index = entry?.let(zip::readText)?.let(::MangaIndex)
index?.getMangaInfo() index?.getMangaInfo()
} }
} }
} }
suspend fun findSavedManga(remoteManga: Manga): Manga? { suspend fun findSavedManga(remoteManga: Manga): LocalManga? {
val files = getAllFiles() val files = getAllFiles()
return runInterruptible(Dispatchers.IO) { val input = files.firstNotNullOfOrNull { file ->
for (file in files) { LocalMangaInput.of(file).takeIf {
val index = ZipFile(file).use { zip -> runCatchingCancellable {
val entry = zip.getEntry(CbzMangaOutput.ENTRY_NAME_INDEX) it.getMangaInfo()
entry?.let(zip::readText)?.let(::MangaIndex) }.getOrNull()?.id == remoteManga.id
} ?: continue
val info = index.getMangaInfo() ?: continue
if (info.id == remoteManga.id) {
val fileUri = file.toUri().toString()
return@runInterruptible info.copy2(
source = MangaSource.LOCAL,
url = fileUri,
chapters = info.chapters?.map { c -> c.copy(url = fileUri) },
)
}
} }
null
} }
return input?.getManga()
} }
suspend fun watchReadableDirs(): Flow<File> { suspend fun watchReadableDirs(): Flow<File> {
@ -245,28 +145,6 @@ class LocalMangaRepository @Inject constructor(private val storageManager: Local
.filterNot { filter.accept(it, it.name) } .filterNot { filter.accept(it, it.name) }
} }
private fun CoroutineScope.getFromFileAsync(
file: File,
context: CoroutineContext,
): Deferred<LocalManga?> = async(context) {
runInterruptible {
runCatchingCancellable { LocalManga(getFromFile(file), file) }.getOrNull()
}
}
private fun zipUri(file: File, entryName: String) = "cbz://${file.path}#$entryName"
private fun findFirstImageEntry(entries: Enumeration<out ZipEntry>): ZipEntry? {
val list = entries.toList()
.filterNot { it.isDirectory }
.sortedWith(compareBy(AlphanumComparator()) { x -> x.name })
val map = MimeTypeMap.getSingleton()
return list.firstOrNull {
map.getMimeTypeFromExtension(it.name.substringAfterLast('.'))
?.startsWith("image/") == true
}
}
override val sortOrders = setOf(SortOrder.ALPHABETICAL, SortOrder.RATING) override val sortOrders = setOf(SortOrder.ALPHABETICAL, SortOrder.RATING)
override suspend fun getPageUrl(page: MangaPage) = page.url override suspend fun getPageUrl(page: MangaPage) = page.url
@ -301,49 +179,16 @@ class LocalMangaRepository @Inject constructor(private val storageManager: Local
return coroutineScope { return coroutineScope {
val dispatcher = Dispatchers.IO.limitedParallelism(MAX_PARALLELISM) val dispatcher = Dispatchers.IO.limitedParallelism(MAX_PARALLELISM)
files.map { file -> files.map { file ->
getFromFileAsync(file, dispatcher) async(dispatcher) {
runCatchingCancellable { LocalMangaInput.of(file).getManga() }.getOrNull()
}
}.awaitAll() }.awaitAll()
}.filterNotNullTo(ArrayList(files.size)) }.filterNotNullTo(ArrayList(files.size))
} }
private suspend fun getAllFiles() = storageManager.getReadableDirs().flatMap { dir -> private suspend fun getAllFiles() = storageManager.getReadableDirs().flatMap { dir ->
dir.listFiles(filenameFilter)?.toList().orEmpty() dir.listFiles()?.toList().orEmpty()
} }
private fun Manga.copy2( private fun Collection<LocalManga>.unwrap(): List<Manga> = map { it.manga }
url: String = this.url,
coverUrl: String = this.coverUrl,
chapters: List<MangaChapter>? = this.chapters,
source: MangaSource = this.source,
) = Manga(
id = id,
title = title,
altTitle = altTitle,
url = url,
publicUrl = publicUrl,
rating = rating,
isNsfw = isNsfw,
coverUrl = coverUrl,
tags = tags,
state = state,
author = author,
largeCoverUrl = largeCoverUrl,
description = description,
chapters = chapters,
source = source,
)
private fun MangaChapter.copy(
url: String = this.url,
source: MangaSource = this.source,
) = MangaChapter(
id = id,
name = name,
number = number,
url = url,
scanlator = scanlator,
uploadDate = uploadDate,
branch = branch,
source = source,
)
} }

@ -6,9 +6,10 @@ import android.webkit.MimeTypeMap
import androidx.documentfile.provider.DocumentFile import androidx.documentfile.provider.DocumentFile
import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import org.koitharu.kotatsu.local.data.LocalManga
import org.koitharu.kotatsu.local.data.LocalStorageManager import org.koitharu.kotatsu.local.data.LocalStorageManager
import org.koitharu.kotatsu.local.domain.CbzMangaOutput import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.domain.LocalMangaRepository import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaSource
@ -24,17 +25,16 @@ import java.io.File
class DirMangaImporter( class DirMangaImporter(
private val context: Context, private val context: Context,
storageManager: LocalStorageManager, storageManager: LocalStorageManager,
private val localMangaRepository: LocalMangaRepository,
) : MangaImporter(storageManager) { ) : MangaImporter(storageManager) {
private val contentResolver = context.contentResolver private val contentResolver = context.contentResolver
override suspend fun import(uri: Uri): Manga { override suspend fun import(uri: Uri): LocalManga {
val root = requireNotNull(DocumentFile.fromTreeUri(context, uri)) { val root = requireNotNull(DocumentFile.fromTreeUri(context, uri)) {
"Provided uri $uri is not a tree" "Provided uri $uri is not a tree"
} }
val manga = Manga(root) val manga = Manga(root)
val output = CbzMangaOutput.get(getOutputDir(), manga) val output = LocalMangaOutput.getOrCreate(getOutputDir(), manga)
try { try {
val dest = output.use { val dest = output.use {
addPages( addPages(
@ -46,9 +46,9 @@ class DirMangaImporter(
it.sortChaptersByName() it.sortChaptersByName()
it.mergeWithExisting() it.mergeWithExisting()
it.finish() it.finish()
it.file it.rootFile
} }
return localMangaRepository.getFromFile(dest) return LocalMangaInput.of(dest).getManga()
} finally { } finally {
withContext(NonCancellable) { withContext(NonCancellable) {
output.cleanup() output.cleanup()
@ -57,7 +57,7 @@ class DirMangaImporter(
} }
} }
private suspend fun addPages(output: CbzMangaOutput, root: DocumentFile, path: String, state: State) { private suspend fun addPages(output: LocalMangaOutput, root: DocumentFile, path: String, state: State) {
var number = 0 var number = 0
for (file in root.listFiles().sortedWith(compareBy(AlphanumComparator()) { it.name.orEmpty() })) { for (file in root.listFiles().sortedWith(compareBy(AlphanumComparator()) { it.name.orEmpty() })) {
when { when {

@ -4,18 +4,17 @@ import android.content.Context
import android.net.Uri import android.net.Uri
import androidx.documentfile.provider.DocumentFile import androidx.documentfile.provider.DocumentFile
import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.android.qualifiers.ApplicationContext
import org.koitharu.kotatsu.local.data.LocalManga
import org.koitharu.kotatsu.local.data.LocalStorageManager
import java.io.File import java.io.File
import java.io.IOException import java.io.IOException
import javax.inject.Inject import javax.inject.Inject
import org.koitharu.kotatsu.local.data.LocalStorageManager
import org.koitharu.kotatsu.local.domain.LocalMangaRepository
import org.koitharu.kotatsu.parsers.model.Manga
abstract class MangaImporter( abstract class MangaImporter(
protected val storageManager: LocalStorageManager, protected val storageManager: LocalStorageManager,
) { ) {
abstract suspend fun import(uri: Uri): Manga abstract suspend fun import(uri: Uri): LocalManga
suspend fun getOutputDir(): File { suspend fun getOutputDir(): File {
return storageManager.getDefaultWriteableDir() ?: throw IOException("External files dir unavailable") return storageManager.getDefaultWriteableDir() ?: throw IOException("External files dir unavailable")
@ -24,13 +23,12 @@ abstract class MangaImporter(
class Factory @Inject constructor( class Factory @Inject constructor(
@ApplicationContext private val context: Context, @ApplicationContext private val context: Context,
private val storageManager: LocalStorageManager, private val storageManager: LocalStorageManager,
private val localMangaRepository: LocalMangaRepository,
) { ) {
fun create(uri: Uri): MangaImporter { fun create(uri: Uri): MangaImporter {
return when { return when {
isDir(uri) -> DirMangaImporter(context, storageManager, localMangaRepository) isDir(uri) -> DirMangaImporter(context, storageManager)
else -> ZipMangaImporter(storageManager, localMangaRepository) else -> ZipMangaImporter(storageManager)
} }
} }

@ -6,9 +6,9 @@ import kotlinx.coroutines.runInterruptible
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import org.koitharu.kotatsu.core.exceptions.UnsupportedFileException import org.koitharu.kotatsu.core.exceptions.UnsupportedFileException
import org.koitharu.kotatsu.local.data.CbzFilter import org.koitharu.kotatsu.local.data.CbzFilter
import org.koitharu.kotatsu.local.data.LocalManga
import org.koitharu.kotatsu.local.data.LocalStorageManager import org.koitharu.kotatsu.local.data.LocalStorageManager
import org.koitharu.kotatsu.local.domain.LocalMangaRepository import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.utils.ext.copyToSuspending import org.koitharu.kotatsu.utils.ext.copyToSuspending
import org.koitharu.kotatsu.utils.ext.resolveName import org.koitharu.kotatsu.utils.ext.resolveName
import java.io.File import java.io.File
@ -16,10 +16,9 @@ import java.io.IOException
class ZipMangaImporter( class ZipMangaImporter(
storageManager: LocalStorageManager, storageManager: LocalStorageManager,
private val localMangaRepository: LocalMangaRepository,
) : MangaImporter(storageManager) { ) : MangaImporter(storageManager) {
override suspend fun import(uri: Uri): Manga { override suspend fun import(uri: Uri): LocalManga {
val contentResolver = storageManager.contentResolver val contentResolver = storageManager.contentResolver
return withContext(Dispatchers.IO) { return withContext(Dispatchers.IO) {
val name = contentResolver.resolveName(uri) ?: throw IOException("Cannot fetch name from uri: $uri") val name = contentResolver.resolveName(uri) ?: throw IOException("Cannot fetch name from uri: $uri")
@ -34,7 +33,7 @@ class ZipMangaImporter(
source.copyToSuspending(output) source.copyToSuspending(output)
} }
} ?: throw IOException("Cannot open input stream: $uri") } ?: throw IOException("Cannot open input stream: $uri")
localMangaRepository.getFromFile(dest) LocalMangaInput.of(dest).getManga()
} }
} }
} }

@ -79,7 +79,7 @@ class ImportService : CoroutineIntentService() {
private suspend fun importImpl(uri: Uri): Manga { private suspend fun importImpl(uri: Uri): Manga {
val importer = importerFactory.create(uri) val importer = importerFactory.create(uri)
return importer.import(uri) return importer.import(uri).manga
} }
private fun sendBroadcast(manga: Manga) { private fun sendBroadcast(manga: Manga) {

@ -193,7 +193,7 @@ class PageLoader @Inject constructor(
.get() .get()
.header(CommonHeaders.REFERER, page.referer) .header(CommonHeaders.REFERER, page.referer)
.header(CommonHeaders.ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8") .header(CommonHeaders.ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8")
.cacheControl(CommonHeaders.CACHE_CONTROL_DISABLED) .cacheControl(CommonHeaders.CACHE_CONTROL_NO_STORE)
.tag(MangaSource::class.java, page.source) .tag(MangaSource::class.java, page.source)
.build() .build()
okHttp.newCall(request).await().use { response -> okHttp.newCall(request).await().use { response ->

@ -19,6 +19,5 @@ class RetainedLifecycleCoroutineScope(
override fun onCleared() { override fun onCleared() {
coroutineContext.cancel() coroutineContext.cancel()
lifecycle.removeOnClearedListener(this)
} }
} }

@ -41,3 +41,7 @@ fun <K, V> Map<K, V>.findKeyByValue(value: V): K? {
inline fun <T> Collection<T>.filterToSet(predicate: (T) -> Boolean): Set<T> { inline fun <T> Collection<T>.filterToSet(predicate: (T) -> Boolean): Set<T> {
return filterTo(ArraySet(size), predicate) return filterTo(ArraySet(size), predicate)
} }
fun <T> Sequence<T>.toListSorted(comparator: Comparator<T>): List<T> {
return toMutableList().apply { sortWith(comparator) }
}

@ -14,6 +14,7 @@ import kotlinx.coroutines.runInterruptible
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import org.koitharu.kotatsu.R import org.koitharu.kotatsu.R
import java.io.File import java.io.File
import java.io.FileFilter
import java.util.zip.ZipEntry import java.util.zip.ZipEntry
import java.util.zip.ZipFile import java.util.zip.ZipFile
@ -77,3 +78,19 @@ private fun computeSizeInternal(file: File): Long {
return file.length() return file.length()
} }
} }
fun File.listFilesRecursive(filter: FileFilter? = null): Sequence<File> = sequence {
listFilesRecursiveImpl(this@listFilesRecursive, filter)
}
private suspend fun SequenceScope<File>.listFilesRecursiveImpl(root: File, filter: FileFilter?) {
val ss = root.list() ?: return
for (s in ss) {
val f = File(root, s)
if (f.isDirectory) {
listFilesRecursiveImpl(f, filter)
} else if (filter == null || filter.accept(f)) {
yield(f)
}
}
}

Loading…
Cancel
Save