From 4f3281be992ca3e9b75b356b10b5090c49de7975 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 5 Jan 2022 10:43:32 +0200 Subject: [PATCH] MangaDex source --- app/build.gradle | 17 +- .../kotatsu/core/model/MangaSource.kt | 3 +- .../kotatsu/core/parser/ParserModule.kt | 1 + .../core/parser/site/MangaDexRepository.kt | 216 ++++++++++++++++++ .../kotatsu/core/prefs/AppSettings.kt | 4 +- .../kotatsu/details/ui/DetailsViewModel.kt | 23 +- .../kotatsu/utils/ScreenOrientationHelper.kt | 4 +- .../koitharu/kotatsu/utils/ext/IteratorExt.kt | 21 ++ .../org/koitharu/kotatsu/utils/ext/JsonExt.kt | 60 +++-- .../kotatsu/utils/json/JSONIterator.kt | 14 ++ .../kotatsu/utils/json/JSONStringIterator.kt | 13 ++ .../kotatsu/utils/json/JsonValuesIterator.kt | 17 ++ 12 files changed, 357 insertions(+), 36 deletions(-) create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaDexRepository.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/utils/ext/IteratorExt.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/utils/json/JSONIterator.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/utils/json/JSONStringIterator.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/utils/json/JsonValuesIterator.kt diff --git a/app/build.gradle b/app/build.gradle index 71d9fa36b..81580d3a2 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -59,13 +59,14 @@ tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { jvmTarget = JavaVersion.VERSION_1_8.toString() freeCompilerArgs += [ '-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi', + '-Xopt-in=kotlin.contracts.ExperimentalContracts', ] } } dependencies { implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar']) - implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2' - implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.0' implementation 'androidx.core:core-ktx:1.7.0' implementation 'androidx.activity:activity-ktx:1.4.0' @@ -85,9 +86,9 @@ dependencies { //noinspection LifecycleAnnotationProcessorWithJava8 kapt 'androidx.lifecycle:lifecycle-compiler:2.4.0' - implementation 'androidx.room:room-runtime:2.3.0' - implementation 'androidx.room:room-ktx:2.3.0' - kapt 'androidx.room:room-compiler:2.3.0' + implementation 'androidx.room:room-runtime:2.4.0' + implementation 'androidx.room:room-ktx:2.4.0' + kapt 'androidx.room:room-compiler:2.4.0' implementation 'com.squareup.okhttp3:okhttp:4.9.1' implementation 'com.squareup.okio:okio:2.10.0' @@ -105,14 +106,14 @@ dependencies { testImplementation 'junit:junit:4.13.2' testImplementation 'com.google.truth:truth:1.1.3' - testImplementation 'org.json:json:20210307' - testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2' + testImplementation 'org.json:json:20211205' + testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.0' testImplementation 'io.insert-koin:koin-test-junit4:3.1.4' androidTestImplementation 'androidx.test:runner:1.4.0' androidTestImplementation 'androidx.test:rules:1.4.0' androidTestImplementation 'androidx.test:core-ktx:1.4.0' androidTestImplementation 'androidx.test.ext:junit-ktx:1.1.3' - androidTestImplementation 'androidx.room:room-testing:2.3.0' + androidTestImplementation 'androidx.room:room-testing:2.4.0' androidTestImplementation 'com.google.truth:truth:1.1.3' } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt index 2d9018f5e..4ef49374c 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt @@ -40,7 +40,8 @@ enum class MangaSource( NINEMANGA_BR("NineManga Brasil", "pt", NineMangaRepository.Brazil::class.java), NINEMANGA_FR("NineManga Français", "fr", NineMangaRepository.Francais::class.java), EXHENTAI("ExHentai", null, ExHentaiRepository::class.java), - MANGAOWL("MangaOwl", "en", MangaOwlRepository::class.java) + MANGAOWL("MangaOwl", "en", MangaOwlRepository::class.java), + MANGADEX("MangaDex", null, MangaDexRepository::class.java), ; @get:Throws(NoBeanDefFoundException::class) diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt index d63676a44..c97f7a4eb 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt @@ -34,4 +34,5 @@ val parserModule factory(named(MangaSource.NINEMANGA_FR)) { NineMangaRepository.Francais(get()) } factory(named(MangaSource.EXHENTAI)) { ExHentaiRepository(get()) } factory(named(MangaSource.MANGAOWL)) { MangaOwlRepository(get()) } + factory(named(MangaSource.MANGADEX)) { MangaDexRepository(get()) } } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaDexRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaDexRepository.kt new file mode 100644 index 000000000..efce0b7a7 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaDexRepository.kt @@ -0,0 +1,216 @@ +package org.koitharu.kotatsu.core.parser.site + +import android.os.Build +import androidx.core.os.LocaleListCompat +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import org.koitharu.kotatsu.base.domain.MangaLoaderContext +import org.koitharu.kotatsu.core.model.* +import org.koitharu.kotatsu.core.parser.RemoteMangaRepository +import org.koitharu.kotatsu.utils.ext.* +import java.text.SimpleDateFormat +import java.util.* + +private const val PAGE_SIZE = 20 +private const val CONTENT_RATING = + "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" +private const val LOCALE_FALLBACK = "en" + +class MangaDexRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) { + + override val source = MangaSource.MANGADEX + override val defaultDomain = "mangadex.org" + + override val sortOrders: EnumSet = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) + + override suspend fun getList2( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder?, + ): List { + val domain = getDomain() + val url = buildString { + append("https://api.") + append(domain) + append("/manga?limit=") + append(PAGE_SIZE) + append("&offset=") + append(offset) + append("&includes[]=cover_art&includes[]=author&includes[]=artist&") + tags?.forEach { tag -> + append("includedTags[]=") + append(tag.key) + append('&') + } + if (!query.isNullOrEmpty()) { + append("title=") + append(query.urlEncoded()) + append('&') + } + append(CONTENT_RATING) + append("&order") + append(when (sortOrder) { + null, + SortOrder.UPDATED, + -> "[latestUploadedChapter]=desc" + SortOrder.ALPHABETICAL -> "[title]=asc" + SortOrder.NEWEST -> "[createdAt]=desc" + SortOrder.POPULARITY -> "[followedCount]=desc" + else -> "[followedCount]=desc" + }) + } + val json = loaderContext.httpGet(url).parseJson().getJSONArray("data") + return json.map { jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + val relations = jo.getJSONArray("relationships").associateByKey("type") + val cover = relations["cover_art"] + ?.getJSONObject("attributes") + ?.getString("fileName") + ?.let { + "https://uploads.$domain/covers/$id/$it" + } + Manga( + id = generateUid(id), + title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { + "Title should not be null" + }, + altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), + url = id, + publicUrl = "https://$domain/title/$id", + rating = Manga.NO_RATING, + isNsfw = attrs.getStringOrNull("contentRating") == "erotica", + coverUrl = cover?.plus(".256.jpg").orEmpty(), + largeCoverUrl = cover, + description = attrs.optJSONObject("description")?.selectByLocale(), + tags = attrs.getJSONArray("tags").mapToSet { tag -> + MangaTag( + title = tag.getJSONObject("attributes") + .getJSONObject("name") + .firstStringValue(), + key = tag.getString("id"), + source = source, + ) + }, + state = when (jo.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + author = (relations["author"] ?: relations["artist"]) + ?.getJSONObject("attributes") + ?.getStringOrNull("name"), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val domain = getDomain() + val attrsDeferred = async { + loaderContext.httpGet( + "https://api.$domain/manga/${manga.url}?includes[]=artist&includes[]=author&includes[]=cover_art" + ).parseJson().getJSONObject("data").getJSONObject("attributes") + } + val feedDeferred = async { + val url = buildString { + append("https://api.") + append(domain) + append("/manga/") + append(manga.url) + append("/feed") + append("?limit=96&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=0&") + append(CONTENT_RATING) + } + loaderContext.httpGet(url).parseJson().getJSONArray("data") + } + val mangaAttrs = attrsDeferred.await() + val feed = feedDeferred.await() + //2022-01-02T00:27:11+00:00 + val dateFormat = SimpleDateFormat( + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + "yyyy-MM-dd'T'HH:mm:ssX" + } else { + "yyyy-MM-dd'T'HH:mm:ss'+00:00'" + }, + Locale.ROOT + ) + manga.copy( + description = mangaAttrs.getJSONObject("description").selectByLocale() + ?: manga.description, + chapters = feed.mapNotNull { jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + if (attrs.optJSONArray("data").isNullOrEmpty()) { + return@mapNotNull null + } + val locale = Locale.forLanguageTag(attrs.getString("translatedLanguage")) + val relations = jo.getJSONArray("relationships").associateByKey("type") + val number = attrs.optInt("chapter", 0) + MangaChapter( + id = generateUid(id), + name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) + ?: "Chapter #$number", + number = number, + url = id, + scanlator = relations["scanlation_group"]?.getStringOrNull("name"), + uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), + branch = locale.displayName.toTitleCase(locale), + source = source, + ) + } + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val domain = getDomain() + val attrs = loaderContext.httpGet("https://api.$domain/chapter/${chapter.url}") + .parseJson() + .getJSONObject("data") + .getJSONObject("attributes") + val data = attrs.getJSONArray("data") + val prefix = "https://uploads.$domain/data/${attrs.getString("hash")}/" + val referer = "https://$domain/" + return List(data.length()) { i -> + val url = prefix + data.getString(i) + MangaPage( + id = generateUid(url), + url = url, + referer = referer, + preview = null, // TODO prefix + dataSaver.getString(i), + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val tags = loaderContext.httpGet("https://api.${getDomain()}/manga/tag").parseJson() + .getJSONArray("data") + return tags.mapToSet { jo -> + MangaTag( + title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue(), + key = jo.getString("id"), + source = source, + ) + } + } + + private fun JSONObject.firstStringValue() = values().next() as String + + private fun JSONObject.selectByLocale(): String? { + val preferredLocales = LocaleListCompat.getAdjustedDefault() + repeat(preferredLocales.size()) { i -> + val locale = preferredLocales.get(i) + getStringOrNull(locale.language)?.let { return it } + getStringOrNull(locale.toLanguageTag())?.let { return it } + } + return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String + } +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/prefs/AppSettings.kt b/app/src/main/java/org/koitharu/kotatsu/core/prefs/AppSettings.kt index fd09b006f..d006296a0 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/prefs/AppSettings.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/prefs/AppSettings.kt @@ -8,7 +8,7 @@ import androidx.collection.arraySetOf import androidx.core.content.edit import androidx.preference.PreferenceManager import kotlinx.coroutines.channels.awaitClose -import kotlinx.coroutines.channels.sendBlocking +import kotlinx.coroutines.channels.trySendBlocking import kotlinx.coroutines.flow.callbackFlow import org.koitharu.kotatsu.core.model.ZoomMode import org.koitharu.kotatsu.local.domain.LocalMangaRepository @@ -143,7 +143,7 @@ class AppSettings private constructor(private val prefs: SharedPreferences) : fun observe() = callbackFlow { val listener = SharedPreferences.OnSharedPreferenceChangeListener { _, key -> - sendBlocking(key) + trySendBlocking(key) } prefs.registerOnSharedPreferenceChangeListener(listener) awaitClose { diff --git a/app/src/main/java/org/koitharu/kotatsu/details/ui/DetailsViewModel.kt b/app/src/main/java/org/koitharu/kotatsu/details/ui/DetailsViewModel.kt index 570172cd4..8b7b241e6 100644 --- a/app/src/main/java/org/koitharu/kotatsu/details/ui/DetailsViewModel.kt +++ b/app/src/main/java/org/koitharu/kotatsu/details/ui/DetailsViewModel.kt @@ -24,7 +24,9 @@ import org.koitharu.kotatsu.local.domain.LocalMangaRepository import org.koitharu.kotatsu.tracker.domain.TrackingRepository import org.koitharu.kotatsu.utils.SingleLiveEvent import org.koitharu.kotatsu.utils.ext.mapToSet +import org.koitharu.kotatsu.utils.ext.toTitleCase import java.io.IOException +import java.util.* class DetailsViewModel( intent: MangaIntent, @@ -127,9 +129,7 @@ class DetailsViewModel( selectedBranch.value = if (hist != null) { manga.chapters?.find { it.id == hist.chapterId }?.branch } else { - manga.chapters - ?.groupBy { it.branch } - ?.maxByOrNull { it.value.size }?.key + predictBranch(manga.chapters) } mangaData.value = manga if (manga.source == MangaSource.LOCAL) { @@ -240,4 +240,21 @@ class DetailsViewModel( } return result } + + private fun predictBranch(chapters: List?): String? { + if (chapters.isNullOrEmpty()) { + return null + } + val groups = chapters.groupBy { it.branch } + val locale = Locale.getDefault() + var language = locale.displayLanguage.toTitleCase(locale) + if (groups.containsKey(language)) { + return language + } + language = locale.displayName.toTitleCase(locale) + if (groups.containsKey(language)) { + return language + } + return groups.maxByOrNull { it.value.size }?.key + } } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/ScreenOrientationHelper.kt b/app/src/main/java/org/koitharu/kotatsu/utils/ScreenOrientationHelper.kt index 8e19bfc43..6c856f4d4 100644 --- a/app/src/main/java/org/koitharu/kotatsu/utils/ScreenOrientationHelper.kt +++ b/app/src/main/java/org/koitharu/kotatsu/utils/ScreenOrientationHelper.kt @@ -7,7 +7,7 @@ import android.database.ContentObserver import android.os.Handler import android.provider.Settings import kotlinx.coroutines.channels.awaitClose -import kotlinx.coroutines.channels.sendBlocking +import kotlinx.coroutines.channels.trySendBlocking import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.onStart @@ -38,7 +38,7 @@ class ScreenOrientationHelper(private val activity: Activity) { fun observeAutoOrientation() = callbackFlow { val observer = object : ContentObserver(Handler(activity.mainLooper)) { override fun onChange(selfChange: Boolean) { - sendBlocking(isAutoRotationEnabled) + trySendBlocking(isAutoRotationEnabled) } } activity.contentResolver.registerContentObserver( diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/ext/IteratorExt.kt b/app/src/main/java/org/koitharu/kotatsu/utils/ext/IteratorExt.kt new file mode 100644 index 000000000..660bd7b2f --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/IteratorExt.kt @@ -0,0 +1,21 @@ +package org.koitharu.kotatsu.utils.ext + +fun Iterator.nextOrNull(): T? = if (hasNext()) next() else null + +fun Iterator.toList(): List { + if (!hasNext()) { + return emptyList() + } + val list = ArrayList() + while (hasNext()) list += next() + return list +} + +fun Iterator.toSet(): Set { + if (!hasNext()) { + return emptySet() + } + val list = LinkedHashSet() + while (hasNext()) list += next() + return list +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt b/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt index bf559cef6..0efa24264 100644 --- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt +++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt @@ -3,6 +3,10 @@ package org.koitharu.kotatsu.utils.ext import androidx.collection.ArraySet import org.json.JSONArray import org.json.JSONObject +import org.koitharu.kotatsu.utils.json.JSONIterator +import org.koitharu.kotatsu.utils.json.JSONStringIterator +import org.koitharu.kotatsu.utils.json.JSONValuesIterator +import kotlin.contracts.contract inline fun > JSONArray.mapTo( destination: C, @@ -16,10 +20,26 @@ inline fun > JSONArray.mapTo( return destination } +inline fun > JSONArray.mapNotNullTo( + destination: C, + block: (JSONObject) -> R? +): C { + val len = length() + for (i in 0 until len) { + val jo = getJSONObject(i) + destination.add(block(jo) ?: continue) + } + return destination +} + inline fun JSONArray.map(block: (JSONObject) -> T): List { return mapTo(ArrayList(length()), block) } +inline fun JSONArray.mapNotNull(block: (JSONObject) -> T?): List { + return mapNotNullTo(ArrayList(length()), block) +} + fun JSONArray.mapIndexed(block: (Int, JSONObject) -> T): List { val len = length() val result = ArrayList(len) @@ -46,26 +66,6 @@ operator fun JSONArray.iterator(): Iterator = JSONIterator(this) fun JSONArray.stringIterator(): Iterator = JSONStringIterator(this) -private class JSONIterator(private val array: JSONArray) : Iterator { - - private val total = array.length() - private var index = 0 - - override fun hasNext() = index < total - 1 - - override fun next(): JSONObject = array.getJSONObject(index++) -} - -private class JSONStringIterator(private val array: JSONArray) : Iterator { - - private val total = array.length() - private var index = 0 - - override fun hasNext() = index < total - 1 - - override fun next(): String = array.getString(index++) -} - fun JSONArray.mapToSet(block: (JSONObject) -> T): Set { val len = length() val result = ArraySet(len) @@ -74,4 +74,24 @@ fun JSONArray.mapToSet(block: (JSONObject) -> T): Set { result.add(block(jo)) } return result +} + +fun JSONObject.values(): Iterator = JSONValuesIterator(this) + +fun JSONArray.associateByKey(key: String): Map { + val destination = LinkedHashMap(length()) + repeat(length()) { i -> + val item = getJSONObject(i) + val keyValue = item.getString(key) + destination[keyValue] = item + } + return destination +} + +fun JSONArray?.isNullOrEmpty(): Boolean { + contract { + returns(false) implies (this@isNullOrEmpty != null) + } + + return this == null || this.length() == 0 } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/json/JSONIterator.kt b/app/src/main/java/org/koitharu/kotatsu/utils/json/JSONIterator.kt new file mode 100644 index 000000000..f99b47c79 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/utils/json/JSONIterator.kt @@ -0,0 +1,14 @@ +package org.koitharu.kotatsu.utils.json + +import org.json.JSONArray +import org.json.JSONObject + +class JSONIterator(private val array: JSONArray) : Iterator { + + private val total = array.length() + private var index = 0 + + override fun hasNext() = index < total - 1 + + override fun next(): JSONObject = array.getJSONObject(index++) +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/json/JSONStringIterator.kt b/app/src/main/java/org/koitharu/kotatsu/utils/json/JSONStringIterator.kt new file mode 100644 index 000000000..a13027da2 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/utils/json/JSONStringIterator.kt @@ -0,0 +1,13 @@ +package org.koitharu.kotatsu.utils.json + +import org.json.JSONArray + +class JSONStringIterator(private val array: JSONArray) : Iterator { + + private val total = array.length() + private var index = 0 + + override fun hasNext() = index < total - 1 + + override fun next(): String = array.getString(index++) +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/json/JsonValuesIterator.kt b/app/src/main/java/org/koitharu/kotatsu/utils/json/JsonValuesIterator.kt new file mode 100644 index 000000000..1bf833310 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/utils/json/JsonValuesIterator.kt @@ -0,0 +1,17 @@ +package org.koitharu.kotatsu.utils.json + +import org.json.JSONObject + +class JSONValuesIterator( + private val jo: JSONObject, +): Iterator { + + private val keyIterator = jo.keys() + + override fun hasNext(): Boolean = keyIterator.hasNext() + + override fun next(): Any { + val key = keyIterator.next() + return jo.get(key) + } +} \ No newline at end of file