From b8c244633586a744dd5fda36a799e5cc4eb73f4c Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Mon, 15 Apr 2024 02:38:23 +0300 Subject: [PATCH] Tracker improvements --- .../shirizu/core/database/ShirizuDatabase.kt | 6 +- .../core/database/dao/FavouritesDao.kt | 6 + .../shirizu/core/database/dao/HistoryDao.kt | 6 + .../shirizu/core/database/dao/TrackLogsDao.kt | 3 + .../shirizu/core/database/dao/TracksDao.kt | 11 ++ .../core/database/entity/TrackEntity.kt | 27 +++- .../core/database/entity/TrackLogEntity.kt | 2 +- .../core/database/entity/TrackWithManga.kt | 13 ++ .../{Migrations1To2.kt => Migration1To2.kt} | 0 .../core/database/migrations/Migration2To3.kt | 16 +++ .../xtimms/shirizu/core/tracker/Tracker.kt | 70 ++++------ .../core/tracker/model/MangaUpdates.kt | 10 ++ .../data/repository/HistoryRepository.kt | 6 +- .../data/repository/TrackingRepository.kt | 132 ++++++++++-------- .../settings/shelf/ShelfSettingsView.kt | 28 ++-- .../org/xtimms/shirizu/utils/lang/Date.kt | 2 + .../xtimms/shirizu/utils/lang/Primitive.kt | 4 + .../shirizu/work/WorkScheduleManager.kt | 8 +- .../shirizu/work/tracker/TrackWorker.kt | 89 ++++-------- 19 files changed, 245 insertions(+), 194 deletions(-) create mode 100644 app/src/main/java/org/xtimms/shirizu/core/database/entity/TrackWithManga.kt rename app/src/main/java/org/xtimms/shirizu/core/database/migrations/{Migrations1To2.kt => Migration1To2.kt} (100%) create mode 100644 app/src/main/java/org/xtimms/shirizu/core/database/migrations/Migration2To3.kt diff --git a/app/src/main/java/org/xtimms/shirizu/core/database/ShirizuDatabase.kt b/app/src/main/java/org/xtimms/shirizu/core/database/ShirizuDatabase.kt index 160510f..3e5d8f4 100644 --- a/app/src/main/java/org/xtimms/shirizu/core/database/ShirizuDatabase.kt +++ b/app/src/main/java/org/xtimms/shirizu/core/database/ShirizuDatabase.kt @@ -35,9 +35,10 @@ import org.xtimms.shirizu.core.database.entity.TagEntity import org.xtimms.shirizu.core.database.entity.TrackEntity import org.xtimms.shirizu.core.database.entity.TrackLogEntity import org.xtimms.shirizu.core.database.migrations.Migration1To2 +import org.xtimms.shirizu.core.database.migrations.Migration2To3 import org.xtimms.shirizu.utils.lang.processLifecycleScope -const val DATABASE_VERSION = 2 +const val DATABASE_VERSION = 3 @Database( entities = [ @@ -83,7 +84,8 @@ abstract class ShirizuDatabase : RoomDatabase() { } fun getDatabaseMigrations(context: Context): Array = arrayOf( - Migration1To2() + Migration1To2(), + Migration2To3() ) fun ShirizuDatabase(context: Context): ShirizuDatabase = Room diff --git a/app/src/main/java/org/xtimms/shirizu/core/database/dao/FavouritesDao.kt b/app/src/main/java/org/xtimms/shirizu/core/database/dao/FavouritesDao.kt index 31e706f..e3009f3 100644 --- a/app/src/main/java/org/xtimms/shirizu/core/database/dao/FavouritesDao.kt +++ b/app/src/main/java/org/xtimms/shirizu/core/database/dao/FavouritesDao.kt @@ -51,6 +51,9 @@ abstract class FavouritesDao { @Query("SELECT * FROM favourites WHERE deleted_at = 0 ORDER BY created_at DESC LIMIT :limit OFFSET :offset") abstract suspend fun findAllRaw(offset: Int, limit: Int): List + @Query("SELECT DISTINCT manga_id FROM favourites WHERE deleted_at = 0 AND category_id IN (SELECT category_id FROM favourite_categories WHERE track = 1)") + abstract suspend fun findIdsWithTrack(): LongArray + @Transaction @Query( "SELECT * FROM favourites WHERE category_id = :categoryId AND deleted_at = 0 " + @@ -107,6 +110,9 @@ abstract class FavouritesDao { @Query("SELECT DISTINCT category_id FROM favourites WHERE manga_id IN (:mangaIds) AND deleted_at = 0") abstract suspend fun findCategoriesIds(mangaIds: Collection): List + @Query("SELECT DISTINCT favourite_categories.category_id FROM favourites LEFT JOIN favourite_categories ON favourites.category_id = favourite_categories.category_id WHERE manga_id = :mangaId AND favourites.deleted_at = 0 AND favourite_categories.deleted_at = 0 AND favourite_categories.track = 1") + abstract suspend fun findCategoriesIdsWithTrack(mangaId: Long): List + /** INSERT **/ @Insert(onConflict = OnConflictStrategy.REPLACE) diff --git a/app/src/main/java/org/xtimms/shirizu/core/database/dao/HistoryDao.kt b/app/src/main/java/org/xtimms/shirizu/core/database/dao/HistoryDao.kt index 02c2692..69b21a8 100644 --- a/app/src/main/java/org/xtimms/shirizu/core/database/dao/HistoryDao.kt +++ b/app/src/main/java/org/xtimms/shirizu/core/database/dao/HistoryDao.kt @@ -54,6 +54,9 @@ abstract class HistoryDao { @Query("SELECT * FROM manga WHERE manga_id IN (SELECT manga_id FROM history WHERE deleted_at = 0)") abstract suspend fun findAllManga(): List + @Query("SELECT manga_id FROM history WHERE deleted_at = 0") + abstract suspend fun findAllIds(): LongArray + @Query( """SELECT tags.* FROM tags LEFT JOIN manga_tags ON tags.tag_id = manga_tags.tag_id @@ -74,6 +77,9 @@ abstract class HistoryDao { @Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0") abstract fun observeCount(): Flow + @Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0") + abstract suspend fun getCount(): Int + @Query("SELECT percent FROM history WHERE manga_id = :id AND deleted_at = 0") abstract suspend fun findProgress(id: Long): Float? diff --git a/app/src/main/java/org/xtimms/shirizu/core/database/dao/TrackLogsDao.kt b/app/src/main/java/org/xtimms/shirizu/core/database/dao/TrackLogsDao.kt index 64d926d..c2e301a 100644 --- a/app/src/main/java/org/xtimms/shirizu/core/database/dao/TrackLogsDao.kt +++ b/app/src/main/java/org/xtimms/shirizu/core/database/dao/TrackLogsDao.kt @@ -28,6 +28,9 @@ interface TrackLogsDao { @Query("DELETE FROM track_logs WHERE manga_id NOT IN (SELECT manga_id FROM tracks)") suspend fun gc() + @Query("DELETE FROM track_logs WHERE id IN (SELECT id FROM track_logs ORDER BY created_at DESC LIMIT 0 OFFSET :size)") + suspend fun trim(size: Int) + @Query("SELECT COUNT(*) FROM track_logs") suspend fun count(): Int } \ No newline at end of file diff --git a/app/src/main/java/org/xtimms/shirizu/core/database/dao/TracksDao.kt b/app/src/main/java/org/xtimms/shirizu/core/database/dao/TracksDao.kt index 4fc1ff0..0d34f8b 100644 --- a/app/src/main/java/org/xtimms/shirizu/core/database/dao/TracksDao.kt +++ b/app/src/main/java/org/xtimms/shirizu/core/database/dao/TracksDao.kt @@ -8,6 +8,7 @@ import androidx.room.Upsert import kotlinx.coroutines.flow.Flow import org.xtimms.shirizu.core.database.entity.MangaWithTags import org.xtimms.shirizu.core.database.entity.TrackEntity +import org.xtimms.shirizu.core.database.entity.TrackWithManga @Dao abstract class TracksDao { @@ -15,6 +16,13 @@ abstract class TracksDao { @Query("SELECT * FROM tracks") abstract suspend fun findAll(): List + @Transaction + @Query("SELECT * FROM tracks ORDER BY last_check_time ASC LIMIT :limit OFFSET :offset") + abstract suspend fun findAll(offset: Int, limit: Int): List + + @Query("SELECT manga_id FROM tracks") + abstract suspend fun findAllIds(): LongArray + @Query("SELECT * FROM tracks WHERE manga_id IN (:ids)") abstract suspend fun findAll(ids: Collection): List @@ -24,6 +32,9 @@ abstract class TracksDao { @Query("SELECT chapters_new FROM tracks WHERE manga_id = :mangaId") abstract suspend fun findNewChapters(mangaId: Long): Int? + @Query("SELECT COUNT(*) FROM tracks") + abstract suspend fun getTracksCount(): Int + @Query("SELECT manga_id, chapters_new FROM tracks") abstract fun observeNewChaptersMap(): Flow { - val knownManga = MutableLongSet() - val result = ArrayList() - // Favourites - val favourites = repository.getAllFavouritesManga() - channels.updateChannels(favourites.keys) - for ((category, mangaList) in favourites) { - if (!category.isTrackingEnabled || mangaList.isEmpty()) { - continue - } - val categoryTracks = repository.getTracks(mangaList) - val channelId = if (channels.isFavouriteNotificationsEnabled(category)) { - channels.getFavouritesChannelId(category.id) - } else { - null - } - for (track in categoryTracks) { - if (knownManga.add(track.manga.id)) { - result.add(TrackingItem(track, channelId)) - } - } - } - // History - val history = repository.getAllHistoryManga() - val historyTracks = repository.getTracks(history) - val channelId = if (channels.isHistoryNotificationsEnabled()) { - channels.getHistoryChannelId() - } else { - null - } - for (track in historyTracks) { - if (knownManga.add(track.manga.id)) { - result.add(TrackingItem(track, channelId)) - } + suspend fun getTracks(limit: Int): List { + repository.updateTracks() + return repository.getTracks(0, limit).map { + val categoryId = repository.getCategoryId(it.manga.id) + TrackingItem( + tracking = it, + channelId = if (categoryId == 0L) { + channels.getHistoryChannelId() + } else { + channels.getFavouritesChannelId(categoryId) + }, + ) } - result.trimToSize() - return result - } - - suspend fun getTracks(ids: Set): List { - return getAllTracks().filterTo(ArrayList(ids.size)) { x -> x.tracking.manga.id in ids } } suspend fun gc() { @@ -76,11 +49,18 @@ class Tracker @Inject constructor( suspend fun fetchUpdates( track: MangaTracking, commit: Boolean - ): MangaUpdates.Success = withMangaLock(track.manga.id) { - val repo = mangaRepositoryFactory.create(track.manga.source) - require(repo is RemoteMangaRepository) { "Repository ${repo.javaClass.simpleName} is not supported" } - val manga = repo.getDetails(track.manga, CachePolicy.WRITE_ONLY) - val updates = compare(track, manga, getBranch(manga)) + ): MangaUpdates = withMangaLock(track.manga.id) { + val updates = runCatchingCancellable { + val repo = mangaRepositoryFactory.create(track.manga.source) + require(repo is RemoteMangaRepository) { "Repository ${repo.javaClass.simpleName} is not supported" } + val manga = repo.getDetails(track.manga, CachePolicy.WRITE_ONLY) + compare(track, manga, getBranch(manga)) + }.getOrElse { error -> + MangaUpdates.Failure( + manga = track.manga, + error = error + ) + } if (commit) { repository.saveUpdates(updates) } diff --git a/app/src/main/java/org/xtimms/shirizu/core/tracker/model/MangaUpdates.kt b/app/src/main/java/org/xtimms/shirizu/core/tracker/model/MangaUpdates.kt index 7d0cde0..7a89ce5 100644 --- a/app/src/main/java/org/xtimms/shirizu/core/tracker/model/MangaUpdates.kt +++ b/app/src/main/java/org/xtimms/shirizu/core/tracker/model/MangaUpdates.kt @@ -3,6 +3,7 @@ package org.xtimms.shirizu.core.tracker.model import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaChapter import org.xtimms.shirizu.core.exceptions.TooManyRequestExceptions +import org.xtimms.shirizu.utils.lang.ifZero sealed interface MangaUpdates { @@ -16,6 +17,15 @@ sealed interface MangaUpdates { ) : MangaUpdates { fun isNotEmpty() = newChapters.isNotEmpty() + + fun lastChapterDate(): Long { + val lastChapter = newChapters.lastOrNull() + return if (lastChapter == null) { + manga.chapters?.lastOrNull()?.uploadDate ?: 0L + } else { + lastChapter.uploadDate.ifZero { System.currentTimeMillis() } + } + } } data class Failure( diff --git a/app/src/main/java/org/xtimms/shirizu/data/repository/HistoryRepository.kt b/app/src/main/java/org/xtimms/shirizu/data/repository/HistoryRepository.kt index 4d8a733..a0dfc09 100644 --- a/app/src/main/java/org/xtimms/shirizu/data/repository/HistoryRepository.kt +++ b/app/src/main/java/org/xtimms/shirizu/data/repository/HistoryRepository.kt @@ -21,8 +21,6 @@ import org.xtimms.shirizu.utils.ReversibleHandle import org.xtimms.shirizu.utils.lang.mapItems import javax.inject.Inject -const val PROGRESS_NONE = -1f - @Reusable class HistoryRepository @Inject constructor( private val db: ShirizuDatabase, @@ -34,6 +32,10 @@ class HistoryRepository @Inject constructor( return entities.map { it.manga.toManga(it.tags.toMangaTags()) } } + suspend fun getCount(): Int { + return db.getHistoryDao().getCount() + } + suspend fun getLastOrNull(): Manga? { val entity = db.getHistoryDao().findAll(0, 1).firstOrNull() ?: return null return entity.manga.toManga(entity.tags.toMangaTags()) diff --git a/app/src/main/java/org/xtimms/shirizu/data/repository/TrackingRepository.kt b/app/src/main/java/org/xtimms/shirizu/data/repository/TrackingRepository.kt index b0856c3..372022e 100644 --- a/app/src/main/java/org/xtimms/shirizu/data/repository/TrackingRepository.kt +++ b/app/src/main/java/org/xtimms/shirizu/data/repository/TrackingRepository.kt @@ -1,7 +1,6 @@ package org.xtimms.shirizu.data.repository import androidx.annotation.VisibleForTesting -import androidx.collection.MutableLongSet import androidx.room.withTransaction import dagger.Reusable import kotlinx.coroutines.ExperimentalCoroutinesApi @@ -12,7 +11,6 @@ import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onStart import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.util.mapToSet import org.xtimms.shirizu.core.database.ShirizuDatabase import org.xtimms.shirizu.core.database.entity.MangaEntity import org.xtimms.shirizu.core.database.entity.TrackEntity @@ -21,19 +19,23 @@ import org.xtimms.shirizu.core.database.entity.toFavouriteCategory import org.xtimms.shirizu.core.database.entity.toManga import org.xtimms.shirizu.core.database.entity.toTrackingLogItem import org.xtimms.shirizu.core.model.FavouriteCategory -import org.xtimms.shirizu.core.model.isLocal import org.xtimms.shirizu.core.parser.local.LocalMangaRepository import org.xtimms.shirizu.core.tracker.model.MangaTracking import org.xtimms.shirizu.core.tracker.model.MangaUpdates import org.xtimms.shirizu.core.tracker.model.TrackingLogItem +import org.xtimms.shirizu.utils.lang.ifZero import org.xtimms.shirizu.utils.lang.mapItems -import java.time.Instant +import org.xtimms.shirizu.utils.lang.toInstantOrNull import java.util.concurrent.atomic.AtomicBoolean import javax.inject.Inject import javax.inject.Provider private const val NO_ID = 0L + +@Deprecated("Use buckets") private const val MAX_QUERY_IDS = 100 +private const val MAX_BUCKET_SIZE = 20 +private const val MAX_LOG_SIZE = 120 @Reusable class TrackingRepository @Inject constructor( @@ -66,35 +68,18 @@ class TrackingRepository @Inject constructor( .onStart { gcIfNotCalled() } } - suspend fun getTracks(mangaList: Collection): List { - val ids = mangaList.mapToSet { it.id } - val dao = db.getTracksDao() - val tracks = if (ids.size <= MAX_QUERY_IDS) { - dao.findAll(ids) - } else { - // TODO split tracks in the worker - ids.windowed(MAX_QUERY_IDS, MAX_QUERY_IDS, true) - .flatMap { dao.findAll(it) } - }.groupBy { it.mangaId } - val idSet = MutableLongSet(mangaList.size) - val result = ArrayList(mangaList.size) - for (item in mangaList) { - val manga = if (item.isLocal) { - localMangaRepositoryProvider.get().getRemoteManga(item) ?: continue - } else { - item - } - if (!idSet.add(manga.id)) { - continue - } - val track = tracks[manga.id]?.lastOrNull() - result += MangaTracking( - manga = manga, - lastChapterId = track?.lastChapterId ?: NO_ID, - lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli), + suspend fun getCategoryId(mangaId: Long): Long { + return db.getFavouritesDao().findCategoriesIdsWithTrack(mangaId).firstOrNull() ?: NO_ID + } + + suspend fun getTracks(offset: Int, limit: Int): List { + return db.getTracksDao().findAll(offset, limit).map { + MangaTracking( + manga = it.manga.toManga(emptySet()), + lastChapterId = it.track.lastChapterId, + lastCheck = it.track.lastCheckTime.toInstantOrNull(), ) } - return result } @VisibleForTesting @@ -103,7 +88,7 @@ class TrackingRepository @Inject constructor( return MangaTracking( manga = manga, lastChapterId = track?.lastChapterId ?: NO_ID, - lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli), + lastCheck = track?.lastCheckTime?.toInstantOrNull(), ) } @@ -133,16 +118,19 @@ class TrackingRepository @Inject constructor( suspend fun clearCounters() = db.getTracksDao().clearCounters() - suspend fun gc() { + suspend fun gc() = db.withTransaction { db.getTracksDao().gc() - db.getTrackLogsDao().gc() + db.getTrackLogsDao().run { + gc() + trim(MAX_LOG_SIZE) + } } - suspend fun saveUpdates(updates: MangaUpdates.Success) { + suspend fun saveUpdates(updates: MangaUpdates) { db.withTransaction { val track = getOrCreateTrack(updates.manga.id).mergeWith(updates) db.getTracksDao().upsert(track) - if (updates.isValid && updates.newChapters.isNotEmpty()) { + if (updates is MangaUpdates.Success && updates.isValid && updates.newChapters.isNotEmpty()) { updatePercent(updates) val logEntity = TrackLogEntity( mangaId = updates.manga.id, @@ -174,7 +162,6 @@ class TrackingRepository @Inject constructor( val lastChapterId = chapters.lastOrNull()?.id ?: NO_ID val entity = TrackEntity( mangaId = manga.id, - totalChapters = chapters.size, lastChapterId = lastChapterId, newChapters = when { track.newChapters == 0 -> 0 @@ -182,8 +169,9 @@ class TrackingRepository @Inject constructor( chapterIndex >= lastNewChapterIndex -> chapters.lastIndex - chapterIndex else -> track.newChapters }, - lastCheck = System.currentTimeMillis(), - lastNotifiedChapterId = lastChapterId, + lastCheckTime = System.currentTimeMillis(), + lastChapterDate = maxOf(track.lastChapterDate, chapters.lastOrNull()?.uploadDate ?: 0L), + lastResult = track.lastResult, ) db.getTracksDao().upsert(entity) } @@ -204,19 +192,36 @@ class TrackingRepository @Inject constructor( } } - suspend fun getAllHistoryManga(): List { - return db.getHistoryDao().findAllManga().toMangaList() + suspend fun updateTracks() = db.withTransaction { + val dao = db.getTracksDao() + dao.gc() + val ids = dao.findAllIds().toMutableSet() + val size = ids.size + // history + val historyIds = db.getHistoryDao().findAllIds() + for (mangaId in historyIds) { + if (!ids.remove(mangaId)) { + dao.upsert(TrackEntity.create(mangaId)) + } + } + + // favourites + val favoritesIds = db.getFavouritesDao().findIdsWithTrack() + for (mangaId in favoritesIds) { + if (!ids.remove(mangaId)) { + dao.upsert(TrackEntity.create(mangaId)) + } + } + + // remove unused + for (mangaId in ids) { + dao.delete(mangaId) + } + size - ids.size } private suspend fun getOrCreateTrack(mangaId: Long): TrackEntity { - return db.getTracksDao().find(mangaId) ?: TrackEntity( - mangaId = mangaId, - totalChapters = 0, - lastChapterId = 0L, - newChapters = 0, - lastCheck = 0L, - lastNotifiedChapterId = 0L, - ) + return db.getTracksDao().find(mangaId) ?: TrackEntity.create(mangaId) } private suspend fun updatePercent(updates: MangaUpdates.Success) { @@ -234,16 +239,27 @@ class TrackingRepository @Inject constructor( db.getHistoryDao().update(history.copy(percent = newPercent)) } - private fun TrackEntity.mergeWith(updates: MangaUpdates.Success): TrackEntity { + private fun TrackEntity.mergeWith(updates: MangaUpdates): TrackEntity { val chapters = updates.manga.chapters.orEmpty() - return TrackEntity( - mangaId = mangaId, - totalChapters = chapters.size, - lastChapterId = chapters.lastOrNull()?.id ?: NO_ID, - newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0, - lastCheck = System.currentTimeMillis(), - lastNotifiedChapterId = NO_ID, - ) + return when (updates) { + is MangaUpdates.Failure -> TrackEntity( + mangaId = mangaId, + lastChapterId = lastChapterId, + newChapters = newChapters, + lastCheckTime = System.currentTimeMillis(), + lastChapterDate = lastChapterDate, + lastResult = TrackEntity.RESULT_FAILED, + ) + + is MangaUpdates.Success -> TrackEntity( + mangaId = mangaId, + lastChapterId = chapters.lastOrNull()?.id ?: NO_ID, + newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0, + lastCheckTime = System.currentTimeMillis(), + lastChapterDate = updates.lastChapterDate().ifZero { lastChapterDate }, + lastResult = if (updates.isNotEmpty()) TrackEntity.RESULT_HAS_UPDATE else TrackEntity.RESULT_NO_UPDATE, + ) + } } private suspend fun gcIfNotCalled() { diff --git a/app/src/main/java/org/xtimms/shirizu/sections/settings/shelf/ShelfSettingsView.kt b/app/src/main/java/org/xtimms/shirizu/sections/settings/shelf/ShelfSettingsView.kt index 69c4332..bd3fa97 100644 --- a/app/src/main/java/org/xtimms/shirizu/sections/settings/shelf/ShelfSettingsView.kt +++ b/app/src/main/java/org/xtimms/shirizu/sections/settings/shelf/ShelfSettingsView.kt @@ -14,7 +14,7 @@ import androidx.compose.material.icons.Icons import androidx.compose.material.icons.outlined.Category import androidx.compose.material.icons.outlined.GridView import androidx.compose.material.icons.outlined.Numbers -import androidx.compose.material.icons.outlined.Update +import androidx.compose.material.icons.outlined.RssFeed import androidx.compose.material3.AlertDialog import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.Icon @@ -39,13 +39,13 @@ import org.xtimms.shirizu.R import org.xtimms.shirizu.core.components.PreferenceItem import org.xtimms.shirizu.core.components.PreferenceSubtitle import org.xtimms.shirizu.core.components.PreferenceSwitch +import org.xtimms.shirizu.core.components.PreferenceSwitchWithDivider import org.xtimms.shirizu.core.components.ScaffoldWithTopAppBar import org.xtimms.shirizu.core.prefs.AppSettings import org.xtimms.shirizu.core.prefs.AppSettings.getInt -import org.xtimms.shirizu.core.prefs.AppSettings.getString import org.xtimms.shirizu.core.prefs.GRID_COLUMNS -import org.xtimms.shirizu.core.prefs.PROXY_ADDRESS import org.xtimms.shirizu.core.prefs.TABS_MANGA_COUNT +import org.xtimms.shirizu.core.prefs.TRACKER import org.xtimms.shirizu.sections.shelf.ShelfViewModel const val SHELF_SETTINGS_DESTINATION = "shelf_settings" @@ -54,9 +54,11 @@ const val SHELF_SETTINGS_DESTINATION = "shelf_settings" fun ShelfSettingsView( shelfViewModel: ShelfViewModel = hiltViewModel(), navigateBack: () -> Unit, - navigateToCategories: () -> Unit + navigateToCategories: () -> Unit, + navigateToTrackerSettings: () -> Unit = {} ) { + var isTrackerEnabled by remember { mutableStateOf(AppSettings.isTrackerEnabled()) } var showGridColumnsDialog by remember { mutableStateOf(false) } val categories by shelfViewModel.categories.collectAsStateWithLifecycle(emptyList()) @@ -106,7 +108,10 @@ fun ShelfSettingsView( item { PreferenceItem( title = stringResource(id = R.string.grid_columns_count), - description = stringResource(id = R.string.grid_columns_count_desc, gridColumns), + description = stringResource( + id = R.string.grid_columns_count_desc, + gridColumns + ), icon = Icons.Outlined.GridView ) { showGridColumnsDialog = true } } @@ -114,10 +119,17 @@ fun ShelfSettingsView( PreferenceSubtitle(text = stringResource(id = R.string.updates)) } item { - PreferenceItem( + PreferenceSwitchWithDivider( title = stringResource(id = R.string.auto_update), - description = "Off", - icon = Icons.Outlined.Update + description = if (isTrackerEnabled) stringResource(id = R.string.on) + else stringResource(id = R.string.off), + icon = Icons.Outlined.RssFeed, + isChecked = isTrackerEnabled, + onClick = navigateToTrackerSettings, + onChecked = { + isTrackerEnabled = !isTrackerEnabled + AppSettings.updateValue(TRACKER, isTrackerEnabled) + } ) } } diff --git a/app/src/main/java/org/xtimms/shirizu/utils/lang/Date.kt b/app/src/main/java/org/xtimms/shirizu/utils/lang/Date.kt index 212a9b1..696081e 100644 --- a/app/src/main/java/org/xtimms/shirizu/utils/lang/Date.kt +++ b/app/src/main/java/org/xtimms/shirizu/utils/lang/Date.kt @@ -69,6 +69,8 @@ fun LocalDateTime.toDate(): Date = Date(this.toEpochSecond( ZoneId.systemDefault().rules.getOffset(this) ) * 1000) +fun Long.toInstantOrNull() = if (this == 0L) null else Instant.ofEpochMilli(this) + sealed class DateTimeAgo { abstract fun format(resources: Resources): String diff --git a/app/src/main/java/org/xtimms/shirizu/utils/lang/Primitive.kt b/app/src/main/java/org/xtimms/shirizu/utils/lang/Primitive.kt index 0ada14c..de4b73c 100644 --- a/app/src/main/java/org/xtimms/shirizu/utils/lang/Primitive.kt +++ b/app/src/main/java/org/xtimms/shirizu/utils/lang/Primitive.kt @@ -21,5 +21,9 @@ inline val String.intState mutableIntStateOf(this.getInt()) } +inline fun Int.ifZero(defaultValue: () -> Int): Int = if (this == 0) defaultValue() else this + +inline fun Long.ifZero(defaultValue: () -> Long): Long = if (this == 0L) defaultValue() else this + // clamp(3.5f, 6.7f) > [0.0f, 1.0f] fun Float.clamp(min: Float, max: Float): Float = (1f - ((this.coerceIn(min, max) - min) / (max - min))) \ No newline at end of file diff --git a/app/src/main/java/org/xtimms/shirizu/work/WorkScheduleManager.kt b/app/src/main/java/org/xtimms/shirizu/work/WorkScheduleManager.kt index cce47d1..6928a60 100644 --- a/app/src/main/java/org/xtimms/shirizu/work/WorkScheduleManager.kt +++ b/app/src/main/java/org/xtimms/shirizu/work/WorkScheduleManager.kt @@ -17,17 +17,11 @@ class WorkScheduleManager @Inject constructor( fun init() { processLifecycleScope.launch(Dispatchers.Default) { - updateWorkerImpl(trackerScheduler, isEnabled = true, force = false) // TODO + updateWorkerImpl(trackerScheduler, AppSettings.isTrackerEnabled(), force = true) updateWorkerImpl(suggestionScheduler, AppSettings.isSuggestionsEnabled(), force = false) } } - private fun updateWorker(scheduler: PeriodicWorkScheduler, isEnabled: Boolean, force: Boolean) { - processLifecycleScope.launch(Dispatchers.Default) { - updateWorkerImpl(scheduler, isEnabled, force) - } - } - private suspend fun updateWorkerImpl(scheduler: PeriodicWorkScheduler, isEnabled: Boolean, force: Boolean) { if (force || scheduler.isScheduled() != isEnabled) { if (isEnabled) { diff --git a/app/src/main/java/org/xtimms/shirizu/work/tracker/TrackWorker.kt b/app/src/main/java/org/xtimms/shirizu/work/tracker/TrackWorker.kt index 59dbc97..e2de4c1 100644 --- a/app/src/main/java/org/xtimms/shirizu/work/tracker/TrackWorker.kt +++ b/app/src/main/java/org/xtimms/shirizu/work/tracker/TrackWorker.kt @@ -8,7 +8,6 @@ import androidx.core.app.NotificationCompat import androidx.core.app.NotificationCompat.VISIBILITY_PUBLIC import androidx.core.app.NotificationCompat.VISIBILITY_SECRET import androidx.core.app.NotificationManagerCompat -import androidx.core.content.edit import androidx.hilt.work.HiltWorker import androidx.work.BackoffPolicy import androidx.work.Constraints @@ -31,22 +30,21 @@ import dagger.Reusable import dagger.assisted.Assisted import dagger.assisted.AssistedInject import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList import kotlinx.coroutines.launch -import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.runCatchingCancellable +import org.koitharu.kotatsu.parsers.util.toIntUp import org.xtimms.shirizu.R +import org.xtimms.shirizu.core.database.ShirizuDatabase import org.xtimms.shirizu.core.exceptions.CloudflareProtectedException import org.xtimms.shirizu.core.logs.FileLogger import org.xtimms.shirizu.core.logs.TrackerLogger @@ -61,6 +59,7 @@ import org.xtimms.shirizu.utils.system.trySetForeground import org.xtimms.shirizu.work.PeriodicWorkScheduler import java.util.concurrent.TimeUnit import javax.inject.Inject +import javax.inject.Provider @HiltWorker class TrackWorker @AssistedInject constructor( @@ -75,10 +74,10 @@ class TrackWorker @AssistedInject constructor( private val notificationManager by lazy { NotificationManagerCompat.from(applicationContext) } override suspend fun doWork(): Result { - trySetForeground() + val isForeground = trySetForeground() logger.log("doWork(): attempt $runAttemptCount") return try { - doWorkImpl() + doWorkImpl(isFullRun = isForeground && TAG_ONESHOT in tags) } catch (e: CancellationException) { throw e } catch (e: Throwable) { @@ -92,49 +91,18 @@ class TrackWorker @AssistedInject constructor( } } - private suspend fun doWorkImpl(): Result { + private suspend fun doWorkImpl(isFullRun: Boolean): Result { if (!AppSettings.isTrackerEnabled()) { return Result.success(workDataOf(0, 0)) } - val retryIds = getRetryIds() - val tracks = if (retryIds.isNotEmpty()) { - tracker.getTracks(retryIds) - } else { - tracker.getAllTracks() - } + val tracks = tracker.getTracks(if (isFullRun) Int.MAX_VALUE else BATCH_SIZE) logger.log("Total ${tracks.size} tracks") if (tracks.isEmpty()) { return Result.success(workDataOf(0, 0)) } - val results = checkUpdatesAsync(tracks) - tracker.gc() - - var success = 0 - var failed = 0 - val retry = HashSet() - results.forEach { x -> - when (x) { - is MangaUpdates.Success -> success++ - is MangaUpdates.Failure -> { - failed++ - if (x.shouldRetry()) { - retry += x.manga.id - } - } - } - } - if (runAttemptCount > MAX_ATTEMPTS) { - retry.clear() - } - setRetryIds(retry) - logger.log("Result: success: $success, failed: $failed, retry: ${retry.size}") - val resultData = workDataOf(success, failed) - return when { - retry.isNotEmpty() -> Result.retry() - success == 0 && failed != 0 -> Result.failure(resultData) - else -> Result.success(resultData) - } + checkUpdatesAsync(tracks) + return Result.success() } private suspend fun checkUpdatesAsync(tracks: List): List { @@ -145,10 +113,13 @@ class TrackWorker @AssistedInject constructor( semaphore.withPermit { send( runCatchingCancellable { - tracker.fetchUpdates(track, commit = true) - .copy(channelId = channelId) - }.onFailure { e -> - logger.log("checkUpdatesAsync", e) + tracker.fetchUpdates(track, commit = true).let { + if (it is MangaUpdates.Success) { + it.copy(channelId = channelId) + } else { + it + } + } }.getOrElse { error -> MangaUpdates.Failure( manga = track.manga, @@ -166,6 +137,7 @@ class TrackWorker @AssistedInject constructor( when (it) { is MangaUpdates.Failure -> { val e = it.error + logger.log("checkUpdatesAsync", e) if (e is CloudflareProtectedException) { e.printStackTrace() } @@ -281,22 +253,6 @@ class TrackWorker @AssistedInject constructor( } }.build() - private suspend fun setRetryIds(ids: Set) = runInterruptible(Dispatchers.IO) { - val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE) - prefs.edit(commit = true) { - if (ids.isEmpty()) { - remove(KEY_RETRY_IDS) - } else { - putStringSet(KEY_RETRY_IDS, ids.mapToSet { it.toString() }) - } - } - } - - private fun getRetryIds(): Set { - val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE) - return prefs.getStringSet(KEY_RETRY_IDS, null)?.mapToSet { it.toLong() }.orEmpty() - } - private fun workDataOf(success: Int, failed: Int): Data { return Data.Builder() .putInt(DATA_KEY_SUCCESS, success) @@ -307,11 +263,15 @@ class TrackWorker @AssistedInject constructor( @Reusable class Scheduler @Inject constructor( private val workManager: WorkManager, + private val dbProvider: Provider, ) : PeriodicWorkScheduler { override suspend fun schedule() { val constraints = createConstraints() - val request = PeriodicWorkRequestBuilder(4, TimeUnit.HOURS) + val runCount = dbProvider.get().getTracksDao().getTracksCount() + val runsPerFullCheck = (runCount / BATCH_SIZE.toFloat()).toIntUp() + val interval = (6 / runsPerFullCheck).coerceAtLeast(2) + val request = PeriodicWorkRequestBuilder(interval.toLong(), TimeUnit.HOURS) .setConstraints(constraints) .addTag(TAG) .setBackoffCriteria(BackoffPolicy.LINEAR, 30, TimeUnit.MINUTES) @@ -363,10 +323,9 @@ class TrackWorker @AssistedInject constructor( const val WORKER_NOTIFICATION_ID = 35 const val TAG = "tracking" const val TAG_ONESHOT = "tracking_oneshot" - const val MAX_PARALLELISM = 3 - const val MAX_ATTEMPTS = 3 + const val MAX_PARALLELISM = 6 const val DATA_KEY_SUCCESS = "success" const val DATA_KEY_FAILED = "failed" - const val KEY_RETRY_IDS = "retry" + const val BATCH_SIZE = 20 } } \ No newline at end of file