Tracker improvements

app/src/main/java/org/xtimms/shirizu/core/database/ShirizuDatabase.kt

@@ -35,9 +35,10 @@ import org.xtimms.shirizu.core.database.entity.TagEntity
 import org.xtimms.shirizu.core.database.entity.TrackEntity
 import org.xtimms.shirizu.core.database.entity.TrackLogEntity
 import org.xtimms.shirizu.core.database.migrations.Migration1To2
+import org.xtimms.shirizu.core.database.migrations.Migration2To3
 import org.xtimms.shirizu.utils.lang.processLifecycleScope
 
-const val DATABASE_VERSION = 2
+const val DATABASE_VERSION = 3
 
 @Database(
     entities = [
@@ -83,7 +84,8 @@ abstract class ShirizuDatabase : RoomDatabase() {
 }
 
 fun getDatabaseMigrations(context: Context): Array<Migration> = arrayOf(
-    Migration1To2()
+    Migration1To2(),
+    Migration2To3()
 )
 
 fun ShirizuDatabase(context: Context): ShirizuDatabase = Room

app/src/main/java/org/xtimms/shirizu/core/database/dao/FavouritesDao.kt

@@ -51,6 +51,9 @@ abstract class FavouritesDao {
     @Query("SELECT * FROM favourites WHERE deleted_at = 0 ORDER BY created_at DESC LIMIT :limit OFFSET :offset")
     abstract suspend fun findAllRaw(offset: Int, limit: Int): List<FavouriteManga>
 
+    @Query("SELECT DISTINCT manga_id FROM favourites WHERE deleted_at = 0 AND category_id IN (SELECT category_id FROM favourite_categories WHERE track = 1)")
+    abstract suspend fun findIdsWithTrack(): LongArray
+
     @Transaction
     @Query(
         "SELECT * FROM favourites WHERE category_id = :categoryId AND deleted_at = 0 " +
@@ -107,6 +110,9 @@ abstract class FavouritesDao {
     @Query("SELECT DISTINCT category_id FROM favourites WHERE manga_id IN (:mangaIds) AND deleted_at = 0")
     abstract suspend fun findCategoriesIds(mangaIds: Collection<Long>): List<Long>
 
+    @Query("SELECT DISTINCT favourite_categories.category_id FROM favourites LEFT JOIN favourite_categories ON favourites.category_id = favourite_categories.category_id WHERE manga_id = :mangaId AND favourites.deleted_at = 0 AND favourite_categories.deleted_at = 0 AND favourite_categories.track = 1")
+    abstract suspend fun findCategoriesIdsWithTrack(mangaId: Long): List<Long>
+
     /** INSERT **/
 
     @Insert(onConflict = OnConflictStrategy.REPLACE)

app/src/main/java/org/xtimms/shirizu/core/database/dao/HistoryDao.kt

@@ -54,6 +54,9 @@ abstract class HistoryDao {
     @Query("SELECT * FROM manga WHERE manga_id IN (SELECT manga_id FROM history WHERE deleted_at = 0)")
     abstract suspend fun findAllManga(): List<MangaEntity>
 
+    @Query("SELECT manga_id FROM history WHERE deleted_at = 0")
+    abstract suspend fun findAllIds(): LongArray
+
     @Query(
         """SELECT tags.* FROM tags
 		LEFT JOIN manga_tags ON tags.tag_id = manga_tags.tag_id
@@ -74,6 +77,9 @@ abstract class HistoryDao {
     @Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0")
     abstract fun observeCount(): Flow<Int>
 
+    @Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0")
+    abstract suspend fun getCount(): Int
+
     @Query("SELECT percent FROM history WHERE manga_id = :id AND deleted_at = 0")
     abstract suspend fun findProgress(id: Long): Float?
 

app/src/main/java/org/xtimms/shirizu/core/database/dao/TrackLogsDao.kt

@@ -28,6 +28,9 @@ interface TrackLogsDao {
     @Query("DELETE FROM track_logs WHERE manga_id NOT IN (SELECT manga_id FROM tracks)")
     suspend fun gc()
 
+    @Query("DELETE FROM track_logs WHERE id IN (SELECT id FROM track_logs ORDER BY created_at DESC LIMIT 0 OFFSET :size)")
+    suspend fun trim(size: Int)
+
     @Query("SELECT COUNT(*) FROM track_logs")
     suspend fun count(): Int
 }

app/src/main/java/org/xtimms/shirizu/core/database/dao/TracksDao.kt

@@ -8,6 +8,7 @@ import androidx.room.Upsert
 import kotlinx.coroutines.flow.Flow
 import org.xtimms.shirizu.core.database.entity.MangaWithTags
 import org.xtimms.shirizu.core.database.entity.TrackEntity
+import org.xtimms.shirizu.core.database.entity.TrackWithManga
 
 @Dao
 abstract class TracksDao {
@@ -15,6 +16,13 @@ abstract class TracksDao {
     @Query("SELECT * FROM tracks")
     abstract suspend fun findAll(): List<TrackEntity>
 
+    @Transaction
+    @Query("SELECT * FROM tracks ORDER BY last_check_time ASC LIMIT :limit OFFSET :offset")
+    abstract suspend fun findAll(offset: Int, limit: Int): List<TrackWithManga>
+
+    @Query("SELECT manga_id FROM tracks")
+    abstract suspend fun findAllIds(): LongArray
+
     @Query("SELECT * FROM tracks WHERE manga_id IN (:ids)")
     abstract suspend fun findAll(ids: Collection<Long>): List<TrackEntity>
 
@@ -24,6 +32,9 @@ abstract class TracksDao {
     @Query("SELECT chapters_new FROM tracks WHERE manga_id = :mangaId")
     abstract suspend fun findNewChapters(mangaId: Long): Int?
 
+    @Query("SELECT COUNT(*) FROM tracks")
+    abstract suspend fun getTracksCount(): Int
+
     @Query("SELECT manga_id, chapters_new FROM tracks")
     abstract fun observeNewChaptersMap(): Flow<Map<@MapColumn(columnName = "manga_id") Long, @MapColumn(
         columnName = "chapters_new"

app/src/main/java/org/xtimms/shirizu/core/database/entity/TrackEntity.kt

@@ -19,11 +19,26 @@ import androidx.room.PrimaryKey
 class TrackEntity(
     @PrimaryKey(autoGenerate = false)
     @ColumnInfo(name = "manga_id") val mangaId: Long,
-    @get:Deprecated(message = "Should not be used", level = DeprecationLevel.WARNING)
-    @ColumnInfo(name = "chapters_total") val totalChapters: Int,
     @ColumnInfo(name = "last_chapter_id") val lastChapterId: Long,
     @ColumnInfo(name = "chapters_new") val newChapters: Int,
-    @ColumnInfo(name = "last_check") val lastCheck: Long,
+    @ColumnInfo(name = "last_check_time") val lastCheckTime: Long,
-    @get:Deprecated(message = "Should not be used", level = DeprecationLevel.WARNING)
+    @ColumnInfo(name = "last_chapter_date") val lastChapterDate: Long,
-    @ColumnInfo(name = "last_notified_id") val lastNotifiedChapterId: Long
+    @ColumnInfo(name = "last_result") val lastResult: Int,
-)
+) {
+
+    companion object {
+        const val RESULT_NONE = 0
+        const val RESULT_HAS_UPDATE = 1
+        const val RESULT_NO_UPDATE = 2
+        const val RESULT_FAILED = 3
+
+        fun create(mangaId: Long) = TrackEntity(
+            mangaId = mangaId,
+            lastChapterId = 0L,
+            newChapters = 0,
+            lastCheckTime = 0L,
+            lastChapterDate = 0,
+            lastResult = RESULT_NONE,
+        )
+    }
+}

app/src/main/java/org/xtimms/shirizu/core/database/entity/TrackLogEntity.kt

@@ -21,5 +21,5 @@ class TrackLogEntity(
     @ColumnInfo(name = "id") val id: Long = 0L,
     @ColumnInfo(name = "manga_id", index = true) val mangaId: Long,
     @ColumnInfo(name = "chapters") val chapters: String,
-    @ColumnInfo(name = "created_at") val createdAt: Long = System.currentTimeMillis(),
+    @ColumnInfo(name = "created_at") val createdAt: Long,
 )

app/src/main/java/org/xtimms/shirizu/core/database/entity/TrackWithManga.kt

@@ -0,0 +1,13 @@
+package org.xtimms.shirizu.core.database.entity
+
+import androidx.room.Embedded
+import androidx.room.Relation
+
+class TrackWithManga(
+    @Embedded val track: TrackEntity,
+    @Relation(
+        parentColumn = "manga_id",
+        entityColumn = "manga_id",
+    )
+    val manga: MangaEntity,
+)

app/src/main/java/org/xtimms/shirizu/core/database/migrations/Migration2To3.kt

@@ -0,0 +1,16 @@
+package org.xtimms.shirizu.core.database.migrations
+
+import androidx.room.migration.Migration
+import androidx.sqlite.db.SupportSQLiteDatabase
+
+class Migration2To3 : Migration(2, 3) {
+
+    override fun migrate(db: SupportSQLiteDatabase) {
+        db.execSQL("CREATE TABLE tracks_bk (manga_id INTEGER NOT NULL, chapters_total INTEGER NOT NULL, last_chapter_id INTEGER NOT NULL, chapters_new INTEGER NOT NULL, last_check INTEGER NOT NULL, last_notified_id INTEGER NOT NULL, PRIMARY KEY(manga_id))")
+        db.execSQL("INSERT INTO tracks_bk SELECT manga_id, chapters_total, last_chapter_id, chapters_new, last_check, last_notified_id FROM tracks")
+        db.execSQL("DROP TABLE tracks")
+        db.execSQL("CREATE TABLE tracks (`manga_id` INTEGER NOT NULL, `last_chapter_id` INTEGER NOT NULL, `chapters_new` INTEGER NOT NULL, `last_check_time` INTEGER NOT NULL, `last_chapter_date` INTEGER NOT NULL, `last_result` INTEGER NOT NULL, PRIMARY KEY(`manga_id`), FOREIGN KEY(`manga_id`) REFERENCES `manga`(`manga_id`) ON UPDATE NO ACTION ON DELETE CASCADE )")
+        db.execSQL("INSERT INTO tracks SELECT manga_id, last_chapter_id, chapters_new, last_check AS last_check_time, 0 AS last_chapter_date, 0 AS last_result FROM tracks_bk")
+        db.execSQL("DROP TABLE tracks_bk")
+    }
+}

app/src/main/java/org/xtimms/shirizu/core/tracker/Tracker.kt

@@ -4,6 +4,7 @@ import androidx.annotation.VisibleForTesting
 import androidx.collection.MutableLongSet
 import coil.request.CachePolicy
 import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
 import org.xtimms.shirizu.core.model.getPreferredBranch
 import org.xtimms.shirizu.core.parser.MangaRepository
 import org.xtimms.shirizu.core.parser.RemoteMangaRepository
@@ -26,47 +27,19 @@ class Tracker @Inject constructor(
     private val mangaRepositoryFactory: MangaRepository.Factory,
 ) {
 
-    suspend fun getAllTracks(): List<TrackingItem> {
+    suspend fun getTracks(limit: Int): List<TrackingItem> {
-        val knownManga = MutableLongSet()
+        repository.updateTracks()
-        val result = ArrayList<TrackingItem>()
+        return repository.getTracks(0, limit).map {
-        // Favourites
+            val categoryId = repository.getCategoryId(it.manga.id)
-        val favourites = repository.getAllFavouritesManga()
+            TrackingItem(
-        channels.updateChannels(favourites.keys)
+                tracking = it,
-        for ((category, mangaList) in favourites) {
+                channelId = if (categoryId == 0L) {
-            if (!category.isTrackingEnabled || mangaList.isEmpty()) {
+                    channels.getHistoryChannelId()
-                continue
+                } else {
-            }
+                    channels.getFavouritesChannelId(categoryId)
-            val categoryTracks = repository.getTracks(mangaList)
+                },
-            val channelId = if (channels.isFavouriteNotificationsEnabled(category)) {
+            )
-                channels.getFavouritesChannelId(category.id)
-            } else {
-                null
-            }
-            for (track in categoryTracks) {
-                if (knownManga.add(track.manga.id)) {
-                    result.add(TrackingItem(track, channelId))
-                }
-            }
         }
-        // History
-        val history = repository.getAllHistoryManga()
-        val historyTracks = repository.getTracks(history)
-        val channelId = if (channels.isHistoryNotificationsEnabled()) {
-            channels.getHistoryChannelId()
-        } else {
-            null
-        }
-        for (track in historyTracks) {
-            if (knownManga.add(track.manga.id)) {
-                result.add(TrackingItem(track, channelId))
-            }
-        }
-        result.trimToSize()
-        return result
-    }
-
-    suspend fun getTracks(ids: Set<Long>): List<TrackingItem> {
-        return getAllTracks().filterTo(ArrayList(ids.size)) { x -> x.tracking.manga.id in ids }
     }
 
     suspend fun gc() {
@@ -76,11 +49,18 @@ class Tracker @Inject constructor(
     suspend fun fetchUpdates(
         track: MangaTracking,
         commit: Boolean
-    ): MangaUpdates.Success = withMangaLock(track.manga.id) {
+    ): MangaUpdates = withMangaLock(track.manga.id) {
-        val repo = mangaRepositoryFactory.create(track.manga.source)
+        val updates = runCatchingCancellable {
-        require(repo is RemoteMangaRepository) { "Repository ${repo.javaClass.simpleName} is not supported" }
+            val repo = mangaRepositoryFactory.create(track.manga.source)
-        val manga = repo.getDetails(track.manga, CachePolicy.WRITE_ONLY)
+            require(repo is RemoteMangaRepository) { "Repository ${repo.javaClass.simpleName} is not supported" }
-        val updates = compare(track, manga, getBranch(manga))
+            val manga = repo.getDetails(track.manga, CachePolicy.WRITE_ONLY)
+            compare(track, manga, getBranch(manga))
+        }.getOrElse { error ->
+            MangaUpdates.Failure(
+                manga = track.manga,
+                error = error
+            )
+        }
         if (commit) {
             repository.saveUpdates(updates)
         }

app/src/main/java/org/xtimms/shirizu/core/tracker/model/MangaUpdates.kt

@@ -3,6 +3,7 @@ package org.xtimms.shirizu.core.tracker.model
 import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaChapter
 import org.xtimms.shirizu.core.exceptions.TooManyRequestExceptions
+import org.xtimms.shirizu.utils.lang.ifZero
 
 sealed interface MangaUpdates {
 
@@ -16,6 +17,15 @@ sealed interface MangaUpdates {
     ) : MangaUpdates {
 
         fun isNotEmpty() = newChapters.isNotEmpty()
+
+        fun lastChapterDate(): Long {
+            val lastChapter = newChapters.lastOrNull()
+            return if (lastChapter == null) {
+                manga.chapters?.lastOrNull()?.uploadDate ?: 0L
+            } else {
+                lastChapter.uploadDate.ifZero { System.currentTimeMillis() }
+            }
+        }
     }
 
     data class Failure(

app/src/main/java/org/xtimms/shirizu/data/repository/HistoryRepository.kt

@@ -21,8 +21,6 @@ import org.xtimms.shirizu.utils.ReversibleHandle
 import org.xtimms.shirizu.utils.lang.mapItems
 import javax.inject.Inject
 
-const val PROGRESS_NONE = -1f
-
 @Reusable
 class HistoryRepository @Inject constructor(
     private val db: ShirizuDatabase,
@@ -34,6 +32,10 @@ class HistoryRepository @Inject constructor(
         return entities.map { it.manga.toManga(it.tags.toMangaTags()) }
     }
 
+    suspend fun getCount(): Int {
+        return db.getHistoryDao().getCount()
+    }
+
     suspend fun getLastOrNull(): Manga? {
         val entity = db.getHistoryDao().findAll(0, 1).firstOrNull() ?: return null
         return entity.manga.toManga(entity.tags.toMangaTags())

app/src/main/java/org/xtimms/shirizu/data/repository/TrackingRepository.kt

@@ -1,7 +1,6 @@
 package org.xtimms.shirizu.data.repository
 
 import androidx.annotation.VisibleForTesting
-import androidx.collection.MutableLongSet
 import androidx.room.withTransaction
 import dagger.Reusable
 import kotlinx.coroutines.ExperimentalCoroutinesApi
@@ -12,7 +11,6 @@ import kotlinx.coroutines.flow.flatMapLatest
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.onStart
 import org.koitharu.kotatsu.parsers.model.Manga
-import org.koitharu.kotatsu.parsers.util.mapToSet
 import org.xtimms.shirizu.core.database.ShirizuDatabase
 import org.xtimms.shirizu.core.database.entity.MangaEntity
 import org.xtimms.shirizu.core.database.entity.TrackEntity
@@ -21,19 +19,23 @@ import org.xtimms.shirizu.core.database.entity.toFavouriteCategory
 import org.xtimms.shirizu.core.database.entity.toManga
 import org.xtimms.shirizu.core.database.entity.toTrackingLogItem
 import org.xtimms.shirizu.core.model.FavouriteCategory
-import org.xtimms.shirizu.core.model.isLocal
 import org.xtimms.shirizu.core.parser.local.LocalMangaRepository
 import org.xtimms.shirizu.core.tracker.model.MangaTracking
 import org.xtimms.shirizu.core.tracker.model.MangaUpdates
 import org.xtimms.shirizu.core.tracker.model.TrackingLogItem
+import org.xtimms.shirizu.utils.lang.ifZero
 import org.xtimms.shirizu.utils.lang.mapItems
-import java.time.Instant
+import org.xtimms.shirizu.utils.lang.toInstantOrNull
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.inject.Inject
 import javax.inject.Provider
 
 private const val NO_ID = 0L
+
+@Deprecated("Use buckets")
 private const val MAX_QUERY_IDS = 100
+private const val MAX_BUCKET_SIZE = 20
+private const val MAX_LOG_SIZE = 120
 
 @Reusable
 class TrackingRepository @Inject constructor(
@@ -66,35 +68,18 @@ class TrackingRepository @Inject constructor(
             .onStart { gcIfNotCalled() }
     }
 
-    suspend fun getTracks(mangaList: Collection<Manga>): List<MangaTracking> {
+    suspend fun getCategoryId(mangaId: Long): Long {
-        val ids = mangaList.mapToSet { it.id }
+        return db.getFavouritesDao().findCategoriesIdsWithTrack(mangaId).firstOrNull() ?: NO_ID
-        val dao = db.getTracksDao()
+    }
-        val tracks = if (ids.size <= MAX_QUERY_IDS) {
+
-            dao.findAll(ids)
+    suspend fun getTracks(offset: Int, limit: Int): List<MangaTracking> {
-        } else {
+        return db.getTracksDao().findAll(offset, limit).map {
-            // TODO split tracks in the worker
+            MangaTracking(
-            ids.windowed(MAX_QUERY_IDS, MAX_QUERY_IDS, true)
+                manga = it.manga.toManga(emptySet()),
-                .flatMap { dao.findAll(it) }
+                lastChapterId = it.track.lastChapterId,
-        }.groupBy { it.mangaId }
+                lastCheck = it.track.lastCheckTime.toInstantOrNull(),
-        val idSet = MutableLongSet(mangaList.size)
-        val result = ArrayList<MangaTracking>(mangaList.size)
-        for (item in mangaList) {
-            val manga = if (item.isLocal) {
-                localMangaRepositoryProvider.get().getRemoteManga(item) ?: continue
-            } else {
-                item
-            }
-            if (!idSet.add(manga.id)) {
-                continue
-            }
-            val track = tracks[manga.id]?.lastOrNull()
-            result += MangaTracking(
-                manga = manga,
-                lastChapterId = track?.lastChapterId ?: NO_ID,
-                lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
             )
         }
-        return result
     }
 
     @VisibleForTesting
@@ -103,7 +88,7 @@ class TrackingRepository @Inject constructor(
         return MangaTracking(
             manga = manga,
             lastChapterId = track?.lastChapterId ?: NO_ID,
-            lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
+            lastCheck = track?.lastCheckTime?.toInstantOrNull(),
         )
     }
 
@@ -133,16 +118,19 @@ class TrackingRepository @Inject constructor(
 
     suspend fun clearCounters() = db.getTracksDao().clearCounters()
 
-    suspend fun gc() {
+    suspend fun gc() = db.withTransaction {
         db.getTracksDao().gc()
-        db.getTrackLogsDao().gc()
+        db.getTrackLogsDao().run {
+            gc()
+            trim(MAX_LOG_SIZE)
+        }
     }
 
-    suspend fun saveUpdates(updates: MangaUpdates.Success) {
+    suspend fun saveUpdates(updates: MangaUpdates) {
         db.withTransaction {
             val track = getOrCreateTrack(updates.manga.id).mergeWith(updates)
             db.getTracksDao().upsert(track)
-            if (updates.isValid && updates.newChapters.isNotEmpty()) {
+            if (updates is MangaUpdates.Success && updates.isValid && updates.newChapters.isNotEmpty()) {
                 updatePercent(updates)
                 val logEntity = TrackLogEntity(
                     mangaId = updates.manga.id,
@@ -174,7 +162,6 @@ class TrackingRepository @Inject constructor(
         val lastChapterId = chapters.lastOrNull()?.id ?: NO_ID
         val entity = TrackEntity(
             mangaId = manga.id,
-            totalChapters = chapters.size,
             lastChapterId = lastChapterId,
             newChapters = when {
                 track.newChapters == 0 -> 0
@@ -182,8 +169,9 @@ class TrackingRepository @Inject constructor(
                 chapterIndex >= lastNewChapterIndex -> chapters.lastIndex - chapterIndex
                 else -> track.newChapters
             },
-            lastCheck = System.currentTimeMillis(),
+            lastCheckTime = System.currentTimeMillis(),
-            lastNotifiedChapterId = lastChapterId,
+            lastChapterDate = maxOf(track.lastChapterDate, chapters.lastOrNull()?.uploadDate ?: 0L),
+            lastResult = track.lastResult,
         )
         db.getTracksDao().upsert(entity)
     }
@@ -204,19 +192,36 @@ class TrackingRepository @Inject constructor(
         }
     }
 
-    suspend fun getAllHistoryManga(): List<Manga> {
+    suspend fun updateTracks() = db.withTransaction {
-        return db.getHistoryDao().findAllManga().toMangaList()
+        val dao = db.getTracksDao()
+        dao.gc()
+        val ids = dao.findAllIds().toMutableSet()
+        val size = ids.size
+        // history
+        val historyIds = db.getHistoryDao().findAllIds()
+        for (mangaId in historyIds) {
+            if (!ids.remove(mangaId)) {
+                dao.upsert(TrackEntity.create(mangaId))
+            }
+        }
+
+        // favourites
+        val favoritesIds = db.getFavouritesDao().findIdsWithTrack()
+        for (mangaId in favoritesIds) {
+            if (!ids.remove(mangaId)) {
+                dao.upsert(TrackEntity.create(mangaId))
+            }
+        }
+
+        // remove unused
+        for (mangaId in ids) {
+            dao.delete(mangaId)
+        }
+        size - ids.size
     }
 
     private suspend fun getOrCreateTrack(mangaId: Long): TrackEntity {
-        return db.getTracksDao().find(mangaId) ?: TrackEntity(
+        return db.getTracksDao().find(mangaId) ?: TrackEntity.create(mangaId)
-            mangaId = mangaId,
-            totalChapters = 0,
-            lastChapterId = 0L,
-            newChapters = 0,
-            lastCheck = 0L,
-            lastNotifiedChapterId = 0L,
-        )
     }
 
     private suspend fun updatePercent(updates: MangaUpdates.Success) {
@@ -234,16 +239,27 @@ class TrackingRepository @Inject constructor(
         db.getHistoryDao().update(history.copy(percent = newPercent))
     }
 
-    private fun TrackEntity.mergeWith(updates: MangaUpdates.Success): TrackEntity {
+    private fun TrackEntity.mergeWith(updates: MangaUpdates): TrackEntity {
         val chapters = updates.manga.chapters.orEmpty()
-        return TrackEntity(
+        return when (updates) {
-            mangaId = mangaId,
+            is MangaUpdates.Failure -> TrackEntity(
-            totalChapters = chapters.size,
+                mangaId = mangaId,
-            lastChapterId = chapters.lastOrNull()?.id ?: NO_ID,
+                lastChapterId = lastChapterId,
-            newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0,
+                newChapters = newChapters,
-            lastCheck = System.currentTimeMillis(),
+                lastCheckTime = System.currentTimeMillis(),
-            lastNotifiedChapterId = NO_ID,
+                lastChapterDate = lastChapterDate,
-        )
+                lastResult = TrackEntity.RESULT_FAILED,
+            )
+
+            is MangaUpdates.Success -> TrackEntity(
+                mangaId = mangaId,
+                lastChapterId = chapters.lastOrNull()?.id ?: NO_ID,
+                newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0,
+                lastCheckTime = System.currentTimeMillis(),
+                lastChapterDate = updates.lastChapterDate().ifZero { lastChapterDate },
+                lastResult = if (updates.isNotEmpty()) TrackEntity.RESULT_HAS_UPDATE else TrackEntity.RESULT_NO_UPDATE,
+            )
+        }
     }
 
     private suspend fun gcIfNotCalled() {

app/src/main/java/org/xtimms/shirizu/sections/settings/shelf/ShelfSettingsView.kt

@@ -14,7 +14,7 @@ import androidx.compose.material.icons.Icons
 import androidx.compose.material.icons.outlined.Category
 import androidx.compose.material.icons.outlined.GridView
 import androidx.compose.material.icons.outlined.Numbers
-import androidx.compose.material.icons.outlined.Update
+import androidx.compose.material.icons.outlined.RssFeed
 import androidx.compose.material3.AlertDialog
 import androidx.compose.material3.ExperimentalMaterial3Api
 import androidx.compose.material3.Icon
@@ -39,13 +39,13 @@ import org.xtimms.shirizu.R
 import org.xtimms.shirizu.core.components.PreferenceItem
 import org.xtimms.shirizu.core.components.PreferenceSubtitle
 import org.xtimms.shirizu.core.components.PreferenceSwitch
+import org.xtimms.shirizu.core.components.PreferenceSwitchWithDivider
 import org.xtimms.shirizu.core.components.ScaffoldWithTopAppBar
 import org.xtimms.shirizu.core.prefs.AppSettings
 import org.xtimms.shirizu.core.prefs.AppSettings.getInt
-import org.xtimms.shirizu.core.prefs.AppSettings.getString
 import org.xtimms.shirizu.core.prefs.GRID_COLUMNS
-import org.xtimms.shirizu.core.prefs.PROXY_ADDRESS
 import org.xtimms.shirizu.core.prefs.TABS_MANGA_COUNT
+import org.xtimms.shirizu.core.prefs.TRACKER
 import org.xtimms.shirizu.sections.shelf.ShelfViewModel
 
 const val SHELF_SETTINGS_DESTINATION = "shelf_settings"
@@ -54,9 +54,11 @@ const val SHELF_SETTINGS_DESTINATION = "shelf_settings"
 fun ShelfSettingsView(
     shelfViewModel: ShelfViewModel = hiltViewModel(),
     navigateBack: () -> Unit,
-    navigateToCategories: () -> Unit
+    navigateToCategories: () -> Unit,
+    navigateToTrackerSettings: () -> Unit = {}
 ) {
 
+    var isTrackerEnabled by remember { mutableStateOf(AppSettings.isTrackerEnabled()) }
     var showGridColumnsDialog by remember { mutableStateOf(false) }
 
     val categories by shelfViewModel.categories.collectAsStateWithLifecycle(emptyList())
@@ -106,7 +108,10 @@ fun ShelfSettingsView(
             item {
                 PreferenceItem(
                     title = stringResource(id = R.string.grid_columns_count),
-                    description = stringResource(id = R.string.grid_columns_count_desc, gridColumns),
+                    description = stringResource(
+                        id = R.string.grid_columns_count_desc,
+                        gridColumns
+                    ),
                     icon = Icons.Outlined.GridView
                 ) { showGridColumnsDialog = true }
             }
@@ -114,10 +119,17 @@ fun ShelfSettingsView(
                 PreferenceSubtitle(text = stringResource(id = R.string.updates))
             }
             item {
-                PreferenceItem(
+                PreferenceSwitchWithDivider(
                     title = stringResource(id = R.string.auto_update),
-                    description = "Off",
+                    description = if (isTrackerEnabled) stringResource(id = R.string.on)
-                    icon = Icons.Outlined.Update
+                    else stringResource(id = R.string.off),
+                    icon = Icons.Outlined.RssFeed,
+                    isChecked = isTrackerEnabled,
+                    onClick = navigateToTrackerSettings,
+                    onChecked = {
+                        isTrackerEnabled = !isTrackerEnabled
+                        AppSettings.updateValue(TRACKER, isTrackerEnabled)
+                    }
                 )
             }
         }

app/src/main/java/org/xtimms/shirizu/utils/lang/Date.kt

@@ -69,6 +69,8 @@ fun LocalDateTime.toDate(): Date = Date(this.toEpochSecond(
     ZoneId.systemDefault().rules.getOffset(this)
 ) * 1000)
 
+fun Long.toInstantOrNull() = if (this == 0L) null else Instant.ofEpochMilli(this)
+
 sealed class DateTimeAgo {
 
     abstract fun format(resources: Resources): String

app/src/main/java/org/xtimms/shirizu/utils/lang/Primitive.kt

@@ -21,5 +21,9 @@ inline val String.intState
         mutableIntStateOf(this.getInt())
     }
 
+inline fun Int.ifZero(defaultValue: () -> Int): Int = if (this == 0) defaultValue() else this
+
+inline fun Long.ifZero(defaultValue: () -> Long): Long = if (this == 0L) defaultValue() else this
+
 // clamp(3.5f, 6.7f) > [0.0f, 1.0f]
 fun Float.clamp(min: Float, max: Float): Float = (1f - ((this.coerceIn(min, max) - min) / (max - min)))

app/src/main/java/org/xtimms/shirizu/work/WorkScheduleManager.kt

@@ -17,17 +17,11 @@ class WorkScheduleManager @Inject constructor(
 
     fun init() {
         processLifecycleScope.launch(Dispatchers.Default) {
-            updateWorkerImpl(trackerScheduler, isEnabled = true, force = false) // TODO
+            updateWorkerImpl(trackerScheduler, AppSettings.isTrackerEnabled(), force = true)
             updateWorkerImpl(suggestionScheduler, AppSettings.isSuggestionsEnabled(), force = false)
         }
     }
 
-    private fun updateWorker(scheduler: PeriodicWorkScheduler, isEnabled: Boolean, force: Boolean) {
-        processLifecycleScope.launch(Dispatchers.Default) {
-            updateWorkerImpl(scheduler, isEnabled, force)
-        }
-    }
-
     private suspend fun updateWorkerImpl(scheduler: PeriodicWorkScheduler, isEnabled: Boolean, force: Boolean) {
         if (force || scheduler.isScheduled() != isEnabled) {
             if (isEnabled) {

app/src/main/java/org/xtimms/shirizu/work/tracker/TrackWorker.kt

@@ -8,7 +8,6 @@ import androidx.core.app.NotificationCompat
 import androidx.core.app.NotificationCompat.VISIBILITY_PUBLIC
 import androidx.core.app.NotificationCompat.VISIBILITY_SECRET
 import androidx.core.app.NotificationManagerCompat
-import androidx.core.content.edit
 import androidx.hilt.work.HiltWorker
 import androidx.work.BackoffPolicy
 import androidx.work.Constraints
@@ -31,22 +30,21 @@ import dagger.Reusable
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedInject
 import kotlinx.coroutines.CancellationException
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.NonCancellable
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.channelFlow
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.runInterruptible
 import kotlinx.coroutines.sync.Semaphore
 import kotlinx.coroutines.sync.withPermit
 import kotlinx.coroutines.withContext
 import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaChapter
-import org.koitharu.kotatsu.parsers.util.mapToSet
 import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
+import org.koitharu.kotatsu.parsers.util.toIntUp
 import org.xtimms.shirizu.R
+import org.xtimms.shirizu.core.database.ShirizuDatabase
 import org.xtimms.shirizu.core.exceptions.CloudflareProtectedException
 import org.xtimms.shirizu.core.logs.FileLogger
 import org.xtimms.shirizu.core.logs.TrackerLogger
@@ -61,6 +59,7 @@ import org.xtimms.shirizu.utils.system.trySetForeground
 import org.xtimms.shirizu.work.PeriodicWorkScheduler
 import java.util.concurrent.TimeUnit
 import javax.inject.Inject
+import javax.inject.Provider
 
 @HiltWorker
 class TrackWorker @AssistedInject constructor(
@@ -75,10 +74,10 @@ class TrackWorker @AssistedInject constructor(
     private val notificationManager by lazy { NotificationManagerCompat.from(applicationContext) }
 
     override suspend fun doWork(): Result {
-        trySetForeground()
+        val isForeground = trySetForeground()
         logger.log("doWork(): attempt $runAttemptCount")
         return try {
-            doWorkImpl()
+            doWorkImpl(isFullRun = isForeground && TAG_ONESHOT in tags)
         } catch (e: CancellationException) {
             throw e
         } catch (e: Throwable) {
@@ -92,49 +91,18 @@ class TrackWorker @AssistedInject constructor(
         }
     }
 
-    private suspend fun doWorkImpl(): Result {
+    private suspend fun doWorkImpl(isFullRun: Boolean): Result {
         if (!AppSettings.isTrackerEnabled()) {
             return Result.success(workDataOf(0, 0))
         }
-        val retryIds = getRetryIds()
+        val tracks = tracker.getTracks(if (isFullRun) Int.MAX_VALUE else BATCH_SIZE)
-        val tracks = if (retryIds.isNotEmpty()) {
-            tracker.getTracks(retryIds)
-        } else {
-            tracker.getAllTracks()
-        }
         logger.log("Total ${tracks.size} tracks")
         if (tracks.isEmpty()) {
             return Result.success(workDataOf(0, 0))
         }
 
-        val results = checkUpdatesAsync(tracks)
+        checkUpdatesAsync(tracks)
-        tracker.gc()
+        return Result.success()
-
-        var success = 0
-        var failed = 0
-        val retry = HashSet<Long>()
-        results.forEach { x ->
-            when (x) {
-                is MangaUpdates.Success -> success++
-                is MangaUpdates.Failure -> {
-                    failed++
-                    if (x.shouldRetry()) {
-                        retry += x.manga.id
-                    }
-                }
-            }
-        }
-        if (runAttemptCount > MAX_ATTEMPTS) {
-            retry.clear()
-        }
-        setRetryIds(retry)
-        logger.log("Result: success: $success, failed: $failed, retry: ${retry.size}")
-        val resultData = workDataOf(success, failed)
-        return when {
-            retry.isNotEmpty() -> Result.retry()
-            success == 0 && failed != 0 -> Result.failure(resultData)
-            else -> Result.success(resultData)
-        }
     }
 
     private suspend fun checkUpdatesAsync(tracks: List<TrackingItem>): List<MangaUpdates> {
@@ -145,10 +113,13 @@ class TrackWorker @AssistedInject constructor(
                     semaphore.withPermit {
                         send(
                             runCatchingCancellable {
-                                tracker.fetchUpdates(track, commit = true)
+                                tracker.fetchUpdates(track, commit = true).let {
-                                    .copy(channelId = channelId)
+                                    if (it is MangaUpdates.Success) {
-                            }.onFailure { e ->
+                                        it.copy(channelId = channelId)
-                                logger.log("checkUpdatesAsync", e)
+                                    } else {
+                                        it
+                                    }
+                                }
                             }.getOrElse { error ->
                                 MangaUpdates.Failure(
                                     manga = track.manga,
@@ -166,6 +137,7 @@ class TrackWorker @AssistedInject constructor(
             when (it) {
                 is MangaUpdates.Failure -> {
                     val e = it.error
+                    logger.log("checkUpdatesAsync", e)
                     if (e is CloudflareProtectedException) {
                         e.printStackTrace()
                     }
@@ -281,22 +253,6 @@ class TrackWorker @AssistedInject constructor(
         }
     }.build()
 
-    private suspend fun setRetryIds(ids: Set<Long>) = runInterruptible(Dispatchers.IO) {
-        val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE)
-        prefs.edit(commit = true) {
-            if (ids.isEmpty()) {
-                remove(KEY_RETRY_IDS)
-            } else {
-                putStringSet(KEY_RETRY_IDS, ids.mapToSet { it.toString() })
-            }
-        }
-    }
-
-    private fun getRetryIds(): Set<Long> {
-        val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE)
-        return prefs.getStringSet(KEY_RETRY_IDS, null)?.mapToSet { it.toLong() }.orEmpty()
-    }
-
     private fun workDataOf(success: Int, failed: Int): Data {
         return Data.Builder()
             .putInt(DATA_KEY_SUCCESS, success)
@@ -307,11 +263,15 @@ class TrackWorker @AssistedInject constructor(
     @Reusable
     class Scheduler @Inject constructor(
         private val workManager: WorkManager,
+        private val dbProvider: Provider<ShirizuDatabase>,
     ) : PeriodicWorkScheduler {
 
         override suspend fun schedule() {
             val constraints = createConstraints()
-            val request = PeriodicWorkRequestBuilder<TrackWorker>(4, TimeUnit.HOURS)
+            val runCount = dbProvider.get().getTracksDao().getTracksCount()
+            val runsPerFullCheck = (runCount / BATCH_SIZE.toFloat()).toIntUp()
+            val interval = (6 / runsPerFullCheck).coerceAtLeast(2)
+            val request = PeriodicWorkRequestBuilder<TrackWorker>(interval.toLong(), TimeUnit.HOURS)
                 .setConstraints(constraints)
                 .addTag(TAG)
                 .setBackoffCriteria(BackoffPolicy.LINEAR, 30, TimeUnit.MINUTES)
@@ -363,10 +323,9 @@ class TrackWorker @AssistedInject constructor(
         const val WORKER_NOTIFICATION_ID = 35
         const val TAG = "tracking"
         const val TAG_ONESHOT = "tracking_oneshot"
-        const val MAX_PARALLELISM = 3
+        const val MAX_PARALLELISM = 6
-        const val MAX_ATTEMPTS = 3
         const val DATA_KEY_SUCCESS = "success"
         const val DATA_KEY_FAILED = "failed"
-        const val KEY_RETRY_IDS = "retry"
+        const val BATCH_SIZE = 20
     }
 }