Improve tracker part 2

app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Date.kt

@@ -18,6 +18,7 @@ fun calculateTimeAgo(instant: Instant, showMonths: Boolean = false): DateTimeAgo
 			if (instant.until(Instant.now(), ChronoUnit.MINUTES) < 3) DateTimeAgo.JustNow
 			else DateTimeAgo.Today
 		}
+
 		diffDays == 1L -> DateTimeAgo.Yesterday
 		diffDays < 6 -> DateTimeAgo.DaysAgo(diffDays.toInt())
 		else -> {
@@ -30,3 +31,5 @@ fun calculateTimeAgo(instant: Instant, showMonths: Boolean = false): DateTimeAgo
 		}
 	}
 }
+
+fun Long.toInstantOrNull() = if (this == 0L) null else Instant.ofEpochMilli(this)

app/src/main/kotlin/org/koitharu/kotatsu/favourites/data/FavouritesDao.kt

@@ -50,6 +50,9 @@ abstract class FavouritesDao {
 	@Query("SELECT * FROM favourites WHERE deleted_at = 0 ORDER BY created_at DESC LIMIT :limit OFFSET :offset")
 	abstract suspend fun findAllRaw(offset: Int, limit: Int): List<FavouriteManga>
 
+	@Query("SELECT DISTINCT manga_id FROM favourites WHERE deleted_at = 0 AND category_id IN (SELECT category_id FROM favourite_categories WHERE track = 1)")
+	abstract suspend fun findIdsWithTrack(): LongArray
+
 	@Transaction
 	@Query(
 		"SELECT * FROM favourites WHERE category_id = :categoryId AND deleted_at = 0 " +
@@ -135,6 +138,9 @@ abstract class FavouritesDao {
 	@Query("SELECT DISTINCT category_id FROM favourites WHERE manga_id IN (:mangaIds) AND deleted_at = 0 ORDER BY favourites.created_at ASC")
 	abstract suspend fun findCategoriesIds(mangaIds: Collection<Long>): List<Long>
 
+	@Query("SELECT DISTINCT favourite_categories.category_id FROM favourites LEFT JOIN favourite_categories ON favourites.category_id = favourite_categories.category_id WHERE manga_id = :mangaId AND favourites.deleted_at = 0 AND favourite_categories.deleted_at = 0 AND favourite_categories.track = 1")
+	abstract suspend fun findCategoriesIdsWithTrack(mangaId: Long): List<Long>
+
 	/** INSERT **/
 
 	@Insert(onConflict = OnConflictStrategy.REPLACE)

app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryDao.kt

@@ -58,6 +58,9 @@ abstract class HistoryDao {
 	@Query("SELECT * FROM manga WHERE manga_id IN (SELECT manga_id FROM history WHERE deleted_at = 0)")
 	abstract suspend fun findAllManga(): List<MangaEntity>
 
+	@Query("SELECT manga_id FROM history WHERE deleted_at = 0")
+	abstract suspend fun findAllIds(): LongArray
+
 	@Query(
 		"""SELECT tags.* FROM tags
 		LEFT JOIN manga_tags ON tags.tag_id = manga_tags.tag_id

app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackEntity.kt

@@ -33,5 +33,14 @@ class TrackEntity(
 		const val RESULT_HAS_UPDATE = 1
 		const val RESULT_NO_UPDATE = 2
 		const val RESULT_FAILED = 3
+
+		fun create(mangaId: Long) = TrackEntity(
+			mangaId = mangaId,
+			lastChapterId = 0L,
+			newChapters = 0,
+			lastCheckTime = 0L,
+			lastChapterDate = 0,
+			lastResult = RESULT_NONE,
+		)
 	}
 }

app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackWithManga.kt

@@ -0,0 +1,14 @@
+package org.koitharu.kotatsu.tracker.data
+
+import androidx.room.Embedded
+import androidx.room.Relation
+import org.koitharu.kotatsu.core.db.entity.MangaEntity
+
+class TrackWithManga(
+	@Embedded val track: TrackEntity,
+	@Relation(
+		parentColumn = "manga_id",
+		entityColumn = "manga_id",
+	)
+	val manga: MangaEntity,
+)

app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TracksDao.kt

@@ -14,6 +14,13 @@ abstract class TracksDao {
 	@Query("SELECT * FROM tracks")
 	abstract suspend fun findAll(): List<TrackEntity>
 
+	@Transaction
+	@Query("SELECT * FROM tracks ORDER BY last_check_time ASC LIMIT :limit OFFSET :offset")
+	abstract suspend fun findAll(offset: Int, limit: Int): List<TrackWithManga>
+
+	@Query("SELECT manga_id FROM tracks")
+	abstract suspend fun findAllIds(): LongArray
+
 	@Query("SELECT * FROM tracks WHERE manga_id IN (:ids)")
 	abstract suspend fun findAll(ids: Collection<Long>): List<TrackEntity>
 

app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/Tracker.kt

@@ -1,7 +1,6 @@
 package org.koitharu.kotatsu.tracker.domain
 
 import androidx.annotation.VisibleForTesting
-import androidx.collection.MutableLongSet
 import coil.request.CachePolicy
 import org.koitharu.kotatsu.core.model.getPreferredBranch
 import org.koitharu.kotatsu.core.parser.MangaRepository
@@ -10,6 +9,7 @@ import org.koitharu.kotatsu.core.prefs.AppSettings
 import org.koitharu.kotatsu.core.util.CompositeMutex2
 import org.koitharu.kotatsu.history.data.HistoryRepository
 import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
 import org.koitharu.kotatsu.tracker.domain.model.MangaTracking
 import org.koitharu.kotatsu.tracker.domain.model.MangaUpdates
 import org.koitharu.kotatsu.tracker.work.TrackerNotificationChannels
@@ -26,56 +26,19 @@ class Tracker @Inject constructor(
 	private val mangaRepositoryFactory: MangaRepository.Factory,
 ) {
 
-	suspend fun getAllTracks(): List<TrackingItem> {
+	suspend fun getTracks(limit: Int): List<TrackingItem> {
-		val sources = settings.trackSources
+		repository.updateTracks()
-		if (sources.isEmpty()) {
+		return repository.getTracks(0, limit).map {
-			return emptyList()
+			val categoryId = repository.getCategoryId(it.manga.id)
-		}
+			TrackingItem(
-		val knownManga = MutableLongSet()
+				tracking = it,
-		val result = ArrayList<TrackingItem>()
+				channelId = if (categoryId == 0L) {
-		// Favourites
-		if (AppSettings.TRACK_FAVOURITES in sources) {
-			val favourites = repository.getAllFavouritesManga()
-			channels.updateChannels(favourites.keys)
-			for ((category, mangaList) in favourites) {
-				if (!category.isTrackingEnabled || mangaList.isEmpty()) {
-					continue
-				}
-				val categoryTracks = repository.getTracks(mangaList)
-				val channelId = if (channels.isFavouriteNotificationsEnabled(category)) {
-					channels.getFavouritesChannelId(category.id)
-				} else {
-					null
-				}
-				for (track in categoryTracks) {
-					if (knownManga.add(track.manga.id)) {
-						result.add(TrackingItem(track, channelId))
-					}
-				}
-			}
-		}
-		// History
-		if (AppSettings.TRACK_HISTORY in sources) {
-			for (i in 0 until historyRepository.getCount() step 20) {
-				val history = historyRepository.getList(i, 20)
-				val historyTracks = repository.getTracks(history)
-				val channelId = if (channels.isHistoryNotificationsEnabled()) {
 					channels.getHistoryChannelId()
 				} else {
-					null
+					channels.getFavouritesChannelId(categoryId)
-				}
+				},
-				for (track in historyTracks) {
+			)
-					if (knownManga.add(track.manga.id)) {
-						result.add(TrackingItem(track, channelId))
-					}
-				}
-			}
-		}
-		return result
 		}
-
-	suspend fun getTracks(ids: Set<Long>): List<TrackingItem> {
-		return getAllTracks().filterTo(ArrayList(ids.size)) { x -> x.tracking.manga.id in ids }
 	}
 
 	suspend fun gc() {
@@ -85,11 +48,18 @@ class Tracker @Inject constructor(
 	suspend fun fetchUpdates(
 		track: MangaTracking,
 		commit: Boolean
-	): MangaUpdates.Success = withMangaLock(track.manga.id) {
+	): MangaUpdates = withMangaLock(track.manga.id) {
+		val updates = runCatchingCancellable {
 			val repo = mangaRepositoryFactory.create(track.manga.source)
 			require(repo is RemoteMangaRepository) { "Repository ${repo.javaClass.simpleName} is not supported" }
 			val manga = repo.getDetails(track.manga, CachePolicy.WRITE_ONLY)
-		val updates = compare(track, manga, getBranch(manga))
+			compare(track, manga, getBranch(manga))
+		}.getOrElse { error ->
+			MangaUpdates.Failure(
+				manga = track.manga,
+				error = error,
+			)
+		}
 		if (commit) {
 			repository.saveUpdates(updates)
 		}

app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/TrackingRepository.kt

@@ -1,7 +1,6 @@
 package org.koitharu.kotatsu.tracker.domain
 
 import androidx.annotation.VisibleForTesting
-import androidx.collection.MutableLongSet
 import androidx.room.withTransaction
 import dagger.Reusable
 import kotlinx.coroutines.flow.Flow
@@ -14,20 +13,19 @@ import org.koitharu.kotatsu.core.db.MangaDatabase
 import org.koitharu.kotatsu.core.db.entity.MangaEntity
 import org.koitharu.kotatsu.core.db.entity.toManga
 import org.koitharu.kotatsu.core.model.FavouriteCategory
-import org.koitharu.kotatsu.core.model.isLocal
+import org.koitharu.kotatsu.core.prefs.AppSettings
 import org.koitharu.kotatsu.core.util.ext.ifZero
 import org.koitharu.kotatsu.core.util.ext.mapItems
+import org.koitharu.kotatsu.core.util.ext.toInstantOrNull
 import org.koitharu.kotatsu.favourites.data.toFavouriteCategory
 import org.koitharu.kotatsu.local.data.LocalMangaRepository
 import org.koitharu.kotatsu.parsers.model.Manga
-import org.koitharu.kotatsu.parsers.util.mapToSet
 import org.koitharu.kotatsu.tracker.data.TrackEntity
 import org.koitharu.kotatsu.tracker.data.TrackLogEntity
 import org.koitharu.kotatsu.tracker.data.toTrackingLogItem
 import org.koitharu.kotatsu.tracker.domain.model.MangaTracking
 import org.koitharu.kotatsu.tracker.domain.model.MangaUpdates
 import org.koitharu.kotatsu.tracker.domain.model.TrackingLogItem
-import java.time.Instant
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.inject.Inject
 import javax.inject.Provider
@@ -42,6 +40,7 @@ private const val MAX_LOG_SIZE = 120
 @Reusable
 class TrackingRepository @Inject constructor(
 	private val db: MangaDatabase,
+	private val settings: AppSettings,
 	private val localMangaRepositoryProvider: Provider<LocalMangaRepository>,
 ) {
 
@@ -70,36 +69,18 @@ class TrackingRepository @Inject constructor(
 			.onStart { gcIfNotCalled() }
 	}
 
-	@Deprecated("")
+	suspend fun getCategoryId(mangaId: Long): Long {
-	suspend fun getTracks(mangaList: Collection<Manga>): List<MangaTracking> {
+		return db.getFavouritesDao().findCategoriesIdsWithTrack(mangaId).firstOrNull() ?: NO_ID
-		val ids = mangaList.mapToSet { it.id }
-		val dao = db.getTracksDao()
-		val tracks = if (ids.size <= MAX_QUERY_IDS) {
-			dao.findAll(ids)
-		} else {
-			// TODO split tracks in the worker
-			ids.windowed(MAX_QUERY_IDS, MAX_QUERY_IDS, true)
-				.flatMap { dao.findAll(it) }
-		}.groupBy { it.mangaId }
-		val idSet = MutableLongSet(mangaList.size)
-		val result = ArrayList<MangaTracking>(mangaList.size)
-		for (item in mangaList) {
-			val manga = if (item.isLocal) {
-				localMangaRepositoryProvider.get().getRemoteManga(item) ?: continue
-			} else {
-				item
-			}
-			if (!idSet.add(manga.id)) {
-				continue
 	}
-			val track = tracks[manga.id]?.lastOrNull()
+
-			result += MangaTracking(
+	suspend fun getTracks(offset: Int, limit: Int): List<MangaTracking> {
-				manga = manga,
+		return db.getTracksDao().findAll(offset, limit).map {
-				lastChapterId = track?.lastChapterId ?: NO_ID,
+			MangaTracking(
-				lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
+				manga = it.manga.toManga(emptySet()),
+				lastChapterId = it.track.lastChapterId,
+				lastCheck = it.track.lastCheckTime.toInstantOrNull(),
 			)
 		}
-		return result
 	}
 
 	@VisibleForTesting
@@ -108,7 +89,7 @@ class TrackingRepository @Inject constructor(
 		return MangaTracking(
 			manga = manga,
 			lastChapterId = track?.lastChapterId ?: NO_ID,
-			lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
+			lastCheck = track?.lastCheckTime?.toInstantOrNull(),
 		)
 	}
 
@@ -145,11 +126,11 @@ class TrackingRepository @Inject constructor(
 		}
 	}
 
-	suspend fun saveUpdates(updates: MangaUpdates.Success) {
+	suspend fun saveUpdates(updates: MangaUpdates) {
 		db.withTransaction {
 			val track = getOrCreateTrack(updates.manga.id).mergeWith(updates)
 			db.getTracksDao().upsert(track)
-			if (updates.isValid && updates.newChapters.isNotEmpty()) {
+			if (updates is MangaUpdates.Success && updates.isValid && updates.newChapters.isNotEmpty()) {
 				updatePercent(updates)
 				val logEntity = TrackLogEntity(
 					mangaId = updates.manga.id,
@@ -211,15 +192,38 @@ class TrackingRepository @Inject constructor(
 		}
 	}
 
+	suspend fun updateTracks() = db.withTransaction {
+		val dao = db.getTracksDao()
+		dao.gc()
+		val ids = dao.findAllIds().toMutableSet()
+		val size = ids.size
+		// history
+		if (AppSettings.TRACK_HISTORY in settings.trackSources) {
+			val historyIds = db.getHistoryDao().findAllIds()
+			for (mangaId in historyIds) {
+				if (!ids.remove(mangaId)) {
+					dao.upsert(TrackEntity.create(mangaId))
+				}
+			}
+		}
+		// favorites
+		if (AppSettings.TRACK_FAVOURITES in settings.trackSources) {
+			val favoritesIds = db.getFavouritesDao().findIdsWithTrack()
+			for (mangaId in favoritesIds) {
+				if (!ids.remove(mangaId)) {
+					dao.upsert(TrackEntity.create(mangaId))
+				}
+			}
+		}
+		// remove unused
+		for (mangaId in ids) {
+			dao.delete(mangaId)
+		}
+		size - ids.size
+	}
+
 	private suspend fun getOrCreateTrack(mangaId: Long): TrackEntity {
-		return db.getTracksDao().find(mangaId) ?: TrackEntity(
+		return db.getTracksDao().find(mangaId) ?: TrackEntity.create(mangaId)
-			mangaId = mangaId,
-			lastChapterId = 0L,
-			newChapters = 0,
-			lastCheckTime = 0L,
-			lastChapterDate = 0,
-			lastResult = TrackEntity.RESULT_NONE,
-		)
 	}
 
 	private suspend fun updatePercent(updates: MangaUpdates.Success) {
@@ -237,9 +241,19 @@ class TrackingRepository @Inject constructor(
 		db.getHistoryDao().update(history.copy(percent = newPercent))
 	}
 
-	private fun TrackEntity.mergeWith(updates: MangaUpdates.Success): TrackEntity {
+	private fun TrackEntity.mergeWith(updates: MangaUpdates): TrackEntity {
 		val chapters = updates.manga.chapters.orEmpty()
-		return TrackEntity(
+		return when (updates) {
+			is MangaUpdates.Failure -> TrackEntity(
+				mangaId = mangaId,
+				lastChapterId = lastChapterId,
+				newChapters = newChapters,
+				lastCheckTime = System.currentTimeMillis(),
+				lastChapterDate = lastChapterDate,
+				lastResult = TrackEntity.RESULT_FAILED,
+			)
+
+			is MangaUpdates.Success -> TrackEntity(
 				mangaId = mangaId,
 				lastChapterId = chapters.lastOrNull()?.id ?: NO_ID,
 				newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0,
@@ -248,6 +262,7 @@ class TrackingRepository @Inject constructor(
 				lastResult = if (updates.isNotEmpty()) TrackEntity.RESULT_HAS_UPDATE else TrackEntity.RESULT_NO_UPDATE,
 			)
 		}
+	}
 
 	private suspend fun gcIfNotCalled() {
 		if (isGcCalled.compareAndSet(false, true)) {

app/src/main/kotlin/org/koitharu/kotatsu/tracker/work/TrackWorker.kt

@@ -11,7 +11,6 @@ import androidx.core.app.NotificationCompat.VISIBILITY_SECRET
 import androidx.core.app.NotificationManagerCompat
 import androidx.core.app.PendingIntentCompat
 import androidx.core.content.ContextCompat
-import androidx.core.content.edit
 import androidx.hilt.work.HiltWorker
 import androidx.work.BackoffPolicy
 import androidx.work.Constraints
@@ -34,14 +33,12 @@ import dagger.Reusable
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedInject
 import kotlinx.coroutines.CancellationException
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.NonCancellable
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.channelFlow
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.runInterruptible
 import kotlinx.coroutines.sync.Semaphore
 import kotlinx.coroutines.sync.withPermit
 import kotlinx.coroutines.withContext
@@ -59,7 +56,6 @@ import org.koitharu.kotatsu.core.util.ext.trySetForeground
 import org.koitharu.kotatsu.details.ui.DetailsActivity
 import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaChapter
-import org.koitharu.kotatsu.parsers.util.mapToSet
 import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
 import org.koitharu.kotatsu.settings.SettingsActivity
 import org.koitharu.kotatsu.settings.work.PeriodicWorkScheduler
@@ -86,7 +82,7 @@ class TrackWorker @AssistedInject constructor(
 		trySetForeground()
 		logger.log("doWork(): attempt $runAttemptCount")
 		return try {
-			doWorkImpl()
+			doWorkImpl(isFullRun = TAG_ONESHOT in tags)
 		} catch (e: CancellationException) {
 			throw e
 		} catch (e: Throwable) {
@@ -100,49 +96,18 @@ class TrackWorker @AssistedInject constructor(
 		}
 	}
 
-	private suspend fun doWorkImpl(): Result {
+	private suspend fun doWorkImpl(isFullRun: Boolean): Result {
 		if (!settings.isTrackerEnabled) {
 			return Result.success(workDataOf(0, 0))
 		}
-		val retryIds = getRetryIds()
+		val tracks = tracker.getTracks(if (isFullRun) Int.MAX_VALUE else BATCH_SIZE)
-		val tracks = if (retryIds.isNotEmpty()) {
-			tracker.getTracks(retryIds)
-		} else {
-			tracker.getAllTracks()
-		}
 		logger.log("Total ${tracks.size} tracks")
 		if (tracks.isEmpty()) {
 			return Result.success(workDataOf(0, 0))
 		}
 
-		val results = checkUpdatesAsync(tracks)
+		checkUpdatesAsync(tracks)
-		tracker.gc()
+		return Result.success()
-
-		var success = 0
-		var failed = 0
-		val retry = HashSet<Long>()
-		results.forEach { x ->
-			when (x) {
-				is MangaUpdates.Success -> success++
-				is MangaUpdates.Failure -> {
-					failed++
-					if (x.shouldRetry()) {
-						retry += x.manga.id
-					}
-				}
-			}
-		}
-		if (runAttemptCount > MAX_ATTEMPTS) {
-			retry.clear()
-		}
-		setRetryIds(retry)
-		logger.log("Result: success: $success, failed: $failed, retry: ${retry.size}")
-		val resultData = workDataOf(success, failed)
-		return when {
-			retry.isNotEmpty() -> Result.retry()
-			success == 0 && failed != 0 -> Result.failure(resultData)
-			else -> Result.success(resultData)
-		}
 	}
 
 	private suspend fun checkUpdatesAsync(tracks: List<TrackingItem>): List<MangaUpdates> {
@@ -153,10 +118,13 @@ class TrackWorker @AssistedInject constructor(
 					semaphore.withPermit {
 						send(
 							runCatchingCancellable {
-								tracker.fetchUpdates(track, commit = true)
+								tracker.fetchUpdates(track, commit = true).let {
-									.copy(channelId = channelId)
+									if (it is MangaUpdates.Success) {
-							}.onFailure { e ->
+										it.copy(channelId = channelId)
-								logger.log("checkUpdatesAsync", e)
+									} else {
+										it
+									}
+								}
 							}.getOrElse { error ->
 								MangaUpdates.Failure(
 									manga = track.manga,
@@ -174,6 +142,7 @@ class TrackWorker @AssistedInject constructor(
 			when (it) {
 				is MangaUpdates.Failure -> {
 					val e = it.error
+					logger.log("checkUpdatesAsync", e)
 					if (e is CloudFlareProtectedException) {
 						CaptchaNotifier(applicationContext).notify(e)
 					}
@@ -323,22 +292,6 @@ class TrackWorker @AssistedInject constructor(
 		)
 	}.build()
 
-	private suspend fun setRetryIds(ids: Set<Long>) = runInterruptible(Dispatchers.IO) {
-		val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE)
-		prefs.edit(commit = true) {
-			if (ids.isEmpty()) {
-				remove(KEY_RETRY_IDS)
-			} else {
-				putStringSet(KEY_RETRY_IDS, ids.mapToSet { it.toString() })
-			}
-		}
-	}
-
-	private fun getRetryIds(): Set<Long> {
-		val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE)
-		return prefs.getStringSet(KEY_RETRY_IDS, null)?.mapToSet { it.toLong() }.orEmpty()
-	}
-
 	private fun workDataOf(success: Int, failed: Int): Data {
 		return Data.Builder()
 			.putInt(DATA_KEY_SUCCESS, success)
@@ -410,6 +363,6 @@ class TrackWorker @AssistedInject constructor(
 		const val MAX_ATTEMPTS = 3
 		const val DATA_KEY_SUCCESS = "success"
 		const val DATA_KEY_FAILED = "failed"
-		const val KEY_RETRY_IDS = "retry"
+		const val BATCH_SIZE = 20
 	}
 }