Improve tracker part 1

app/src/main/kotlin/org/koitharu/kotatsu/core/db/MangaDatabase.kt

@@ -31,6 +31,7 @@ import org.koitharu.kotatsu.core.db.migrations.Migration15To16
 import org.koitharu.kotatsu.core.db.migrations.Migration16To17
 import org.koitharu.kotatsu.core.db.migrations.Migration17To18
 import org.koitharu.kotatsu.core.db.migrations.Migration18To19
+import org.koitharu.kotatsu.core.db.migrations.Migration19To20
 import org.koitharu.kotatsu.core.db.migrations.Migration1To2
 import org.koitharu.kotatsu.core.db.migrations.Migration2To3
 import org.koitharu.kotatsu.core.db.migrations.Migration3To4
@@ -57,7 +58,7 @@ import org.koitharu.kotatsu.tracker.data.TrackEntity
 import org.koitharu.kotatsu.tracker.data.TrackLogEntity
 import org.koitharu.kotatsu.tracker.data.TracksDao
 
-const val DATABASE_VERSION = 19
+const val DATABASE_VERSION = 20
 
 @Database(
 	entities = [
@@ -116,6 +117,7 @@ fun getDatabaseMigrations(context: Context): Array<Migration> = arrayOf(
 	Migration16To17(context),
 	Migration17To18(),
 	Migration18To19(),
+	Migration19To20(),
 )
 
 fun MangaDatabase(context: Context): MangaDatabase = Room

app/src/main/kotlin/org/koitharu/kotatsu/core/db/dao/TrackLogsDao.kt

@@ -1,6 +1,10 @@
 package org.koitharu.kotatsu.core.db.dao
 
-import androidx.room.*
+import androidx.room.Dao
+import androidx.room.Insert
+import androidx.room.OnConflictStrategy
+import androidx.room.Query
+import androidx.room.Transaction
 import kotlinx.coroutines.flow.Flow
 import org.koitharu.kotatsu.tracker.data.TrackLogEntity
 import org.koitharu.kotatsu.tracker.data.TrackLogWithManga
@@ -24,6 +28,9 @@ interface TrackLogsDao {
 	@Query("DELETE FROM track_logs WHERE manga_id NOT IN (SELECT manga_id FROM tracks)")
 	suspend fun gc()
 
+	@Query("DELETE FROM track_logs WHERE id IN (SELECT id FROM track_logs ORDER BY created_at DESC LIMIT 0 OFFSET :size)")
+	suspend fun trim(size: Int)
+
 	@Query("SELECT COUNT(*) FROM track_logs")
 	suspend fun count(): Int
 }

app/src/main/kotlin/org/koitharu/kotatsu/core/db/migrations/Migration19To20.kt

@@ -0,0 +1,16 @@
+package org.koitharu.kotatsu.core.db.migrations
+
+import androidx.room.migration.Migration
+import androidx.sqlite.db.SupportSQLiteDatabase
+
+class Migration19To20 : Migration(19, 20) {
+
+	override fun migrate(db: SupportSQLiteDatabase) {
+		db.execSQL("CREATE TABLE tracks_bk (manga_id INTEGER NOT NULL, chapters_total INTEGER NOT NULL, last_chapter_id INTEGER NOT NULL, chapters_new INTEGER NOT NULL, last_check INTEGER NOT NULL, last_notified_id INTEGER NOT NULL, PRIMARY KEY(manga_id))")
+		db.execSQL("INSERT INTO tracks_bk SELECT manga_id, chapters_total, last_chapter_id, chapters_new, last_check, last_notified_id FROM tracks")
+		db.execSQL("DROP TABLE tracks")
+		db.execSQL("CREATE TABLE tracks (`manga_id` INTEGER NOT NULL, `last_chapter_id` INTEGER NOT NULL, `chapters_new` INTEGER NOT NULL, `last_check_time` INTEGER NOT NULL, `last_chapter_date` INTEGER NOT NULL, `last_result` INTEGER NOT NULL, PRIMARY KEY(`manga_id`), FOREIGN KEY(`manga_id`) REFERENCES `manga`(`manga_id`) ON UPDATE NO ACTION ON DELETE CASCADE )")
+		db.execSQL("INSERT INTO tracks SELECT manga_id, last_chapter_id, chapters_new, last_check AS last_check_time, 0 AS last_chapter_date, 0 AS last_result FROM tracks_bk")
+		db.execSQL("DROP TABLE tracks_bk")
+	}
+}

app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Primitive.kt

@@ -2,6 +2,8 @@ package org.koitharu.kotatsu.core.util.ext
 
 inline fun Int.ifZero(defaultValue: () -> Int): Int = if (this == 0) defaultValue() else this
 
+inline fun Long.ifZero(defaultValue: () -> Long): Long = if (this == 0L) defaultValue() else this
+
 fun longOf(a: Int, b: Int): Long {
 	return a.toLong() shl 32 or (b.toLong() and 0xffffffffL)
 }

app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryDao.kt

@@ -78,6 +78,9 @@ abstract class HistoryDao {
 	@Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0")
 	abstract fun observeCount(): Flow<Int>
 
+	@Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0")
+	abstract suspend fun getCount(): Int
+
 	@Query("SELECT percent FROM history WHERE manga_id = :id AND deleted_at = 0")
 	abstract suspend fun findProgress(id: Long): Float?
 

app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryRepository.kt

@@ -45,6 +45,10 @@ class HistoryRepository @Inject constructor(
 		return entities.map { it.manga.toManga(it.tags.toMangaTags()) }
 	}
 
+	suspend fun getCount(): Int {
+		return db.getHistoryDao().getCount()
+	}
+
 	suspend fun getLastOrNull(): Manga? {
 		val entity = db.getHistoryDao().findAll(0, 1).firstOrNull() ?: return null
 		return entity.manga.toManga(entity.tags.toMangaTags())

app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackEntity.kt

@@ -20,11 +20,18 @@ import org.koitharu.kotatsu.core.db.entity.MangaEntity
 class TrackEntity(
 	@PrimaryKey(autoGenerate = false)
 	@ColumnInfo(name = "manga_id") val mangaId: Long,
-	@get:Deprecated(message = "Should not be used", level = DeprecationLevel.WARNING)
-	@ColumnInfo(name = "chapters_total") val totalChapters: Int,
 	@ColumnInfo(name = "last_chapter_id") val lastChapterId: Long,
 	@ColumnInfo(name = "chapters_new") val newChapters: Int,
-	@ColumnInfo(name = "last_check") val lastCheck: Long,
+	@ColumnInfo(name = "last_check_time") val lastCheckTime: Long,
-	@get:Deprecated(message = "Should not be used", level = DeprecationLevel.WARNING)
+	@ColumnInfo(name = "last_chapter_date") val lastChapterDate: Long,
-	@ColumnInfo(name = "last_notified_id") val lastNotifiedChapterId: Long
+	@ColumnInfo(name = "last_result") val lastResult: Int,
-)
+) {
+
+	companion object {
+
+		const val RESULT_NONE = 0
+		const val RESULT_HAS_UPDATE = 1
+		const val RESULT_NO_UPDATE = 2
+		const val RESULT_FAILED = 3
+	}
+}

app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackLogEntity.kt

@@ -13,14 +13,14 @@ import org.koitharu.kotatsu.core.db.entity.MangaEntity
 			entity = MangaEntity::class,
 			parentColumns = ["manga_id"],
 			childColumns = ["manga_id"],
-			onDelete = ForeignKey.CASCADE
+			onDelete = ForeignKey.CASCADE,
-		)
+		),
-	]
+	],
 )
 class TrackLogEntity(
 	@PrimaryKey(autoGenerate = true)
 	@ColumnInfo(name = "id") val id: Long = 0L,
 	@ColumnInfo(name = "manga_id", index = true) val mangaId: Long,
 	@ColumnInfo(name = "chapters") val chapters: String,
-	@ColumnInfo(name = "created_at") val createdAt: Long = System.currentTimeMillis(),
+	@ColumnInfo(name = "created_at") val createdAt: Long,
 )

app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/Tracker.kt

@@ -56,20 +56,21 @@ class Tracker @Inject constructor(
 		}
 		// History
 		if (AppSettings.TRACK_HISTORY in sources) {
-			val history = repository.getAllHistoryManga()
+			for (i in 0 until historyRepository.getCount() step 20) {
-			val historyTracks = repository.getTracks(history)
+				val history = historyRepository.getList(i, 20)
-			val channelId = if (channels.isHistoryNotificationsEnabled()) {
+				val historyTracks = repository.getTracks(history)
-				channels.getHistoryChannelId()
+				val channelId = if (channels.isHistoryNotificationsEnabled()) {
-			} else {
+					channels.getHistoryChannelId()
-				null
+				} else {
-			}
+					null
-			for (track in historyTracks) {
+				}
-				if (knownManga.add(track.manga.id)) {
+				for (track in historyTracks) {
-					result.add(TrackingItem(track, channelId))
+					if (knownManga.add(track.manga.id)) {
+						result.add(TrackingItem(track, channelId))
+					}
 				}
 			}
 		}
-		result.trimToSize()
 		return result
 	}
 

app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/TrackingRepository.kt

@@ -15,6 +15,7 @@ import org.koitharu.kotatsu.core.db.entity.MangaEntity
 import org.koitharu.kotatsu.core.db.entity.toManga
 import org.koitharu.kotatsu.core.model.FavouriteCategory
 import org.koitharu.kotatsu.core.model.isLocal
+import org.koitharu.kotatsu.core.util.ext.ifZero
 import org.koitharu.kotatsu.core.util.ext.mapItems
 import org.koitharu.kotatsu.favourites.data.toFavouriteCategory
 import org.koitharu.kotatsu.local.data.LocalMangaRepository
@@ -32,7 +33,11 @@ import javax.inject.Inject
 import javax.inject.Provider
 
 private const val NO_ID = 0L
+
+@Deprecated("Use buckets")
 private const val MAX_QUERY_IDS = 100
+private const val MAX_BUCKET_SIZE = 20
+private const val MAX_LOG_SIZE = 120
 
 @Reusable
 class TrackingRepository @Inject constructor(
@@ -65,6 +70,7 @@ class TrackingRepository @Inject constructor(
 			.onStart { gcIfNotCalled() }
 	}
 
+	@Deprecated("")
 	suspend fun getTracks(mangaList: Collection<Manga>): List<MangaTracking> {
 		val ids = mangaList.mapToSet { it.id }
 		val dao = db.getTracksDao()
@@ -90,7 +96,7 @@ class TrackingRepository @Inject constructor(
 			result += MangaTracking(
 				manga = manga,
 				lastChapterId = track?.lastChapterId ?: NO_ID,
-				lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
+				lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
 			)
 		}
 		return result
@@ -102,7 +108,7 @@ class TrackingRepository @Inject constructor(
 		return MangaTracking(
 			manga = manga,
 			lastChapterId = track?.lastChapterId ?: NO_ID,
-			lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
+			lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
 		)
 	}
 
@@ -131,9 +137,12 @@ class TrackingRepository @Inject constructor(
 
 	suspend fun clearCounters() = db.getTracksDao().clearCounters()
 
-	suspend fun gc() {
+	suspend fun gc() = db.withTransaction {
 		db.getTracksDao().gc()
-		db.getTrackLogsDao().gc()
+		db.getTrackLogsDao().run {
+			gc()
+			trim(MAX_LOG_SIZE)
+		}
 	}
 
 	suspend fun saveUpdates(updates: MangaUpdates.Success) {
@@ -172,7 +181,6 @@ class TrackingRepository @Inject constructor(
 		val lastChapterId = chapters.lastOrNull()?.id ?: NO_ID
 		val entity = TrackEntity(
 			mangaId = manga.id,
-			totalChapters = chapters.size,
 			lastChapterId = lastChapterId,
 			newChapters = when {
 				track.newChapters == 0 -> 0
@@ -180,8 +188,9 @@ class TrackingRepository @Inject constructor(
 				chapterIndex >= lastNewChapterIndex -> chapters.lastIndex - chapterIndex
 				else -> track.newChapters
 			},
-			lastCheck = System.currentTimeMillis(),
+			lastCheckTime = System.currentTimeMillis(),
-			lastNotifiedChapterId = lastChapterId,
+			lastChapterDate = maxOf(track.lastChapterDate, chapters.lastOrNull()?.uploadDate ?: 0L),
+			lastResult = track.lastResult,
 		)
 		db.getTracksDao().upsert(entity)
 	}
@@ -202,18 +211,14 @@ class TrackingRepository @Inject constructor(
 		}
 	}
 
-	suspend fun getAllHistoryManga(): List<Manga> {
-		return db.getHistoryDao().findAllManga().toMangaList()
-	}
-
 	private suspend fun getOrCreateTrack(mangaId: Long): TrackEntity {
 		return db.getTracksDao().find(mangaId) ?: TrackEntity(
 			mangaId = mangaId,
-			totalChapters = 0,
 			lastChapterId = 0L,
 			newChapters = 0,
-			lastCheck = 0L,
+			lastCheckTime = 0L,
-			lastNotifiedChapterId = 0L,
+			lastChapterDate = 0,
+			lastResult = TrackEntity.RESULT_NONE,
 		)
 	}
 
@@ -236,11 +241,11 @@ class TrackingRepository @Inject constructor(
 		val chapters = updates.manga.chapters.orEmpty()
 		return TrackEntity(
 			mangaId = mangaId,
-			totalChapters = chapters.size,
 			lastChapterId = chapters.lastOrNull()?.id ?: NO_ID,
 			newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0,
-			lastCheck = System.currentTimeMillis(),
+			lastCheckTime = System.currentTimeMillis(),
-			lastNotifiedChapterId = NO_ID,
+			lastChapterDate = updates.lastChapterDate().ifZero { lastChapterDate },
+			lastResult = if (updates.isNotEmpty()) TrackEntity.RESULT_HAS_UPDATE else TrackEntity.RESULT_NO_UPDATE,
 		)
 	}
 

app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/model/MangaUpdates.kt

@@ -1,6 +1,7 @@
 package org.koitharu.kotatsu.tracker.domain.model
 
 import org.koitharu.kotatsu.core.exceptions.TooManyRequestExceptions
+import org.koitharu.kotatsu.core.util.ext.ifZero
 import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaChapter
 
@@ -16,6 +17,15 @@ sealed interface MangaUpdates {
 	) : MangaUpdates {
 
 		fun isNotEmpty() = newChapters.isNotEmpty()
+
+		fun lastChapterDate(): Long {
+			val lastChapter = newChapters.lastOrNull()
+			return if (lastChapter == null) {
+				manga.chapters?.lastOrNull()?.uploadDate ?: 0L
+			} else {
+				lastChapter.uploadDate.ifZero { System.currentTimeMillis() }
+			}
+		}
 	}
 
 	data class Failure(