Mangafire: Get/generate vrf for ajax + Add referer, fix access denied (#2310)

Co-authored-by: Vicente <vicente@per>
Co-authored-by: Draken <131387159+dragonx943@users.noreply.github.com>
Co-authored-by: Koitharu <nvasya95@gmail.com>
Co-authored-by: epikaigle444 <epikaigle444@gmail.com>
Co-authored-by: Max Bethmann <bethibande@gmail.com>
Co-authored-by: vtorres-t <73935362+vtorres-t@users.noreply.github.com>
Co-authored-by: epikaigle444 <epikaigle444@gmail.com>
Co-authored-by: Naga <94557604+NagaYZ@users.noreply.github.com>

src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaFireParser.kt

@@ -14,11 +14,40 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.bitmap.Rect
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.core.PagedMangaParser
-import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.model.ContentRating
-import org.koitharu.kotatsu.parsers.util.*
+import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaListFilter
+import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities
+import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions
+import org.koitharu.kotatsu.parsers.model.MangaPage
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
+import org.koitharu.kotatsu.parsers.model.MangaState
+import org.koitharu.kotatsu.parsers.model.MangaTag
+import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
+import org.koitharu.kotatsu.parsers.model.SortOrder
+import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
+import org.koitharu.kotatsu.parsers.util.generateUid
+import org.koitharu.kotatsu.parsers.util.getCookies
+import org.koitharu.kotatsu.parsers.util.mapChapters
+import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
+import org.koitharu.kotatsu.parsers.util.nullIfEmpty
+import org.koitharu.kotatsu.parsers.util.ownTextOrNull
+import org.koitharu.kotatsu.parsers.util.parseFailed
+import org.koitharu.kotatsu.parsers.util.parseHtml
+import org.koitharu.kotatsu.parsers.util.parseJson
+import org.koitharu.kotatsu.parsers.util.parseSafe
+import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
+import org.koitharu.kotatsu.parsers.util.splitByWhitespace
 import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
+import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.toTitleCase
+import org.koitharu.kotatsu.parsers.util.urlEncoded
 import java.text.SimpleDateFormat
-import java.util.*
+import java.util.Base64
+import java.util.EnumSet
+import java.util.Locale
 import kotlin.math.min
 
 private const val PIECE_SIZE = 200
@@ -41,6 +70,10 @@ internal abstract class MangaFireParser(
 		SortOrder.RELEVANCE,
 	)
 
+    override fun getRequestHeaders() = super.getRequestHeaders().newBuilder()
+        .add("Referer", "https://$domain/")
+        .build()
+
 	override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
 		super.onCreateConfig(keys)
 		keys.add(userAgentKey)
@@ -95,25 +128,30 @@ internal abstract class MangaFireParser(
 			addQueryParameter("page", page.toString())
 			addQueryParameter("language[]", siteLang)
 
-			when {
+            when {
-				!filter.query.isNullOrEmpty() -> {
+                !filter.query.isNullOrEmpty() -> {
-					val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "+") { part ->
+                    val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "+") { part ->
-						part.urlEncoded()
+                        part.urlEncoded()
-					}
+                    }
-					addEncodedQueryParameter("keyword", encodedQuery)
+                    addEncodedQueryParameter("keyword", encodedQuery)
-					addQueryParameter(
+
-						name = "sort",
+                    // Generate VRF for search query
-						value = when (order) {
+                    val searchVrf = VrfGenerator.generate(filter.query.trim())
-							SortOrder.UPDATED -> "recently_updated"
+                    addQueryParameter("vrf", searchVrf)
-							SortOrder.POPULARITY -> "most_viewed"
+
-							SortOrder.RATING -> "scores"
+                    addQueryParameter(
-							SortOrder.NEWEST -> "release_date"
+                        name = "sort",
-							SortOrder.ALPHABETICAL -> "title_az"
+                        value = when (order) {
-							SortOrder.RELEVANCE -> "most_relevance"
+                            SortOrder.UPDATED -> "recently_updated"
-							else -> ""
+                            SortOrder.POPULARITY -> "most_viewed"
-						},
+                            SortOrder.RATING -> "scores"
-					)
+                            SortOrder.NEWEST -> "release_date"
-				}
+                            SortOrder.ALPHABETICAL -> "title_az"
+                            SortOrder.RELEVANCE -> "most_relevance"
+                            else -> ""
+                        },
+                    )
+                }
 
 				else -> {
 					filter.tagsExclude.forEach { tag ->
@@ -260,14 +298,18 @@ internal abstract class MangaFireParser(
 		}
 	}
 
-	private suspend fun getChaptersBranch(mangaId: String, branch: ChapterBranch): List<MangaChapter> {
+    private suspend fun getChaptersBranch(mangaId: String, branch: ChapterBranch): List<MangaChapter> {
-		val chapterElements = webClient
+        val readVrfInput = "$mangaId@${branch.type}@${branch.langCode}"
-			.httpGet("https://$domain/ajax/read/$mangaId/${branch.type}/${branch.langCode}")
+        val readVrf = VrfGenerator.generate(readVrfInput)
-			.parseJson()
+
-			.getJSONObject("result")
+        val response = webClient
-			.getString("html")
+            .httpGet("https://$domain/ajax/read/$mangaId/${branch.type}/${branch.langCode}?vrf=$readVrf")
-			.let(Jsoup::parseBodyFragment)
+
-			.select("ul li a")
+        val chapterElements = response.parseJson()
+            .getJSONObject("result")
+            .getString("html")
+            .let(Jsoup::parseBodyFragment)
+            .select("ul li a")
 
 		if (branch.type == "chapter") {
 			val doc = webClient
@@ -276,31 +318,32 @@ internal abstract class MangaFireParser(
 				.getString("result")
 				.let(Jsoup::parseBodyFragment)
 
-			doc.select("ul li a").withIndex().forEach { (i, it) ->
+            doc.select("ul li a").withIndex().forEach { (i, it) ->
-				val date = it.select("span")[1].ownText()
+                val date = it.select("span").getOrNull(1)?.ownText() ?: ""
-				chapterElements[i].attr("upload-date", date)
+                chapterElements[i].attr("upload-date", date)
-				chapterElements[i].attr("other-title", it.attr("title"))
+                chapterElements[i].attr("other-title", it.attr("title"))
-			}
+            }
-		}
+        }
-
+
-		return chapterElements.mapChapters(reversed = true) { _, it ->
+        return chapterElements.mapChapters(reversed = true) { _, it ->
-			MangaChapter(
+            val chapterId = it.attr("data-id")
-				id = generateUid(it.attr("href")),
+            MangaChapter(
-				title = it.attr("title").ifBlank {
+                id = generateUid(it.attr("href")),
-					"${branch.type.toTitleCase()} ${it.attr("data-number")}"
+                title = it.attr("title").ifBlank {
-				},
+                    "${branch.type.toTitleCase()} ${it.attr("data-number")}"
-				number = it.attr("data-number").toFloat(),
+                },
-				volume = it.attr("other-title").let {
+                number = it.attr("data-number").toFloatOrNull() ?: -1f,
-					volumeNumRegex.find(it)?.groupValues?.getOrNull(2)?.toInt() ?: 0
+                volume = it.attr("other-title").let { title ->
-				},
+                    volumeNumRegex.find(title)?.groupValues?.getOrNull(2)?.toInt() ?: 0
-				url = "${branch.type}/${it.attr("data-id")}",
+                },
-				scanlator = null,
+                url = "$mangaId/${branch.type}/${branch.langCode}/$chapterId",
-				uploadDate = dateFormat.parseSafe(it.attr("upload-date")),
+                scanlator = null,
-				branch = "${branch.langTitle} ${branch.type.toTitleCase()}",
+                uploadDate = dateFormat.parseSafe(it.attr("upload-date")),
-				source = source,
+                branch = "${branch.langTitle} ${branch.type.toTitleCase()}",
-			)
+                source = source,
-		}
+            )
-	}
+        }
+    }
 
 	private val dateFormat = SimpleDateFormat("MMM dd, yyyy", Locale.ENGLISH)
 	private val volumeNumRegex = Regex("""vol(ume)?\s*(\d+)""", RegexOption.IGNORE_CASE)
@@ -387,12 +430,15 @@ internal abstract class MangaFireParser(
 		}
 	}
 
-	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+    override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
-		val images = webClient
+        val chapterId = chapter.url.substringAfterLast('/')
-			.httpGet("https://$domain/ajax/read/${chapter.url}")
+        val vrf = VrfGenerator.generate("chapter@$chapterId")
-			.parseJson()
+
-			.getJSONObject("result")
+        val images = webClient
-			.getJSONArray("images")
+            .httpGet("https://$domain/ajax/read/chapter/$chapterId?vrf=$vrf")
+            .parseJson()
+            .getJSONObject("result")
+            .getJSONArray("images")
 
 		val pages = ArrayList<MangaPage>(images.length())
 
@@ -491,3 +537,186 @@ internal abstract class MangaFireParser(
 	class PortugueseBR(context: MangaLoaderContext) :
 		MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br")
 }
+
+
+private object VrfGenerator {
+    private fun atob(data: String): ByteArray = Base64.getDecoder().decode(data)
+
+    private fun btoa(data: ByteArray): String = Base64.getEncoder().encodeToString(data)
+
+    private fun rc4(key: ByteArray, input: ByteArray): ByteArray {
+        val s = IntArray(256) { it }
+        var j = 0
+
+        // KSA
+        for (i in 0..255) {
+            j = (j + s[i] + key[i % key.size].toInt().and(0xFF)) and 0xFF
+            val temp = s[i]
+            s[i] = s[j]
+            s[j] = temp
+        }
+
+        // PRGA
+        val output = ByteArray(input.size)
+        var i = 0
+        j = 0
+        for (y in input.indices) {
+            i = (i + 1) and 0xFF
+            j = (j + s[i]) and 0xFF
+            val temp = s[i]
+            s[i] = s[j]
+            s[j] = temp
+            val k = s[(s[i] + s[j]) and 0xFF]
+            output[y] = (input[y].toInt() xor k).toByte()
+        }
+        return output
+    }
+
+    private fun transform(
+        input: ByteArray,
+        initSeedBytes: ByteArray,
+        prefixKeyBytes: ByteArray,
+        prefixLen: Int,
+        schedule: List<(Int) -> Int>,
+    ): ByteArray {
+        val out = mutableListOf<Byte>()
+        for (i in input.indices) {
+            if (i < prefixLen) {
+                out.add(prefixKeyBytes[i])
+            }
+            val transformed = schedule[i % 10](
+                (input[i].toInt() xor initSeedBytes[i % 32].toInt()) and 0xFF,
+            ) and 0xFF
+            out.add(transformed.toByte())
+        }
+        return out.toByteArray()
+    }
+
+    private val scheduleC = listOf<(Int) -> Int>(
+        { c -> (c - 48 + 256) and 0xFF },
+        { c -> (c - 19 + 256) and 0xFF },
+        { c -> (c xor 241) and 0xFF },
+        { c -> (c - 19 + 256) and 0xFF },
+        { c -> (c + 223) and 0xFF },
+        { c -> (c - 19 + 256) and 0xFF },
+        { c -> (c - 170 + 256) and 0xFF },
+        { c -> (c - 19 + 256) and 0xFF },
+        { c -> (c - 48 + 256) and 0xFF },
+        { c -> (c xor 8) and 0xFF },
+    )
+
+    private val scheduleY = listOf<(Int) -> Int>(
+        { c -> ((c shl 4) or (c ushr 4)) and 0xFF },
+        { c -> (c + 223) and 0xFF },
+        { c -> ((c shl 4) or (c ushr 4)) and 0xFF },
+        { c -> (c xor 163) and 0xFF },
+        { c -> (c - 48 + 256) and 0xFF },
+        { c -> (c + 82) and 0xFF },
+        { c -> (c + 223) and 0xFF },
+        { c -> (c - 48 + 256) and 0xFF },
+        { c -> (c xor 83) and 0xFF },
+        { c -> ((c shl 4) or (c ushr 4)) and 0xFF },
+    )
+
+    private val scheduleB = listOf<(Int) -> Int>(
+        { c -> (c - 19 + 256) and 0xFF },
+        { c -> (c + 82) and 0xFF },
+        { c -> (c - 48 + 256) and 0xFF },
+        { c -> (c - 170 + 256) and 0xFF },
+        { c -> ((c shl 4) or (c ushr 4)) and 0xFF },
+        { c -> (c - 48 + 256) and 0xFF },
+        { c -> (c - 170 + 256) and 0xFF },
+        { c -> (c xor 8) and 0xFF },
+        { c -> (c + 82) and 0xFF },
+        { c -> (c xor 163) and 0xFF },
+    )
+
+    private val scheduleJ = listOf<(Int) -> Int>(
+        { c -> (c + 223) and 0xFF },
+        { c -> ((c shl 4) or (c ushr 4)) and 0xFF },
+        { c -> (c + 223) and 0xFF },
+        { c -> (c xor 83) and 0xFF },
+        { c -> (c - 19 + 256) and 0xFF },
+        { c -> (c + 223) and 0xFF },
+        { c -> (c - 170 + 256) and 0xFF },
+        { c -> (c + 223) and 0xFF },
+        { c -> (c - 170 + 256) and 0xFF },
+        { c -> (c xor 83) and 0xFF },
+    )
+
+    private val scheduleE = listOf<(Int) -> Int>(
+        { c -> (c + 82) and 0xFF },
+        { c -> (c xor 83) and 0xFF },
+        { c -> (c xor 163) and 0xFF },
+        { c -> (c + 82) and 0xFF },
+        { c -> (c - 170 + 256) and 0xFF },
+        { c -> (c xor 8) and 0xFF },
+        { c -> (c xor 241) and 0xFF },
+        { c -> (c + 82) and 0xFF },
+        { c -> (c + 176) and 0xFF },
+        { c -> ((c shl 4) or (c ushr 4)) and 0xFF },
+    )
+
+    private val rc4Keys = mapOf(
+        "l" to "u8cBwTi1CM4XE3BkwG5Ble3AxWgnhKiXD9Cr279yNW0=",
+        "g" to "t00NOJ/Fl3wZtez1xU6/YvcWDoXzjrDHJLL2r/IWgcY=",
+        "B" to "S7I+968ZY4Fo3sLVNH/ExCNq7gjuOHjSRgSqh6SsPJc=",
+        "m" to "7D4Q8i8dApRj6UWxXbIBEa1UqvjI+8W0UvPH9talJK8=",
+        "F" to "0JsmfWZA1kwZeWLk5gfV5g41lwLL72wHbam5ZPfnOVE=",
+    )
+
+    private val seeds32 = mapOf(
+        "A" to "pGjzSCtS4izckNAOhrY5unJnO2E1VbrU+tXRYG24vTo=",
+        "V" to "dFcKX9Qpu7mt/AD6mb1QF4w+KqHTKmdiqp7penubAKI=",
+        "N" to "owp1QIY/kBiRWrRn9TLN2CdZsLeejzHhfJwdiQMjg3w=",
+        "P" to "H1XbRvXOvZAhyyPaO68vgIUgdAHn68Y6mrwkpIpEue8=",
+        "k" to "2Nmobf/mpQ7+Dxq1/olPSDj3xV8PZkPbKaucJvVckL0=",
+    )
+
+    private val prefixKeys = mapOf(
+        "O" to "Rowe+rg/0g==",
+        "v" to "8cULcnOMJVY8AA==",
+        "L" to "n2+Og2Gth8Hh",
+        "p" to "aRpvzH+yoA==",
+        "W" to "ZB4oBi0=",
+    )
+
+    fun generate(input: String): String {
+        var bytes = input.toByteArray()
+        // RC4 1
+        bytes = rc4(atob(rc4Keys["l"]!!), bytes)
+
+        // Step C1
+        bytes = transform(bytes, atob(seeds32["A"]!!), atob(prefixKeys["O"]!!), 7, scheduleC)
+
+        // RC4 2
+        bytes = rc4(atob(rc4Keys["g"]!!), bytes)
+
+        // Step Y
+        bytes = transform(bytes, atob(seeds32["V"]!!), atob(prefixKeys["v"]!!), 10, scheduleY)
+
+        // RC4 3
+        bytes = rc4(atob(rc4Keys["B"]!!), bytes)
+
+        // Step B
+        bytes = transform(bytes, atob(seeds32["N"]!!), atob(prefixKeys["L"]!!), 9, scheduleB)
+
+        // RC4 4
+        bytes = rc4(atob(rc4Keys["m"]!!), bytes)
+
+        // Step J
+        bytes = transform(bytes, atob(seeds32["P"]!!), atob(prefixKeys["p"]!!), 7, scheduleJ)
+
+        // RC4 5
+        bytes = rc4(atob(rc4Keys["F"]!!), bytes)
+
+        // Step E
+        bytes = transform(bytes, atob(seeds32["k"]!!), atob(prefixKeys["W"]!!), 5, scheduleE)
+
+        // Base64URL encode
+        return btoa(bytes)
+            .replace("+", "-")
+            .replace("/", "_")
+            .replace("=", "")
+    }
+}