From 69e0a531df1636fdffed16152ad4bf7cc3fc2fb3 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 19 Jul 2023 11:47:49 +0300 Subject: [PATCH] [Madara] Refactoring --- .../parsers/site/TuMangaOnlineParser.kt | 4 +- .../parsers/site/madara/MadaraParser.kt | 235 ++++++------------ .../parsers/site/madara/pt/LimaScans.kt | 16 +- .../kotatsu/parsers/util/CryptoAES.kt | 122 +++++---- 4 files changed, 139 insertions(+), 238 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt index d8fe1a9a..1437a3c2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt @@ -95,13 +95,13 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser( oneShotChapterFromElement(item) } } else { - val chapters = mutableListOf() + val chapters = ChaptersListBuilder(10) doc.select(regularChapterListSelector()).reversed().forEachIndexed { i, item -> val chaptername = item.select("div.col-10.text-truncate").text().replace(" ", " ").trim() val scanelement = item.select("ul.chapter-list > li") scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) } } - chapters + chapters.toList() }, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 2e714433..774661fa 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -8,15 +8,12 @@ import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.cryptoaes.CryptoAES import java.text.DateFormat import java.text.SimpleDateFormat import java.util.* - internal abstract class MadaraParser( context: MangaLoaderContext, source: MangaSource, @@ -396,13 +393,11 @@ internal abstract class MadaraParser( val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() - val chapterProtector = doc.selectFirst("#chapter-protector-data") - + val chapterProtector = doc.getElementById("chapter-protector-data") if (chapterProtector == null) { - val root = doc.body().selectFirst("div.main-col-inner")?.selectFirst("div.reading-content") - ?: throw ParseException("Root not found", fullUrl) + val root = doc.body().selectFirstOrThrow("div.main-col-inner").selectFirstOrThrow("div.reading-content") return root.select("div.page-break").map { div -> - val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") + val img = div.selectFirstOrThrow("img") val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") MangaPage( id = generateUid(url), @@ -412,8 +407,6 @@ internal abstract class MadaraParser( ) } } else { - - val chapterProtectorHtml = chapterProtector.html() val password = chapterProtectorHtml.substringAfter("wpmangaprotectornonce='").substringBefore("';") val chapterData = JSONObject( @@ -423,166 +416,74 @@ internal abstract class MadaraParser( val salt = chapterData.getString("s").toString().decodeHex() val ciphertext = SALTED + salt + unsaltedCiphertext - val rawImgArray = CryptoAES.decrypt(Base64.getEncoder().encodeToString(ciphertext), password) - val imgArrayString = rawImgArray - .replace("[", "") - .replace("]", "") - .replace("\\", "") - .replace("\"", "") - + val rawImgArray = CryptoAES(context).decrypt(context.encodeBase64(ciphertext), password) + val imgArrayString = rawImgArray.filterNot { c -> c == '[' || c == ']' || c == '\\' || c == '"' } return imgArrayString.split(",").map { url -> MangaPage( - id = generateUid(url.toString()), - url = url.toString(), + id = generateUid(url), + url = url, preview = null, source = source, ) } } - - - } - - fun String.decodeHex(): ByteArray { - check(length % 2 == 0) { "Must have an even length" } - - return chunked(2) - .map { it.toInt(16).toByte() } - .toByteArray() - } - - companion object { - const val URL_SEARCH_PREFIX = "slug:" - val SALTED = "Salted__".toByteArray(Charsets.UTF_8) } protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - date ?: return 0 + // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it + val d = date?.lowercase() ?: return 0 return when { - date.endsWith(" ago", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Portuguese. - date.endsWith(" atrás", ignoreCase = true) -> { - parseRelativeDate(date) - } - // other translated 'ago' in Portuguese. - date.startsWith("há ", ignoreCase = true) -> { - parseRelativeDate(date) - } - // other translated 'ago' in Spanish - date.endsWith(" hace", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Turkish. - date.endsWith(" önce", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Viêt Nam. - date.endsWith(" trước", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in French. - date.startsWith("il y a", ignoreCase = true) -> { - parseRelativeDate(date) - } - //If there is no ago but just a motion of time - - // short Hours - date.endsWith(" h", ignoreCase = true) -> { - parseRelativeDate(date) - } - // short Day - date.endsWith(" d", ignoreCase = true) -> { - parseRelativeDate(date) - } - - // Day in Portuguese - date.endsWith(" días", ignoreCase = true) -> { - parseRelativeDate(date) - } - - date.endsWith(" día", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Day in French - date.endsWith(" jour", ignoreCase = true) -> { - parseRelativeDate(date) - } - - date.endsWith(" jours", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Hours in Portuguese - date.endsWith(" horas", ignoreCase = true) -> { - parseRelativeDate(date) - } - - date.endsWith(" hora", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Hours in french - date.endsWith(" heure", ignoreCase = true) -> { - parseRelativeDate(date) - } - - date.endsWith(" heures", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Minutes in English - date.endsWith(" mins", ignoreCase = true) -> { - parseRelativeDate(date) - } - - // Minutes in Portuguese - date.endsWith(" minutos", ignoreCase = true) -> { - parseRelativeDate(date) - } - - date.endsWith(" minuto", ignoreCase = true) -> { - parseRelativeDate(date) - } - //Minutes in French - date.endsWith(" minute", ignoreCase = true) -> { - parseRelativeDate(date) - } - - date.endsWith(" minutes", ignoreCase = true) -> { - parseRelativeDate(date) - } + d.endsWith(" ago") || d.endsWith(" atrás") || // Handle translated 'ago' in Portuguese. + d.startsWith("há ") || // other translated 'ago' in Portuguese. + d.endsWith(" hace") || // other translated 'ago' in Spanish + d.endsWith(" önce") || // Handle translated 'ago' in Turkish. + d.endsWith(" trước") || // Handle translated 'ago' in Viêt Nam. + d.startsWith("il y a") || // Handle translated 'ago' in French. + //If there is no ago but just a motion of time + // short Hours + d.endsWith(" h") || + // short Day + d.endsWith(" d") || + // Day in Portuguese + d.endsWith(" días") || d.endsWith(" día") || + // Day in French + d.endsWith(" jour") || d.endsWith(" jours") || + // Hours in Portuguese + d.endsWith(" horas") || d.endsWith(" hora") || + // Hours in french + d.endsWith(" heure") || d.endsWith(" heures") || + // Minutes in English + d.endsWith(" mins") || + // Minutes in Portuguese + d.endsWith(" minutos") || d.endsWith(" minuto") || + //Minutes in French + d.endsWith(" minute") || d.endsWith(" minutes") -> parseRelativeDate(date) // Handle 'yesterday' and 'today', using midnight - date.startsWith("year", ignoreCase = true) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } + d.startsWith("year") -> Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) // yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis - date.startsWith("today", ignoreCase = true) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } + d.startsWith("today") -> Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis - date.contains(Regex("""\d(st|nd|rd|th)""")) -> { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - }.let { dateFormat.tryParse(it.joinToString(" ")) } - } + date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + }.let { dateFormat.tryParse(it.joinToString(" ")) } else -> dateFormat.tryParse(date) } @@ -649,13 +550,27 @@ internal abstract class MadaraParser( return result.ifEmpty { null } } - private fun createRequestTemplate() = - ("action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default").split( - '&', - ).map { - val pos = it.indexOf('=') - it.substring(0, pos) to it.substring(pos + 1) - }.toMutableMap() + private companion object { + + private fun createRequestTemplate() = + ("action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default").split( + '&', + ).map { + val pos = it.indexOf('=') + it.substring(0, pos) to it.substring(pos + 1) + }.toMutableMap() + fun String.decodeHex(): ByteArray { + check(length % 2 == 0) { "Must have an even length" } + + return chunked(2).map { it.toInt(16).toByte() }.toByteArray() + } + const val URL_SEARCH_PREFIX = "slug:" + val SALTED = "Salted__".toByteArray(Charsets.UTF_8) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/LimaScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/LimaScans.kt index f9a83668..e34b2d88 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/LimaScans.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/LimaScans.kt @@ -6,12 +6,7 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.site.madara.MadaraParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.domain -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat @MangaSourceParser("LIMASCANS", "Lima Scans", "pt") @@ -33,8 +28,8 @@ internal class LimaScans(context: MangaLoaderContext) : val dateFormat = SimpleDateFormat(datePattern, sourceLocale) return doc.select(selectchapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") val link = href + stylepage val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectdate)?.text() val name = a.selectFirst("p")?.text() ?: a.ownText() @@ -44,10 +39,7 @@ internal class LimaScans(context: MangaLoaderContext) : name = name, number = i + 1, branch = null, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), + uploadDate = parseChapterDate(dateFormat, dateText), scanlator = null, source = source, ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CryptoAES.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CryptoAES.kt index 9f707593..6f4cd1d9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CryptoAES.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/CryptoAES.kt @@ -1,23 +1,22 @@ -package org.koitharu.kotatsu.parsers.util.cryptoaes +package org.koitharu.kotatsu.parsers.util +import org.koitharu.kotatsu.parsers.MangaLoaderContext import java.security.MessageDigest -import java.util.Arrays -import java.util.Base64 +import java.util.* import javax.crypto.Cipher import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec +private const val HASH_CIPHER = "AES/CBC/PKCS7PADDING" +private const val AES = "AES" +private const val KDF_DIGEST = "MD5" + /** * Conforming with CryptoJS AES method */ -@Suppress("unused", "FunctionName") -object CryptoAES { - - private const val KEY_SIZE = 256 - private const val IV_SIZE = 128 - private const val HASH_CIPHER = "AES/CBC/PKCS7PADDING" - private const val AES = "AES" - private const val KDF_DIGEST = "MD5" +class CryptoAES( + private val context: MangaLoaderContext, +) { /** * Decrypt using CryptoJS defaults compatible method. @@ -27,19 +26,18 @@ object CryptoAES { * @param cipherText base64 encoded ciphertext * @param password passphrase */ + @Throws(Exception::class) fun decrypt(cipherText: String, password: String): String { - try { - val ctBytes = Base64.getDecoder().decode(cipherText) - val saltBytes = Arrays.copyOfRange(ctBytes, 8, 16) - val cipherTextBytes = Arrays.copyOfRange(ctBytes, 16, ctBytes.size) - val md5: MessageDigest = MessageDigest.getInstance("MD5") - val keyAndIV = generateKeyAndIV(32, 16, 1, saltBytes, password.toByteArray(Charsets.UTF_8), md5) - return decryptAES(cipherTextBytes, - keyAndIV?.get(0) ?: ByteArray(32), - keyAndIV?.get(1) ?: ByteArray(16)) - } catch (e: Exception) { - return "" - } + val ctBytes = context.decodeBase64(cipherText) + val saltBytes = Arrays.copyOfRange(ctBytes, 8, 16) + val cipherTextBytes = Arrays.copyOfRange(ctBytes, 16, ctBytes.size) + val md5: MessageDigest = MessageDigest.getInstance(KDF_DIGEST) + val keyAndIV = generateKeyAndIV(32, 16, 1, saltBytes, password.toByteArray(Charsets.UTF_8), md5) + return decryptAES( + cipherTextBytes, + keyAndIV.getOrNull(0) ?: ByteArray(32), + keyAndIV.getOrNull(1) ?: ByteArray(16), + ) } /** @@ -49,13 +47,10 @@ object CryptoAES { * @param keyBytes key as a bytearray * @param ivBytes iv as a bytearray */ + @Throws(Exception::class) fun decrypt(cipherText: String, keyBytes: ByteArray, ivBytes: ByteArray): String { - return try { - val cipherTextBytes = Base64.getDecoder().decode(cipherText) - decryptAES(cipherTextBytes, keyBytes, ivBytes) - } catch (e: Exception) { - "" - } + val cipherTextBytes = context.decodeBase64(cipherText) + return decryptAES(cipherTextBytes, keyBytes, ivBytes) } /** @@ -65,15 +60,12 @@ object CryptoAES { * @param keyBytes key as a bytearray * @param ivBytes iv as a bytearray */ + @Throws(Exception::class) private fun decryptAES(cipherTextBytes: ByteArray, keyBytes: ByteArray, ivBytes: ByteArray): String { - return try { - val cipher = Cipher.getInstance(HASH_CIPHER) - val keyS = SecretKeySpec(keyBytes, AES) - cipher.init(Cipher.DECRYPT_MODE, keyS, IvParameterSpec(ivBytes)) - cipher.doFinal(cipherTextBytes).toString(Charsets.UTF_8) - } catch (e: Exception) { - "" - } + val cipher = Cipher.getInstance(HASH_CIPHER) + val keyS = SecretKeySpec(keyBytes, AES) + cipher.init(Cipher.DECRYPT_MODE, keyS, IvParameterSpec(ivBytes)) + return cipher.doFinal(cipherTextBytes).toString(Charsets.UTF_8) } /** @@ -92,41 +84,43 @@ object CryptoAES { * @param md the message digest algorithm to use * @return an two-element array with the generated key and IV */ - private fun generateKeyAndIV(keyLength: Int, ivLength: Int, iterations: Int, salt: ByteArray, password: ByteArray, md: MessageDigest): Array? { + @Suppress("SameParameterValue") + @Throws(Exception::class) + private fun generateKeyAndIV( + keyLength: Int, + ivLength: Int, + iterations: Int, + salt: ByteArray?, + password: ByteArray, + md: MessageDigest, + ): Array { val digestLength = md.digestLength val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength val generatedData = ByteArray(requiredLength) var generatedLength = 0 - return try { - md.reset() + md.reset() - // Repeat process until sufficient data has been generated - while (generatedLength < keyLength + ivLength) { + // Repeat process until sufficient data has been generated + while (generatedLength < keyLength + ivLength) { - // Digest data (last digest if available, password data, salt if available) - if (generatedLength > 0) md.update(generatedData, generatedLength - digestLength, digestLength) - md.update(password) - if (salt != null) md.update(salt, 0, 8) - md.digest(generatedData, generatedLength, digestLength) + // Digest data (last digest if available, password data, salt if available) + if (generatedLength > 0) md.update(generatedData, generatedLength - digestLength, digestLength) + md.update(password) + if (salt != null) md.update(salt, 0, 8) + md.digest(generatedData, generatedLength, digestLength) - // additional rounds - for (i in 1 until iterations) { - md.update(generatedData, generatedLength, digestLength) - md.digest(generatedData, generatedLength, digestLength) - } - generatedLength += digestLength + // additional rounds + for (i in 1 until iterations) { + md.update(generatedData, generatedLength, digestLength) + md.digest(generatedData, generatedLength, digestLength) } - - // Copy key and IV into separate byte arrays - val result = arrayOfNulls(2) - result[0] = generatedData.copyOfRange(0, keyLength) - if (ivLength > 0) result[1] = generatedData.copyOfRange(keyLength, keyLength + ivLength) - result - } catch (e: Exception) { - throw e - } finally { - // Clean out temporary data - Arrays.fill(generatedData, 0.toByte()) + generatedLength += digestLength } + + // Copy key and IV into separate byte arrays + val result = arrayOfNulls(2) + result[0] = generatedData.copyOfRange(0, keyLength) + if (ivLength > 0) result[1] = generatedData.copyOfRange(keyLength, keyLength + ivLength) + return result } }