KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Madara] Refactoring

Browse Source
pull/217/head
Koitharu 3 years ago
parent c85101991f
commit 69e0a531df
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7
4 changed files with 139 additions and 238 deletions
Show Stats Download Patch File Download Diff File

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt
Unescape Escape View File

@ -95,13 +95,13 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser(
oneShotChapterFromElement(item) oneShotChapterFromElement(item)
} }
} else { } else {
val chapters = mutableListOf<MangaChapter>() val chapters = ChaptersListBuilder(10)
doc.select(regularChapterListSelector()).reversed().forEachIndexed { i, item -> doc.select(regularChapterListSelector()).reversed().forEachIndexed { i, item ->
val chaptername = item.select("div.col-10.text-truncate").text().replace("&nbsp;", " ").trim() val chaptername = item.select("div.col-10.text-truncate").text().replace("&nbsp;", " ").trim()
val scanelement = item.select("ul.chapter-list > li") val scanelement = item.select("ul.chapter-list > li")
scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) } scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) }
} }
chapters chapters.toList()
}, },
) )
} }

235
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt
Unescape Escape View File

@ -8,15 +8,12 @@ import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.cryptoaes.CryptoAES
import java.text.DateFormat import java.text.DateFormat
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
internal abstract class MadaraParser( internal abstract class MadaraParser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaSource, source: MangaSource,
@ -396,13 +393,11 @@ internal abstract class MadaraParser(
val fullUrl = chapter.url.toAbsoluteUrl(domain) val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml() val doc = webClient.httpGet(fullUrl).parseHtml()
val chapterProtector = doc.selectFirst("#chapter-protector-data") val chapterProtector = doc.getElementById("chapter-protector-data")
if (chapterProtector == null) { if (chapterProtector == null) {
val root = doc.body().selectFirst("div.main-col-inner")?.selectFirst("div.reading-content") val root = doc.body().selectFirstOrThrow("div.main-col-inner").selectFirstOrThrow("div.reading-content")
?: throw ParseException("Root not found", fullUrl)
return root.select("div.page-break").map { div -> return root.select("div.page-break").map { div ->
val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") val img = div.selectFirstOrThrow("img")
val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found")
MangaPage( MangaPage(
id = generateUid(url), id = generateUid(url),
@ -412,8 +407,6 @@ internal abstract class MadaraParser(
) )
} }
} else { } else {
val chapterProtectorHtml = chapterProtector.html() val chapterProtectorHtml = chapterProtector.html()
val password = chapterProtectorHtml.substringAfter("wpmangaprotectornonce='").substringBefore("';") val password = chapterProtectorHtml.substringAfter("wpmangaprotectornonce='").substringBefore("';")
val chapterData = JSONObject( val chapterData = JSONObject(
@ -423,166 +416,74 @@ internal abstract class MadaraParser(
val salt = chapterData.getString("s").toString().decodeHex() val salt = chapterData.getString("s").toString().decodeHex()
val ciphertext = SALTED + salt + unsaltedCiphertext val ciphertext = SALTED + salt + unsaltedCiphertext
val rawImgArray = CryptoAES.decrypt(Base64.getEncoder().encodeToString(ciphertext), password) val rawImgArray = CryptoAES(context).decrypt(context.encodeBase64(ciphertext), password)
val imgArrayString = rawImgArray val imgArrayString = rawImgArray.filterNot { c -> c == '[' || c == ']' || c == '\\' || c == '"' }
.replace("[", "")
.replace("]", "")
.replace("\\", "")
.replace("\"", "")
return imgArrayString.split(",").map { url -> return imgArrayString.split(",").map { url ->
MangaPage( MangaPage(
id = generateUid(url.toString()), id = generateUid(url),
url = url.toString(), url = url,
preview = null, preview = null,
source = source, source = source,
) )
} }
} }
}
fun String.decodeHex(): ByteArray {
check(length % 2 == 0) { "Must have an even length" }
return chunked(2)
.map { it.toInt(16).toByte() }
.toByteArray()
}
companion object {
const val URL_SEARCH_PREFIX = "slug:"
val SALTED = "Salted__".toByteArray(Charsets.UTF_8)
} }
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
date ?: return 0 // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when { return when {
date.endsWith(" ago", ignoreCase = true) -> { d.endsWith(" ago") || d.endsWith(" atrás") || // Handle translated 'ago' in Portuguese.
parseRelativeDate(date) d.startsWith("há ") || // other translated 'ago' in Portuguese.
} d.endsWith(" hace") || // other translated 'ago' in Spanish
// Handle translated 'ago' in Portuguese. d.endsWith(" önce") || // Handle translated 'ago' in Turkish.
date.endsWith(" atrás", ignoreCase = true) -> { d.endsWith(" trước") || // Handle translated 'ago' in Viêt Nam.
parseRelativeDate(date) d.startsWith("il y a") || // Handle translated 'ago' in French.
} //If there is no ago but just a motion of time
// other translated 'ago' in Portuguese. // short Hours
date.startsWith("há ", ignoreCase = true) -> { d.endsWith(" h") ||
parseRelativeDate(date) // short Day
} d.endsWith(" d") ||
// other translated 'ago' in Spanish // Day in Portuguese
date.endsWith(" hace", ignoreCase = true) -> { d.endsWith(" días") || d.endsWith(" día") ||
parseRelativeDate(date) // Day in French
} d.endsWith(" jour") || d.endsWith(" jours") ||
// Handle translated 'ago' in Turkish. // Hours in Portuguese
date.endsWith(" önce", ignoreCase = true) -> { d.endsWith(" horas") || d.endsWith(" hora") ||
parseRelativeDate(date) // Hours in french
} d.endsWith(" heure") || d.endsWith(" heures") ||
// Handle translated 'ago' in Viêt Nam. // Minutes in English
date.endsWith(" trước", ignoreCase = true) -> { d.endsWith(" mins") ||
parseRelativeDate(date) // Minutes in Portuguese
} d.endsWith(" minutos") || d.endsWith(" minuto") ||
// Handle translated 'ago' in French. //Minutes in French
date.startsWith("il y a", ignoreCase = true) -> { d.endsWith(" minute") || d.endsWith(" minutes") -> parseRelativeDate(date)
parseRelativeDate(date)
}
//If there is no ago but just a motion of time
// short Hours
date.endsWith(" h", ignoreCase = true) -> {
parseRelativeDate(date)
}
// short Day
date.endsWith(" d", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Day in Portuguese
date.endsWith(" días", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" día", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Day in French
date.endsWith(" jour", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" jours", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Hours in Portuguese
date.endsWith(" horas", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" hora", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Hours in french
date.endsWith(" heure", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" heures", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Minutes in English
date.endsWith(" mins", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Minutes in Portuguese
date.endsWith(" minutos", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" minuto", ignoreCase = true) -> {
parseRelativeDate(date)
}
//Minutes in French
date.endsWith(" minute", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" minutes", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle 'yesterday' and 'today', using midnight // Handle 'yesterday' and 'today', using midnight
date.startsWith("year", ignoreCase = true) -> { d.startsWith("year") -> Calendar.getInstance().apply {
Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) // yesterday
add(Calendar.DAY_OF_MONTH, -1) // yesterday set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.HOUR_OF_DAY, 0) set(Calendar.MINUTE, 0)
set(Calendar.MINUTE, 0) set(Calendar.SECOND, 0)
set(Calendar.SECOND, 0) set(Calendar.MILLISECOND, 0)
set(Calendar.MILLISECOND, 0) }.timeInMillis
}.timeInMillis
}
date.startsWith("today", ignoreCase = true) -> { d.startsWith("today") -> Calendar.getInstance().apply {
Calendar.getInstance().apply { set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.HOUR_OF_DAY, 0) set(Calendar.MINUTE, 0)
set(Calendar.MINUTE, 0) set(Calendar.SECOND, 0)
set(Calendar.SECOND, 0) set(Calendar.MILLISECOND, 0)
set(Calendar.MILLISECOND, 0) }.timeInMillis
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> { date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it if (it.contains(Regex("""\d\D\D"""))) {
date.split(" ").map { it.replace(Regex("""\D"""), "")
if (it.contains(Regex("""\d\D\D"""))) { } else {
it.replace(Regex("""\D"""), "") it
} else { }
it }.let { dateFormat.tryParse(it.joinToString(" ")) }
}
}.let { dateFormat.tryParse(it.joinToString(" ")) }
}
else -> dateFormat.tryParse(date) else -> dateFormat.tryParse(date)
} }
@ -649,13 +550,27 @@ internal abstract class MadaraParser(
return result.ifEmpty { null } return result.ifEmpty { null }
} }
private fun createRequestTemplate() = private companion object {
("action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default").split(
'&', private fun createRequestTemplate() =
).map { ("action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" +
val pos = it.indexOf('=') "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" +
it.substring(0, pos) to it.substring(pos + 1) "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" +
}.toMutableMap() "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" +
"%5D=desc&vars%5Bmanga_archives_item_layout%5D=default").split(
'&',
).map {
val pos = it.indexOf('=')
it.substring(0, pos) to it.substring(pos + 1)
}.toMutableMap()
fun String.decodeHex(): ByteArray {
check(length % 2 == 0) { "Must have an even length" }
return chunked(2).map { it.toInt(16).toByte() }.toByteArray()
}
const val URL_SEARCH_PREFIX = "slug:"
val SALTED = "Salted__".toByteArray(Charsets.UTF_8)
}
} }

16
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/LimaScans.kt
Unescape Escape View File

@ -6,12 +6,7 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.madara.MadaraParser import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.mapChapters
import org.koitharu.kotatsu.parsers.util.parseFailed
import org.koitharu.kotatsu.parsers.util.parseHtml
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
@MangaSourceParser("LIMASCANS", "Lima Scans", "pt") @MangaSourceParser("LIMASCANS", "Lima Scans", "pt")
@ -33,8 +28,8 @@ internal class LimaScans(context: MangaLoaderContext) :
val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectchapter).mapChapters(reversed = true) { i, li -> return doc.select(selectchapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a") val a = li.selectFirstOrThrow("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") val href = a.attrAsRelativeUrl("href")
val link = href + stylepage val link = href + stylepage
val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectdate)?.text() val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectdate)?.text()
val name = a.selectFirst("p")?.text() ?: a.ownText() val name = a.selectFirst("p")?.text() ?: a.ownText()
@ -44,10 +39,7 @@ internal class LimaScans(context: MangaLoaderContext) :
name = name, name = name,
number = i + 1, number = i + 1,
branch = null, branch = null,
uploadDate = parseChapterDate( uploadDate = parseChapterDate(dateFormat, dateText),
dateFormat,
dateText,
),
scanlator = null, scanlator = null,
source = source, source = source,
) )

122
src/main/kotlin/org/koitharu/kotatsu/parsers/util/CryptoAES.kt
Unescape Escape View File

@ -1,23 +1,22 @@
package org.koitharu.kotatsu.parsers.util.cryptoaes package org.koitharu.kotatsu.parsers.util
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import java.security.MessageDigest import java.security.MessageDigest
import java.util.Arrays import java.util.*
import java.util.Base64
import javax.crypto.Cipher import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec import javax.crypto.spec.SecretKeySpec
private const val HASH_CIPHER = "AES/CBC/PKCS7PADDING"
private const val AES = "AES"
private const val KDF_DIGEST = "MD5"
/** /**
* Conforming with CryptoJS AES method * Conforming with CryptoJS AES method
*/ */
@Suppress("unused", "FunctionName") class CryptoAES(
object CryptoAES { private val context: MangaLoaderContext,
) {
private const val KEY_SIZE = 256
private const val IV_SIZE = 128
private const val HASH_CIPHER = "AES/CBC/PKCS7PADDING"
private const val AES = "AES"
private const val KDF_DIGEST = "MD5"
/** /**
* Decrypt using CryptoJS defaults compatible method. * Decrypt using CryptoJS defaults compatible method.
@ -27,19 +26,18 @@ object CryptoAES {
* @param cipherText base64 encoded ciphertext * @param cipherText base64 encoded ciphertext
* @param password passphrase * @param password passphrase
*/ */
@Throws(Exception::class)
fun decrypt(cipherText: String, password: String): String { fun decrypt(cipherText: String, password: String): String {
try { val ctBytes = context.decodeBase64(cipherText)
val ctBytes = Base64.getDecoder().decode(cipherText) val saltBytes = Arrays.copyOfRange(ctBytes, 8, 16)
val saltBytes = Arrays.copyOfRange(ctBytes, 8, 16) val cipherTextBytes = Arrays.copyOfRange(ctBytes, 16, ctBytes.size)
val cipherTextBytes = Arrays.copyOfRange(ctBytes, 16, ctBytes.size) val md5: MessageDigest = MessageDigest.getInstance(KDF_DIGEST)
val md5: MessageDigest = MessageDigest.getInstance("MD5") val keyAndIV = generateKeyAndIV(32, 16, 1, saltBytes, password.toByteArray(Charsets.UTF_8), md5)
val keyAndIV = generateKeyAndIV(32, 16, 1, saltBytes, password.toByteArray(Charsets.UTF_8), md5) return decryptAES(
return decryptAES(cipherTextBytes, cipherTextBytes,
keyAndIV?.get(0) ?: ByteArray(32), keyAndIV.getOrNull(0) ?: ByteArray(32),
keyAndIV?.get(1) ?: ByteArray(16)) keyAndIV.getOrNull(1) ?: ByteArray(16),
} catch (e: Exception) { )
return ""
}
} }
/** /**
@ -49,13 +47,10 @@ object CryptoAES {
* @param keyBytes key as a bytearray * @param keyBytes key as a bytearray
* @param ivBytes iv as a bytearray * @param ivBytes iv as a bytearray
*/ */
@Throws(Exception::class)
fun decrypt(cipherText: String, keyBytes: ByteArray, ivBytes: ByteArray): String { fun decrypt(cipherText: String, keyBytes: ByteArray, ivBytes: ByteArray): String {
return try { val cipherTextBytes = context.decodeBase64(cipherText)
val cipherTextBytes = Base64.getDecoder().decode(cipherText) return decryptAES(cipherTextBytes, keyBytes, ivBytes)
decryptAES(cipherTextBytes, keyBytes, ivBytes)
} catch (e: Exception) {
""
}
} }
/** /**
@ -65,15 +60,12 @@ object CryptoAES {
* @param keyBytes key as a bytearray * @param keyBytes key as a bytearray
* @param ivBytes iv as a bytearray * @param ivBytes iv as a bytearray
*/ */
@Throws(Exception::class)
private fun decryptAES(cipherTextBytes: ByteArray, keyBytes: ByteArray, ivBytes: ByteArray): String { private fun decryptAES(cipherTextBytes: ByteArray, keyBytes: ByteArray, ivBytes: ByteArray): String {
return try { val cipher = Cipher.getInstance(HASH_CIPHER)
val cipher = Cipher.getInstance(HASH_CIPHER) val keyS = SecretKeySpec(keyBytes, AES)
val keyS = SecretKeySpec(keyBytes, AES) cipher.init(Cipher.DECRYPT_MODE, keyS, IvParameterSpec(ivBytes))
cipher.init(Cipher.DECRYPT_MODE, keyS, IvParameterSpec(ivBytes)) return cipher.doFinal(cipherTextBytes).toString(Charsets.UTF_8)
cipher.doFinal(cipherTextBytes).toString(Charsets.UTF_8)
} catch (e: Exception) {
""
}
} }
/** /**
@ -92,41 +84,43 @@ object CryptoAES {
* @param md the message digest algorithm to use * @param md the message digest algorithm to use
* @return an two-element array with the generated key and IV * @return an two-element array with the generated key and IV
*/ */
private fun generateKeyAndIV(keyLength: Int, ivLength: Int, iterations: Int, salt: ByteArray, password: ByteArray, md: MessageDigest): Array<ByteArray?>? { @Suppress("SameParameterValue")
@Throws(Exception::class)
private fun generateKeyAndIV(
keyLength: Int,
ivLength: Int,
iterations: Int,
salt: ByteArray?,
password: ByteArray,
md: MessageDigest,
): Array<ByteArray?> {
val digestLength = md.digestLength val digestLength = md.digestLength
val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength
val generatedData = ByteArray(requiredLength) val generatedData = ByteArray(requiredLength)
var generatedLength = 0 var generatedLength = 0
return try { md.reset()
md.reset()
// Repeat process until sufficient data has been generated // Repeat process until sufficient data has been generated
while (generatedLength < keyLength + ivLength) { while (generatedLength < keyLength + ivLength) {
// Digest data (last digest if available, password data, salt if available) // Digest data (last digest if available, password data, salt if available)
if (generatedLength > 0) md.update(generatedData, generatedLength - digestLength, digestLength) if (generatedLength > 0) md.update(generatedData, generatedLength - digestLength, digestLength)
md.update(password) md.update(password)
if (salt != null) md.update(salt, 0, 8) if (salt != null) md.update(salt, 0, 8)
md.digest(generatedData, generatedLength, digestLength) md.digest(generatedData, generatedLength, digestLength)
// additional rounds // additional rounds
for (i in 1 until iterations) { for (i in 1 until iterations) {
md.update(generatedData, generatedLength, digestLength) md.update(generatedData, generatedLength, digestLength)
md.digest(generatedData, generatedLength, digestLength) md.digest(generatedData, generatedLength, digestLength)
}
generatedLength += digestLength
} }
generatedLength += digestLength
// Copy key and IV into separate byte arrays
val result = arrayOfNulls<ByteArray>(2)
result[0] = generatedData.copyOfRange(0, keyLength)
if (ivLength > 0) result[1] = generatedData.copyOfRange(keyLength, keyLength + ivLength)
result
} catch (e: Exception) {
throw e
} finally {
// Clean out temporary data
Arrays.fill(generatedData, 0.toByte())
} }
// Copy key and IV into separate byte arrays
val result = arrayOfNulls<ByteArray>(2)
result[0] = generatedData.copyOfRange(0, keyLength)
if (ivLength > 0) result[1] = generatedData.copyOfRange(keyLength, keyLength + ivLength)
return result
} }
} }

Write Preview
Loading…
Cancel
Save