KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Madara] Refactoring

Browse Source
pull/217/head
Koitharu 3 years ago
parent c85101991f
commit 69e0a531df
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7
4 changed files with 139 additions and 238 deletions
Show Stats Download Patch File Download Diff File

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt
Unescape Escape View File

@ -95,13 +95,13 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser(
oneShotChapterFromElement(item)
}
} else {
val chapters = mutableListOf<MangaChapter>()
val chapters = ChaptersListBuilder(10)
doc.select(regularChapterListSelector()).reversed().forEachIndexed { i, item ->
val chaptername = item.select("div.col-10.text-truncate").text().replace("&nbsp;", " ").trim()
val scanelement = item.select("ul.chapter-list > li")
scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) }
}
chapters
chapters.toList()
},
)
}

169
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt
Unescape Escape View File

@ -8,15 +8,12 @@ import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.cryptoaes.CryptoAES
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
internal abstract class MadaraParser(
context: MangaLoaderContext,
source: MangaSource,
@ -396,13 +393,11 @@ internal abstract class MadaraParser(
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chapterProtector = doc.selectFirst("#chapter-protector-data")
val chapterProtector = doc.getElementById("chapter-protector-data")
if (chapterProtector == null) {
val root = doc.body().selectFirst("div.main-col-inner")?.selectFirst("div.reading-content")
?: throw ParseException("Root not found", fullUrl)
val root = doc.body().selectFirstOrThrow("div.main-col-inner").selectFirstOrThrow("div.reading-content")
return root.select("div.page-break").map { div ->
val img = div.selectFirst("img") ?: div.parseFailed("Page image not found")
val img = div.selectFirstOrThrow("img")
val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
@ -412,8 +407,6 @@ internal abstract class MadaraParser(
)
}
} else {
val chapterProtectorHtml = chapterProtector.html()
val password = chapterProtectorHtml.substringAfter("wpmangaprotectornonce='").substringBefore("';")
val chapterData = JSONObject(
@ -423,166 +416,74 @@ internal abstract class MadaraParser(
val salt = chapterData.getString("s").toString().decodeHex()
val ciphertext = SALTED + salt + unsaltedCiphertext
val rawImgArray = CryptoAES.decrypt(Base64.getEncoder().encodeToString(ciphertext), password)
val imgArrayString = rawImgArray
.replace("[", "")
.replace("]", "")
.replace("\\", "")
.replace("\"", "")
val rawImgArray = CryptoAES(context).decrypt(context.encodeBase64(ciphertext), password)
val imgArrayString = rawImgArray.filterNot { c -> c == '[' || c == ']' || c == '\\' || c == '"' }
return imgArrayString.split(",").map { url ->
MangaPage(
id = generateUid(url.toString()),
url = url.toString(),
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}
fun String.decodeHex(): ByteArray {
check(length % 2 == 0) { "Must have an even length" }
return chunked(2)
.map { it.toInt(16).toByte() }
.toByteArray()
}
companion object {
const val URL_SEARCH_PREFIX = "slug:"
val SALTED = "Salted__".toByteArray(Charsets.UTF_8)
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
date ?: return 0
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
date.endsWith(" ago", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in Portuguese.
date.endsWith(" atrás", ignoreCase = true) -> {
parseRelativeDate(date)
}
// other translated 'ago' in Portuguese.
date.startsWith("há ", ignoreCase = true) -> {
parseRelativeDate(date)
}
// other translated 'ago' in Spanish
date.endsWith(" hace", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in Turkish.
date.endsWith(" önce", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in Viêt Nam.
date.endsWith(" trước", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in French.
date.startsWith("il y a", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" ago") || d.endsWith(" atrás") || // Handle translated 'ago' in Portuguese.
d.startsWith("há ") || // other translated 'ago' in Portuguese.
d.endsWith(" hace") || // other translated 'ago' in Spanish
d.endsWith(" önce") || // Handle translated 'ago' in Turkish.
d.endsWith(" trước") || // Handle translated 'ago' in Viêt Nam.
d.startsWith("il y a") || // Handle translated 'ago' in French.
//If there is no ago but just a motion of time
// short Hours
date.endsWith(" h", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" h") ||
// short Day
date.endsWith(" d", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" d") ||
// Day in Portuguese
date.endsWith(" días", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" día", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" días") || d.endsWith(" día") ||
// Day in French
date.endsWith(" jour", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" jours", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" jour") || d.endsWith(" jours") ||
// Hours in Portuguese
date.endsWith(" horas", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" hora", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" horas") || d.endsWith(" hora") ||
// Hours in french
date.endsWith(" heure", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" heures", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" heure") || d.endsWith(" heures") ||
// Minutes in English
date.endsWith(" mins", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" mins") ||
// Minutes in Portuguese
date.endsWith(" minutos", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" minuto", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" minutos") || d.endsWith(" minuto") ||
//Minutes in French
date.endsWith(" minute", ignoreCase = true) -> {
parseRelativeDate(date)
}
date.endsWith(" minutes", ignoreCase = true) -> {
parseRelativeDate(date)
}
d.endsWith(" minute") || d.endsWith(" minutes") -> parseRelativeDate(date)
// Handle 'yesterday' and 'today', using midnight
date.startsWith("year", ignoreCase = true) -> {
Calendar.getInstance().apply {
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.startsWith("today", ignoreCase = true) -> {
Calendar.getInstance().apply {
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
date.split(" ").map {
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
it.replace(Regex("""\D"""), "")
} else {
it
}
}.let { dateFormat.tryParse(it.joinToString(" ")) }
}
else -> dateFormat.tryParse(date)
}
@ -649,13 +550,27 @@ internal abstract class MadaraParser(
return result.ifEmpty { null }
}
private companion object {
private fun createRequestTemplate() =
("action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default").split(
("action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" +
"orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" +
"%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" +
"%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" +
"%5D=desc&vars%5Bmanga_archives_item_layout%5D=default").split(
'&',
).map {
val pos = it.indexOf('=')
it.substring(0, pos) to it.substring(pos + 1)
}.toMutableMap()
fun String.decodeHex(): ByteArray {
check(length % 2 == 0) { "Must have an even length" }
return chunked(2).map { it.toInt(16).toByte() }.toByteArray()
}
const val URL_SEARCH_PREFIX = "slug:"
val SALTED = "Salted__".toByteArray(Charsets.UTF_8)
}
}

16
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/pt/LimaScans.kt
Unescape Escape View File

@ -6,12 +6,7 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.mapChapters
import org.koitharu.kotatsu.parsers.util.parseFailed
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@MangaSourceParser("LIMASCANS", "Lima Scans", "pt")
@ -33,8 +28,8 @@ internal class LimaScans(context: MangaLoaderContext) :
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectchapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val link = href + stylepage
val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectdate)?.text()
val name = a.selectFirst("p")?.text() ?: a.ownText()
@ -44,10 +39,7 @@ internal class LimaScans(context: MangaLoaderContext) :
name = name,
number = i + 1,
branch = null,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
uploadDate = parseChapterDate(dateFormat, dateText),
scanlator = null,
source = source,
)

74
src/main/kotlin/org/koitharu/kotatsu/parsers/util/CryptoAES.kt
Unescape Escape View File

@ -1,24 +1,23 @@
package org.koitharu.kotatsu.parsers.util.cryptoaes
package org.koitharu.kotatsu.parsers.util
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import java.security.MessageDigest
import java.util.Arrays
import java.util.Base64
import java.util.*
import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
/**
* Conforming with CryptoJS AES method
*/
@Suppress("unused", "FunctionName")
object CryptoAES {
private const val KEY_SIZE = 256
private const val IV_SIZE = 128
private const val HASH_CIPHER = "AES/CBC/PKCS7PADDING"
private const val AES = "AES"
private const val KDF_DIGEST = "MD5"
/**
* Conforming with CryptoJS AES method
*/
class CryptoAES(
private val context: MangaLoaderContext,
) {
/**
* Decrypt using CryptoJS defaults compatible method.
* Uses KDF equivalent to OpenSSL's EVP_BytesToKey function
@ -27,19 +26,18 @@ object CryptoAES {
* @param cipherText base64 encoded ciphertext
* @param password passphrase
*/
@Throws(Exception::class)
fun decrypt(cipherText: String, password: String): String {
try {
val ctBytes = Base64.getDecoder().decode(cipherText)
val ctBytes = context.decodeBase64(cipherText)
val saltBytes = Arrays.copyOfRange(ctBytes, 8, 16)
val cipherTextBytes = Arrays.copyOfRange(ctBytes, 16, ctBytes.size)
val md5: MessageDigest = MessageDigest.getInstance("MD5")
val md5: MessageDigest = MessageDigest.getInstance(KDF_DIGEST)
val keyAndIV = generateKeyAndIV(32, 16, 1, saltBytes, password.toByteArray(Charsets.UTF_8), md5)
return decryptAES(cipherTextBytes,
keyAndIV?.get(0) ?: ByteArray(32),
keyAndIV?.get(1) ?: ByteArray(16))
} catch (e: Exception) {
return ""
}
return decryptAES(
cipherTextBytes,
keyAndIV.getOrNull(0) ?: ByteArray(32),
keyAndIV.getOrNull(1) ?: ByteArray(16),
)
}
/**
@ -49,13 +47,10 @@ object CryptoAES {
* @param keyBytes key as a bytearray
* @param ivBytes iv as a bytearray
*/
@Throws(Exception::class)
fun decrypt(cipherText: String, keyBytes: ByteArray, ivBytes: ByteArray): String {
return try {
val cipherTextBytes = Base64.getDecoder().decode(cipherText)
decryptAES(cipherTextBytes, keyBytes, ivBytes)
} catch (e: Exception) {
""
}
val cipherTextBytes = context.decodeBase64(cipherText)
return decryptAES(cipherTextBytes, keyBytes, ivBytes)
}
/**
@ -65,15 +60,12 @@ object CryptoAES {
* @param keyBytes key as a bytearray
* @param ivBytes iv as a bytearray
*/
@Throws(Exception::class)
private fun decryptAES(cipherTextBytes: ByteArray, keyBytes: ByteArray, ivBytes: ByteArray): String {
return try {
val cipher = Cipher.getInstance(HASH_CIPHER)
val keyS = SecretKeySpec(keyBytes, AES)
cipher.init(Cipher.DECRYPT_MODE, keyS, IvParameterSpec(ivBytes))
cipher.doFinal(cipherTextBytes).toString(Charsets.UTF_8)
} catch (e: Exception) {
""
}
return cipher.doFinal(cipherTextBytes).toString(Charsets.UTF_8)
}
/**
@ -92,12 +84,20 @@ object CryptoAES {
* @param md the message digest algorithm to use
* @return an two-element array with the generated key and IV
*/
private fun generateKeyAndIV(keyLength: Int, ivLength: Int, iterations: Int, salt: ByteArray, password: ByteArray, md: MessageDigest): Array<ByteArray?>? {
@Suppress("SameParameterValue")
@Throws(Exception::class)
private fun generateKeyAndIV(
keyLength: Int,
ivLength: Int,
iterations: Int,
salt: ByteArray?,
password: ByteArray,
md: MessageDigest,
): Array<ByteArray?> {
val digestLength = md.digestLength
val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength
val generatedData = ByteArray(requiredLength)
var generatedLength = 0
return try {
md.reset()
// Repeat process until sufficient data has been generated
@ -121,12 +121,6 @@ object CryptoAES {
val result = arrayOfNulls<ByteArray>(2)
result[0] = generatedData.copyOfRange(0, keyLength)
if (ivLength > 0) result[1] = generatedData.copyOfRange(keyLength, keyLength + ivLength)
result
} catch (e: Exception) {
throw e
} finally {
// Clean out temporary data
Arrays.fill(generatedData, 0.toByte())
}
return result
}
}

Write Preview
Loading…
Cancel
Save