KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CuuTruyen] Fix attempt 1

Browse Source
master
Koitharu 2 years ago
parent d32d1f5044
commit 955c75a99f
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7
6 changed files with 299 additions and 271 deletions
Show Stats Download Patch File Download Diff File

2
.github/summary.yaml vendored
Unescape Escape View File

@ -1 +1 @@
total: 1110
total: 1111

507
src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/CuuTruyenParser.kt
Unescape Escape View File

@ -2,286 +2,261 @@ package org.koitharu.kotatsu.parsers.site.vi
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import okhttp3.Interceptor
import okhttp3.Response
import okhttp3.HttpUrl
import okhttp3.*
import okhttp3.ResponseBody.Companion.toResponseBody
import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.bitmap.Bitmap
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.awt.Color
import java.awt.Graphics2D
import java.awt.image.BufferedImage
import java.io.ByteArrayOutputStream
import java.text.SimpleDateFormat
import java.util.*
import javax.imageio.ImageIO
@Broken
@MangaSourceParser("CUUTRUYEN", "CuuTruyen", "vi")
internal class CuuTruyenParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaParserSource.CUUTRUYEN, 20), Interceptor {
override val configKeyDomain =
ConfigKey.Domain("cuutruyen.net", "nettrom.com", "hetcuutruyen.net", "cuutruyent9sv7.xyz")
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
)
override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities(
isSearchSupported = true,
)
override suspend fun getFilterOptions() = MangaListFilterOptions()
override fun getRequestHeaders(): Headers = Headers.Builder()
.add("User-Agent", UserAgents.KOTATSU)
.build()
private val decryptionKey = "3141592653589793".toByteArray()
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when {
!filter.query.isNullOrEmpty() -> {
append("/api/v2/mangas/search?q=")
append(filter.query.urlEncoded())
append("&page=")
append(page.toString())
}
else -> {
val tag = filter.tags.oneOrThrowIfMany()
if (tag != null) {
append("/api/v2/tags/")
append(tag.key)
} else {
append("/api/v2/mangas")
when (order) {
SortOrder.UPDATED -> append("/recently_updated")
SortOrder.POPULARITY -> append("/top")
SortOrder.NEWEST -> append("/recently_updated")
else -> append("/recently_updated")
}
}
append("?page=")
append(page.toString())
}
}
append("&per_page=")
append(pageSize)
}
val json = webClient.httpGet(url).parseJson()
val data = json.getJSONArray("data")
return data.mapJSON { jo ->
Manga(
id = generateUid(jo.getLong("id")),
url = "/api/v2/mangas/${jo.getLong("id")}",
publicUrl = "https://$domain/manga/${jo.getLong("id")}",
title = jo.getString("name"),
altTitle = null,
coverUrl = jo.getString("cover_url"),
largeCoverUrl = jo.getString("cover_mobile_url"),
author = jo.getStringOrNull("author_name"),
tags = emptySet(),
state = null,
description = null,
isNsfw = isNsfwSource,
source = source,
rating = RATING_UNKNOWN,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val url = "https://" + domain + manga.url
val chapters = async {
webClient.httpGet("$url/chapters").parseJson().getJSONArray("data")
}
val json = webClient.httpGet(url).parseJson().getJSONObject("data")
val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
manga.copy(
title = json.getStringOrNull("name") ?: manga.title,
isNsfw = json.getBooleanOrDefault("is_nsfw", manga.isNsfw),
author = json.optJSONObject("author")?.getStringOrNull("name")?.substringBefore(','),
description = json.getString("full_description"),
tags = json.optJSONArray("tags")?.mapJSONToSet { jo ->
MangaTag(
title = jo.getString("name").toTitleCase(sourceLocale),
key = jo.getString("slug"),
source = source,
)
}.orEmpty(),
chapters = chapters.await().mapJSON { jo ->
val chapterId = jo.getLong("id")
val number = jo.getFloatOrDefault("number", 0f)
MangaChapter(
id = generateUid(chapterId),
name = jo.getStringOrNull("name") ?: number.formatSimple(),
number = number,
volume = 0,
url = "/api/v2/chapters/$chapterId",
scanlator = jo.optString("group_name"),
uploadDate = chapterDateFormat.tryParse(jo.getStringOrNull("created_at")),
branch = null,
source = source,
)
}.reversed(),
)
}
private val pageSizesMap = mutableMapOf<Long, Pair<Int, Int>>()
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val url = "https://$domain${chapter.url}"
val json = webClient.httpGet(url).parseJson().getJSONObject("data")
return json.getJSONArray("pages").mapJSON { jo ->
val imageUrl = jo.getString("image_url")
val id = jo.getLong("id")
pageSizesMap[id] = jo.getInt("width") to jo.getInt("height")
MangaPage(
id = generateUid(id),
url = imageUrl,
preview = null,
source = source,
)
}
}
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (!request.url.host.contains(domain, ignoreCase = true)) {
return response
}
val body = response.body ?: return response
val contentType = body.contentType()
val bytes = body.bytes()
val pageId = getPageIdFromUrl(request.url)
val (originalWidth, originalHeight) = pageSizesMap[pageId] ?: (0 to 0)
val decrypted = decryptDRM(bytes, decryptionKey)
val reconstructed = decrypted?.let {
reconstructImage(it, originalWidth, originalHeight)
} ?: bytes
val newBody = reconstructed.toResponseBody(contentType)
return response.newBuilder().body(newBody).build()
}
private fun getPageIdFromUrl(url: HttpUrl): Long {
return url.pathSegments.lastOrNull()?.toLongOrNull() ?: 0L
}
private fun getOriginalWidthFromRequest(request: okhttp3.Request): Int {
val width = request.url.queryParameter("width")?.toIntOrNull() ?: 0
return width
}
private fun getOriginalHeightFromRequest(request: okhttp3.Request): Int {
val height = request.url.queryParameter("height")?.toIntOrNull() ?: 0
return height
}
private fun decryptDRM(drmData: ByteArray, key: ByteArray): ByteArray? {
return try {
drmData.mapIndexed { index, byte ->
(byte.toInt() xor key[index % key.size].toInt()).toByte()
}.toByteArray()
} catch (e: Exception) {
null
}
}
private fun reconstructImage(decrypted: ByteArray, originalWidth: Int, originalHeight: Int): ByteArray? {
return try {
val delimiter = "#v".toByteArray()
val delimiterIndex = decrypted.indexOfFirst {
decrypted.sliceArray(it until (it + delimiter.size)).contentEquals(delimiter)
}
if (delimiterIndex == -1) {
return null
}
val segmentsInfoStart = delimiterIndex + delimiter.size
val segmentsData = decrypted.sliceArray(segmentsInfoStart until decrypted.size)
val segments = String(segmentsData).split("|").filter { it.contains("-") }
if (segments.isEmpty()) {
return null
}
val segmentInfo = segments.mapNotNull { seg ->
try {
val (dyStr, heightStr) = seg.split("-")
val dy = if (dyStr.startsWith("dy")) dyStr.substring(2).trim() else dyStr.trim()
val dyInt = dy.toInt()
val height = heightStr.trim().toInt()
dyInt to height
} catch (e: Exception) {
null
}
}
if (segmentInfo.isEmpty()) {
return null
}
var finalSegmentInfo = segmentInfo
val totalHeight = finalSegmentInfo.sumOf { it.second }
if (totalHeight != originalHeight) {
val remainingHeight = originalHeight - totalHeight
if (remainingHeight > 0) {
finalSegmentInfo = finalSegmentInfo.toMutableList().apply { add(0 to remainingHeight) }
}
}
val originalImage = ImageIO.read(decrypted.inputStream()) ?: return null
val newImage = BufferedImage(originalWidth, originalHeight, BufferedImage.TYPE_INT_RGB)
val graphics: Graphics2D = newImage.createGraphics()
var sy = 0
for ((dy, segHeight) in finalSegmentInfo) {
if (sy + segHeight > originalHeight) {
break
}
val subImage = originalImage.getSubimage(0, sy, originalWidth, segHeight)
graphics.drawImage(subImage, 0, dy, null)
sy += segHeight
}
graphics.dispose()
val outputStream = ByteArrayOutputStream()
ImageIO.write(newImage, "JPEG", outputStream)
outputStream.toByteArray()
} catch (e: Exception) {
null
}
}
private fun ByteArray.indexOfFirst(predicate: (Int) -> Boolean): Int {
for (i in indices) {
if (predicate(i)) return i
}
return -1
}
override val configKeyDomain = ConfigKey.Domain(
"cuutruyen.net",
"nettrom.com",
"hetcuutruyen.net",
"cuutruyent9sv7.xyz",
)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
)
override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities(
isSearchSupported = true,
)
override suspend fun getFilterOptions() = MangaListFilterOptions()
override fun getRequestHeaders(): Headers = Headers.Builder()
.add("User-Agent", UserAgents.KOTATSU)
.build()
private val decryptionKey = "3141592653589793".toByteArray()
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when {
!filter.query.isNullOrEmpty() -> {
append("/api/v2/mangas/search?q=")
append(filter.query.urlEncoded())
append("&page=")
append(page.toString())
}
else -> {
val tag = filter.tags.oneOrThrowIfMany()
if (tag != null) {
append("/api/v2/tags/")
append(tag.key)
} else {
append("/api/v2/mangas")
when (order) {
SortOrder.UPDATED -> append("/recently_updated")
SortOrder.POPULARITY -> append("/top")
SortOrder.NEWEST -> append("/recently_updated")
else -> append("/recently_updated")
}
}
append("?page=")
append(page.toString())
}
}
append("&per_page=")
append(pageSize)
}
val json = webClient.httpGet(url).parseJson()
val data = json.getJSONArray("data")
return data.mapJSON { jo ->
Manga(
id = generateUid(jo.getLong("id")),
url = "/api/v2/mangas/${jo.getLong("id")}",
publicUrl = "https://$domain/manga/${jo.getLong("id")}",
title = jo.getString("name"),
altTitle = null,
coverUrl = jo.getString("cover_url"),
largeCoverUrl = jo.getString("cover_mobile_url"),
author = jo.getStringOrNull("author_name"),
tags = emptySet(),
state = null,
description = null,
isNsfw = isNsfwSource,
source = source,
rating = RATING_UNKNOWN,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val url = "https://" + domain + manga.url
val chapters = async {
webClient.httpGet("$url/chapters").parseJson().getJSONArray("data")
}
val json = webClient.httpGet(url).parseJson().getJSONObject("data")
val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
manga.copy(
title = json.getStringOrNull("name") ?: manga.title,
isNsfw = json.getBooleanOrDefault("is_nsfw", manga.isNsfw),
author = json.optJSONObject("author")?.getStringOrNull("name")?.substringBefore(','),
description = json.getString("full_description"),
tags = json.optJSONArray("tags")?.mapJSONToSet { jo ->
MangaTag(
title = jo.getString("name").toTitleCase(sourceLocale),
key = jo.getString("slug"),
source = source,
)
}.orEmpty(),
chapters = chapters.await().mapJSON { jo ->
val chapterId = jo.getLong("id")
val number = jo.getFloatOrDefault("number", 0f)
MangaChapter(
id = generateUid(chapterId),
name = jo.getStringOrNull("name") ?: number.formatSimple(),
number = number,
volume = 0,
url = "/api/v2/chapters/$chapterId",
scanlator = jo.optString("group_name"),
uploadDate = chapterDateFormat.tryParse(jo.getStringOrNull("created_at")),
branch = null,
source = source,
)
}.reversed(),
)
}
private val pageSizesMap = mutableMapOf<Long, Pair<Int, Int>>()
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val url = "https://$domain${chapter.url}"
val json = webClient.httpGet(url).parseJson().getJSONObject("data")
return json.getJSONArray("pages").mapJSON { jo ->
val imageUrl = jo.getString("image_url")
val id = jo.getLong("id")
pageSizesMap[id] = jo.getInt("width") to jo.getInt("height")
MangaPage(
id = generateUid(id),
url = imageUrl,
preview = null,
source = source,
)
}
}
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (!request.url.host.contains(domain, ignoreCase = true)) {
return response
}
val pageId = getPageIdFromUrl(request.url)
val (originalWidth, originalHeight) = pageSizesMap[pageId] ?: (0 to 0)
val decryptedResponse = response.map { body ->
val bytes = body.bytes()
val decrypted = decryptDRM(bytes, decryptionKey)
(swapSegments(decrypted, originalWidth, originalHeight) ?: decrypted).toResponseBody(body.contentType())
}
return context.redrawImageResponse(decryptedResponse) {
redrawImage(it)
}
}
private fun getPageIdFromUrl(url: HttpUrl): Long {
return url.pathSegments.lastOrNull()?.toLongOrNull() ?: 0L
}
private fun getOriginalWidthFromRequest(request: Request): Int {
val width = request.url.queryParameter("width")?.toIntOrNull() ?: 0
return width
}
private fun getOriginalHeightFromRequest(request: Request): Int {
val height = request.url.queryParameter("height")?.toIntOrNull() ?: 0
return height
}
private fun decryptDRM(drmData: ByteArray, key: ByteArray): ByteArray = runCatchingCancellable {
drmData.mapIndexed { index, byte ->
(byte.toInt() xor key[index % key.size].toInt()).toByte()
}.toByteArray()
}.getOrDefault(drmData)
private fun redrawImage(source: Bitmap): Bitmap {
return source
}
private fun swapSegments(decrypted: ByteArray, originalWidth: Int, originalHeight: Int): ByteArray? {
val delimiter = "#v".toByteArray()
val delimiterIndex = decrypted.indexOfFirst {
decrypted.sliceArray(it until (it + delimiter.size)).contentEquals(delimiter)
}
if (delimiterIndex == -1) {
return null
}
val segmentsInfoStart = delimiterIndex + delimiter.size
val segmentsData = decrypted.sliceArray(segmentsInfoStart until decrypted.size)
val segments = String(segmentsData).split("|").filter { it.contains("-") }
if (segments.isEmpty()) {
return null
}
val segmentInfo = segments.mapNotNull { seg ->
try {
val (dyStr, heightStr) = seg.split("-")
val dy = if (dyStr.startsWith("dy")) dyStr.substring(2).trim() else dyStr.trim()
val dyInt = dy.toInt()
val height = heightStr.trim().toInt()
dyInt to height
} catch (e: Exception) {
null
}
}
if (segmentInfo.isEmpty()) {
return null
}
var finalSegmentInfo = segmentInfo
val totalHeight = finalSegmentInfo.sumOf { it.second }
if (totalHeight != originalHeight) {
val remainingHeight = originalHeight - totalHeight
if (remainingHeight > 0) {
finalSegmentInfo = finalSegmentInfo.toMutableList().apply { add(0 to remainingHeight) }
}
}
return decrypted
}
private fun ByteArray.indexOfFirst(predicate: (Int) -> Boolean): Int {
for (i in indices) {
if (predicate(i)) return i
}
return -1
}
}

7
src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt
Unescape Escape View File

@ -6,6 +6,7 @@ import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.Call
import okhttp3.Headers
import okhttp3.Response
import okhttp3.ResponseBody
public suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation ->
val callback = ContinuationCallCallback(this, continuation)
@ -37,3 +38,9 @@ public fun Response.Builder.setHeader(name: String, value: String?): Response.Bu
} else {
header(name, value)
}
public inline fun Response.map(mapper: (ResponseBody) -> ResponseBody): Response = body?.use { responseBody ->
newBuilder()
.body(mapper(responseBody))
.build()
} ?: this

2
src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt
Unescape Escape View File

@ -108,4 +108,4 @@ public fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) {
}.getOrDefault(0L)
}
private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" }
internal fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" }

17
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt
Unescape Escape View File

@ -1,17 +1,23 @@
package org.koitharu.kotatsu.parsers
import com.koushikdutta.quack.QuackContext
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.ResponseBody.Companion.toResponseBody
import org.koitharu.kotatsu.parsers.bitmap.Bitmap
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.requireBody
import org.koitharu.kotatsu.test_util.BitmapTestImpl
import java.awt.image.BufferedImage
import java.security.SecureRandom
import java.security.cert.X509Certificate
import java.util.concurrent.TimeUnit
import javax.imageio.ImageIO
import javax.net.ssl.SSLContext
import javax.net.ssl.SSLSocketFactory
import javax.net.ssl.X509TrustManager
@ -46,12 +52,17 @@ internal object MangaLoaderContextMock : MangaLoaderContext() {
override fun getDefaultUserAgent(): String = UserAgents.FIREFOX_MOBILE
override fun redrawImageResponse(response: Response, redraw: (image: Bitmap) -> Bitmap): Response {
return response
override fun redrawImageResponse(response: Response, redraw: (Bitmap) -> Bitmap): Response {
val srcImage = response.requireBody().byteStream().use(ImageIO::read)
checkNotNull(srcImage) { "Cannot decode image" }
val resImage = (redraw(BitmapTestImpl(srcImage)) as BitmapTestImpl)
return response.newBuilder()
.body(resImage.compress("png").toResponseBody("image/png".toMediaTypeOrNull()))
.build()
}
override fun createBitmap(width: Int, height: Int): Bitmap {
throw UnsupportedOperationException()
return BitmapTestImpl(BufferedImage(width, height, BufferedImage.TYPE_INT_RGB))
}
suspend fun doRequest(url: String, source: MangaSource?): Response {

35
src/test/kotlin/org/koitharu/kotatsu/test_util/BitmapTestImpl.kt
Unescape Escape View File

@ -0,0 +1,35 @@
package org.koitharu.kotatsu.test_util
import org.koitharu.kotatsu.parsers.bitmap.Bitmap
import org.koitharu.kotatsu.parsers.bitmap.Rect
import java.awt.image.BufferedImage
import java.io.ByteArrayOutputStream
import javax.imageio.ImageIO
class BitmapTestImpl(
val image: BufferedImage,
) : Bitmap {
override val width: Int
get() = image.width
override val height: Int
get() = image.height
override fun drawBitmap(
sourceBitmap: Bitmap,
src: Rect,
dst: Rect,
) {
val graphics = image.createGraphics()
val subImage = (sourceBitmap as BitmapTestImpl).image.getSubimage(
src.left, src.top, src.width, src.height,
)
graphics.drawImage(subImage, dst.left, dst.top, dst.width, dst.height, null)
graphics.dispose()
}
fun compress(format: String): ByteArray = ByteArrayOutputStream().also { stream ->
ImageIO.write(image, format, stream)
}.toByteArray()
}
Write Preview
Loading…
Cancel
Save