Add NudeMoon parser (broken)

pull/26/head
Koitharu 6 years ago
parent 4dc9df0515
commit ff3ebbf1d9

@ -103,6 +103,6 @@ dependencies {
debugImplementation 'com.github.ChuckerTeam.Chucker:library:3.3.0' debugImplementation 'com.github.ChuckerTeam.Chucker:library:3.3.0'
releaseImplementation 'com.github.ChuckerTeam.Chucker:library-no-op:3.3.0' releaseImplementation 'com.github.ChuckerTeam.Chucker:library-no-op:3.3.0'
testImplementation 'junit:junit:4.13' testImplementation 'junit:junit:4.13.1'
testImplementation 'org.json:json:20200518' testImplementation 'org.json:json:20200518'
} }

@ -10,7 +10,9 @@ import coil.ImageLoader
import coil.util.CoilUtils import coil.util.CoilUtils
import com.chuckerteam.chucker.api.ChuckerCollector import com.chuckerteam.chucker.api.ChuckerCollector
import com.chuckerteam.chucker.api.ChuckerInterceptor import com.chuckerteam.chucker.api.ChuckerInterceptor
import okhttp3.CookieJar
import okhttp3.OkHttpClient import okhttp3.OkHttpClient
import org.koin.android.ext.android.get
import org.koin.android.ext.koin.androidContext import org.koin.android.ext.koin.androidContext
import org.koin.android.ext.koin.androidLogger import org.koin.android.ext.koin.androidLogger
import org.koin.core.context.startKoin import org.koin.core.context.startKoin
@ -37,10 +39,6 @@ import java.util.concurrent.TimeUnit
class KotatsuApp : Application() { class KotatsuApp : Application() {
private val cookieJar by lazy {
PersistentCookieJar(SetCookieCache(), SharedPrefsCookiePersistor(applicationContext))
}
private val chuckerCollector by lazy(LazyThreadSafetyMode.NONE) { private val chuckerCollector by lazy(LazyThreadSafetyMode.NONE) {
ChuckerCollector(applicationContext) ChuckerCollector(applicationContext)
} }
@ -48,20 +46,24 @@ class KotatsuApp : Application() {
override fun onCreate() { override fun onCreate() {
super.onCreate() super.onCreate()
if (BuildConfig.DEBUG) { if (BuildConfig.DEBUG) {
StrictMode.setThreadPolicy(StrictMode.ThreadPolicy.Builder() StrictMode.setThreadPolicy(
.detectAll() StrictMode.ThreadPolicy.Builder()
.penaltyLog() .detectAll()
.build()) .penaltyLog()
StrictMode.setVmPolicy(StrictMode.VmPolicy.Builder() .build()
.detectAll() )
.setClassInstanceLimit(LocalMangaRepository::class.java, 1) StrictMode.setVmPolicy(
.setClassInstanceLimit(PagesCache::class.java, 1) StrictMode.VmPolicy.Builder()
.setClassInstanceLimit(MangaLoaderContext::class.java, 1) .detectAll()
.penaltyLog() .setClassInstanceLimit(LocalMangaRepository::class.java, 1)
.build()) .setClassInstanceLimit(PagesCache::class.java, 1)
.setClassInstanceLimit(MangaLoaderContext::class.java, 1)
.penaltyLog()
.build()
)
} }
initKoin() initKoin()
initCoil() initCoil(get())
Thread.setDefaultUncaughtExceptionHandler(AppCrashHandler(applicationContext)) Thread.setDefaultUncaughtExceptionHandler(AppCrashHandler(applicationContext))
if (BuildConfig.DEBUG) { if (BuildConfig.DEBUG) {
initErrorHandler() initErrorHandler()
@ -78,8 +80,14 @@ class KotatsuApp : Application() {
androidContext(applicationContext) androidContext(applicationContext)
modules( modules(
module { module {
single<CookieJar> {
PersistentCookieJar(
SetCookieCache(),
SharedPrefsCookiePersistor(applicationContext)
)
}
factory { factory {
okHttp() okHttp(get())
.cache(CacheUtils.createHttpCache(applicationContext)) .cache(CacheUtils.createHttpCache(applicationContext))
.build() .build()
} }
@ -100,11 +108,11 @@ class KotatsuApp : Application() {
} }
} }
private fun initCoil() { private fun initCoil(cookieJar: CookieJar) {
Coil.setImageLoader( Coil.setImageLoader(
ImageLoader.Builder(applicationContext) ImageLoader.Builder(applicationContext)
.okHttpClient( .okHttpClient(
okHttp() okHttp(cookieJar)
.cache(CoilUtils.createDefaultCache(applicationContext)) .cache(CoilUtils.createDefaultCache(applicationContext))
.build() .build()
).componentRegistry( ).componentRegistry(
@ -124,7 +132,7 @@ class KotatsuApp : Application() {
} }
} }
private fun okHttp() = OkHttpClient.Builder().apply { private fun okHttp(cookieJar: CookieJar) = OkHttpClient.Builder().apply {
connectTimeout(20, TimeUnit.SECONDS) connectTimeout(20, TimeUnit.SECONDS)
readTimeout(60, TimeUnit.SECONDS) readTimeout(60, TimeUnit.SECONDS)
writeTimeout(20, TimeUnit.SECONDS) writeTimeout(20, TimeUnit.SECONDS)

@ -15,10 +15,10 @@
*/ */
package org.koitharu.kotatsu.core.local.cookies package org.koitharu.kotatsu.core.local.cookies
import org.koitharu.kotatsu.core.local.cookies.persistence.CookiePersistor
import okhttp3.Cookie import okhttp3.Cookie
import okhttp3.HttpUrl import okhttp3.HttpUrl
import org.koitharu.kotatsu.core.local.cookies.cache.CookieCache import org.koitharu.kotatsu.core.local.cookies.cache.CookieCache
import org.koitharu.kotatsu.core.local.cookies.persistence.CookiePersistor
import java.util.* import java.util.*
class PersistentCookieJar( class PersistentCookieJar(

@ -22,6 +22,7 @@ enum class MangaSource(
HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java), HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java),
YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java), YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java),
MANGATOWN("MangaTown", "en", MangaTownRepository::class.java), MANGATOWN("MangaTown", "en", MangaTownRepository::class.java),
MANGALIB("MangaLib", "ru", MangaLibRepository::class.java) MANGALIB("MangaLib", "ru", MangaLibRepository::class.java),
NUDEMOON("Nude-Moon", "ru", NudeMoonRepository::class.java)
// HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java) // HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java)
} }

@ -0,0 +1,151 @@
package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.*
import java.util.regex.Pattern
class NudeMoonRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) {
override val source = MangaSource.NUDEMOON
override val sortOrders = setOf(SortOrder.NEWEST, SortOrder.POPULARITY, SortOrder.RATING)
init {
loaderContext.insertCookies(
conf.getDomain(DEFAULT_DOMAIN),
"NMfYa=1;",
"nm_mobile=0;"
)
}
override suspend fun getList(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
): List<Manga> {
val domain = conf.getDomain(DEFAULT_DOMAIN)
val url = when {
!query.isNullOrEmpty() -> "https://$domain//search?stext=${query.urlEncoded()}&rowstart=$offset"
tag != null -> "https://$domain/tags/${tag.key}&rowstart=$offset"
else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
}
val doc = loaderContext.httpGet(url) {
addHeader("Cookie", "NMfYa=1; nm_mobile=0;")
}.parseHtml()
val root = doc.body().run {
selectFirst("td.shoutbox") ?: selectFirst("td.main-bg")
} ?: throw ParseException("Cannot find root")
return root.select("table.news_pic2").mapNotNull { row ->
val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
?: return@mapNotNull null
val href = a.absUrl("href")
val title = a.selectFirst("h2")?.text().orEmpty()
val info = row.selectFirst("div.tbl2") ?: return@mapNotNull null
Manga(
id = href.longHashCode(),
url = href,
title = title.substringAfter(" / "),
altTitle = title.substringBefore(" / ", "")
.takeUnless { it.isBlank() },
author = info.getElementsContainingOwnText("Автор:")?.firstOrNull()
?.nextElementSibling()?.ownText(),
coverUrl = row.selectFirst("img.news_pic2")?.absUrl("src")
.orEmpty(),
tags = row.selectFirst("span.tag-links")?.select("a")
?.mapToSet {
MangaTag(
title = it.text(),
key = it.attr("href").substringAfterLast('/').urlEncoded(),
source = source
)
}.orEmpty(),
source = source
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = loaderContext.httpGet(manga.url).parseHtml()
val root = doc.body().selectFirst("table.shoutbox")
?: throw ParseException("Cannot find root")
val info = root.select("div.tbl2")
return manga.copy(
description = info.select("div.blockquote").lastOrNull()?.html(),
tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet {
MangaTag(
title = it.text(),
key = it.attr("href").substringAfterLast('/').urlEncoded(),
source = source
)
} ?: manga.tags,
chapters = listOf(
MangaChapter(
id = manga.id,
url = manga.url,
source = source,
number = 1,
name = manga.title
)
)
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
conf.getDomain(DEFAULT_DOMAIN)
val doc = loaderContext.httpGet(chapter.url).parseHtml()
val root = doc.body().selectFirst("td.main-body")
?: throw ParseException("Cannot find root")
return root.getElementsByAttributeValueMatching("href", pageUrlPatter).mapNotNull { a ->
val url = a.absUrl("href")
MangaPage(
id = url.longHashCode(),
url = url,
preview = a.selectFirst("img")?.absUrl("src"),
source = source
)
}
}
override suspend fun getPageFullUrl(page: MangaPage): String {
val doc = loaderContext.httpGet(page.url).parseHtml()
return doc.body().getElementById("gallery").attr("src").inContextOf(doc)
}
override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(DEFAULT_DOMAIN)
val doc = loaderContext.httpGet("https://$domain/all_manga").parseHtml()
val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
?.selectFirst("td.textbox")?.selectFirst("td.small")
?: throw ParseException("Tags root not found")
return root.select("a").mapToSet {
MangaTag(
title = it.text(),
key = it.attr("href").substringAfterLast('/')
.removeSuffix("+").urlEncoded(),
source = source
)
}
}
override fun onCreatePreferences() = setOf(R.string.key_parser_domain)
private fun getSortKey(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
SortOrder.POPULARITY -> "views"
SortOrder.NEWEST -> "date"
SortOrder.RATING -> "like"
else -> "like"
}
private companion object {
private const val DEFAULT_DOMAIN = "nude-moon.me"
private val pageUrlPatter = Pattern.compile(".*\\?page=[0-9]+$")
}
}

@ -1,9 +1,6 @@
package org.koitharu.kotatsu.domain package org.koitharu.kotatsu.domain
import okhttp3.FormBody import okhttp3.*
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import org.koin.core.component.KoinComponent import org.koin.core.component.KoinComponent
import org.koin.core.component.get import org.koin.core.component.get
import org.koin.core.component.inject import org.koin.core.component.inject
@ -14,6 +11,7 @@ import org.koitharu.kotatsu.utils.ext.await
open class MangaLoaderContext : KoinComponent { open class MangaLoaderContext : KoinComponent {
private val okHttp by inject<OkHttpClient>() private val okHttp by inject<OkHttpClient>()
private val cookieJar by inject<CookieJar>()
suspend fun httpGet(url: String, block: (Request.Builder.() -> Unit)? = null): Response { suspend fun httpGet(url: String, block: (Request.Builder.() -> Unit)? = null): Response {
val request = Request.Builder() val request = Request.Builder()
@ -44,4 +42,19 @@ open class MangaLoaderContext : KoinComponent {
} }
open fun getSettings(source: MangaSource) = SourceConfig(get(), source) open fun getSettings(source: MangaSource) = SourceConfig(get(), source)
fun insertCookies(domain: String, vararg cookies: String) {
val url = HttpUrl.Builder()
.scheme(SCHEME_HTTP)
.host(domain)
.build()
cookieJar.saveFromResponse(url, cookies.mapNotNull {
Cookie.parse(url, it)
})
}
private companion object {
private const val SCHEME_HTTP = "http"
}
} }

@ -23,7 +23,7 @@ class RemoteListPresenter : BasePresenter<MangaListView<Unit>>() {
presenterScope.launch { presenterScope.launch {
viewState.onLoadingStateChanged(true) viewState.onLoadingStateChanged(true)
try { try {
val list = withContext(Dispatchers.IO) { val list = withContext(Dispatchers.Default) {
MangaProviderFactory.create(source).getList( MangaProviderFactory.create(source).getList(
offset = offset, offset = offset,
sortOrder = filter?.sortOrder, sortOrder = filter?.sortOrder,
@ -64,7 +64,7 @@ class RemoteListPresenter : BasePresenter<MangaListView<Unit>>() {
isFilterInitialized = true isFilterInitialized = true
presenterScope.launch { presenterScope.launch {
try { try {
val (sorts, tags) = withContext(Dispatchers.IO) { val (sorts, tags) = withContext(Dispatchers.Default) {
val repo = MangaProviderFactory.create(source) val repo = MangaProviderFactory.create(source)
repo.sortOrders.sortedBy { it.ordinal } to repo.getTags().sortedBy { it.title } repo.sortOrders.sortedBy { it.ordinal } to repo.getTags().sortedBy { it.title }
} }

@ -55,9 +55,9 @@ class PageHolderDelegate(
state = State.CONVERTED state = State.CONVERTED
callback.onImageReady(file.toUri()) callback.onImageReady(file.toUri())
} catch (e2: Throwable) { } catch (e2: Throwable) {
e2.addSuppressed(e) e.addSuppressed(e2)
state = State.ERROR state = State.ERROR
callback.onError(e2) callback.onError(e)
} }
} }
} else { } else {
@ -73,6 +73,7 @@ class PageHolderDelegate(
try { try {
val file = withContext(Dispatchers.IO) { val file = withContext(Dispatchers.IO) {
val pageUrl = MangaProviderFactory.create(data.source).getPageFullUrl(data) val pageUrl = MangaProviderFactory.create(data.source).getPageFullUrl(data)
check(pageUrl.isNotEmpty()) { "Cannot obtain full image url" }
loader.loadFile(pageUrl, force) loader.loadFile(pageUrl, force)
} }
this@PageHolderDelegate.file = file this@PageHolderDelegate.file = file

@ -1,5 +1,7 @@
package org.koitharu.kotatsu.utils.ext package org.koitharu.kotatsu.utils.ext
import androidx.collection.ArraySet
fun <T> MutableCollection<T>.replaceWith(subject: Iterable<T>) { fun <T> MutableCollection<T>.replaceWith(subject: Iterable<T>) {
clear() clear()
addAll(subject) addAll(subject)
@ -24,4 +26,11 @@ inline fun <T> Iterable<T>.sumByLong(selector: (T) -> Long): Long {
fun <T> List<T>.medianOrNull(): T? = when { fun <T> List<T>.medianOrNull(): T? = when {
isEmpty() -> null isEmpty() -> null
else -> get((size / 2).coerceIn(indices)) else -> get((size / 2).coerceIn(indices))
}
inline fun <T, R> Collection<T>.mapToSet(transform: (T) -> R): Set<R> {
val destination = ArraySet<R>(size)
for (item in this)
destination.add(transform(item))
return destination
} }

@ -5,7 +5,9 @@ import okhttp3.internal.closeQuietly
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.internal.StringUtil
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.jsoup.nodes.Node
import org.jsoup.select.Elements import org.jsoup.select.Elements
fun Response.parseHtml(): Document { fun Response.parseHtml(): Document {
@ -59,4 +61,12 @@ inline fun Elements.findText(predicate: (String) -> Boolean): String? {
} }
} }
return null return null
}
fun String.inContextOf(node: Node): String {
return if (this.isEmpty()) {
""
} else {
StringUtil.resolve(node.baseUri(), this)
}
} }

@ -15,6 +15,7 @@ fun String.longHashCode(): Long {
return h return h
} }
@Deprecated("Use String.inContextOf")
fun String.withDomain(domain: String, ssl: Boolean = true) = when { fun String.withDomain(domain: String, ssl: Boolean = true) = when {
this.startsWith("//") -> buildString { this.startsWith("//") -> buildString {
append("http") append("http")

@ -6,7 +6,7 @@ buildscript {
jcenter() jcenter()
} }
dependencies { dependencies {
classpath 'com.android.tools.build:gradle:4.1.0-rc03' classpath 'com.android.tools.build:gradle:4.1.0'
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
// NOTE: Do not place your application dependencies here; they belong // NOTE: Do not place your application dependencies here; they belong

Loading…
Cancel
Save