Fix MangaLib provider

pull/26/head v0.3.2
Koitharu 6 years ago
parent 50f8cb9193
commit 01607ec1e2

@ -17,7 +17,7 @@ android {
minSdkVersion 21 minSdkVersion 21
targetSdkVersion 29 targetSdkVersion 29
versionCode gitCommits versionCode gitCommits
versionName '0.3.1' versionName '0.3.2'
buildConfigField 'String', 'GIT_BRANCH', "\"${gitBranch}\"" buildConfigField 'String', 'GIT_BRANCH', "\"${gitBranch}\""

@ -22,6 +22,6 @@ enum class MangaSource(
HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java), HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java),
YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java), YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java),
MANGATOWN("MangaTown", "en", MangaTownRepository::class.java), MANGATOWN("MangaTown", "en", MangaTownRepository::class.java),
MANGALIB("MangaLib", "ru", MangaLibRepository::class.java), MANGALIB("MangaLib", "ru", MangaLibRepository::class.java)
HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java) // HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java)
} }

@ -1,18 +1,15 @@
package org.koitharu.kotatsu.core.parser package org.koitharu.kotatsu.core.parser
import org.koin.core.KoinComponent
import org.koin.core.inject
import org.koitharu.kotatsu.core.model.MangaPage import org.koitharu.kotatsu.core.model.MangaPage
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.model.MangaTag import org.koitharu.kotatsu.core.model.MangaTag
import org.koitharu.kotatsu.core.model.SortOrder import org.koitharu.kotatsu.core.model.SortOrder
import org.koitharu.kotatsu.domain.MangaLoaderContext import org.koitharu.kotatsu.domain.MangaLoaderContext
abstract class RemoteMangaRepository : MangaRepository, KoinComponent { abstract class RemoteMangaRepository(protected val loaderContext: MangaLoaderContext) : MangaRepository {
protected abstract val source: MangaSource protected abstract val source: MangaSource
protected val loaderContext by inject<MangaLoaderContext>()
protected val conf by lazy(LazyThreadSafetyMode.NONE) { protected val conf by lazy(LazyThreadSafetyMode.NONE) {
loaderContext.getSettings(source) loaderContext.getSettings(source)
} }

@ -4,9 +4,12 @@ import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.ParseException import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.* import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.* import org.koitharu.kotatsu.utils.ext.*
abstract class ChanRepository : RemoteMangaRepository() { abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(
loaderContext
) {
protected abstract val defaultDomain: String protected abstract val defaultDomain: String

@ -4,9 +4,13 @@ import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.ParseException import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.* import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.utils.ext.* import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.map
import org.koitharu.kotatsu.utils.ext.mapIndexed
import org.koitharu.kotatsu.utils.ext.parseHtml
import org.koitharu.kotatsu.utils.ext.parseJson
class DesuMeRepository : RemoteMangaRepository() { class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) {
override val source = MangaSource.DESUME override val source = MangaSource.DESUME

@ -4,9 +4,11 @@ import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.ParseException import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.* import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.* import org.koitharu.kotatsu.utils.ext.*
abstract class GroupleRepository : RemoteMangaRepository() { abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
RemoteMangaRepository(loaderContext) {
protected abstract val defaultDomain: String protected abstract val defaultDomain: String
@ -28,8 +30,11 @@ abstract class GroupleRepository : RemoteMangaRepository() {
"https://$domain/search", "https://$domain/search",
mapOf("q" to query, "offset" to offset.toString()) mapOf("q" to query, "offset" to offset.toString())
) )
tag == null -> loaderContext.httpGet("https://$domain/list?sortType=${getSortKey( tag == null -> loaderContext.httpGet(
sortOrder)}&offset=$offset") "https://$domain/list?sortType=${getSortKey(
sortOrder
)}&offset=$offset"
)
else -> loaderContext.httpGet( else -> loaderContext.httpGet(
"https://$domain/list/genre/${tag.key}?sortType=${getSortKey( "https://$domain/list/genre/${tag.key}?sortType=${getSortKey(
sortOrder sortOrder

@ -5,11 +5,12 @@ import org.koitharu.kotatsu.core.model.Manga
import org.koitharu.kotatsu.core.model.MangaChapter import org.koitharu.kotatsu.core.model.MangaChapter
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.model.MangaTag import org.koitharu.kotatsu.core.model.MangaTag
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.longHashCode import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.parseHtml import org.koitharu.kotatsu.utils.ext.parseHtml
import org.koitharu.kotatsu.utils.ext.withDomain import org.koitharu.kotatsu.utils.ext.withDomain
class HenChanRepository : ChanRepository() { class HenChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loaderContext) {
override val defaultDomain = "h-chan.me" override val defaultDomain = "h-chan.me"
override val source = MangaSource.HENCHAN override val source = MangaSource.HENCHAN

@ -1,11 +1,10 @@
package org.koitharu.kotatsu.core.parser.site package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.core.model.MangaSource /*
class HentaiLibRepository(loaderContext: MangaLoaderContext) : MangaLibRepository(loaderContext) {
class HentaiLibRepository : MangaLibRepository() {
protected override val defaultDomain = "hentailib.me" protected override val defaultDomain = "hentailib.me"
override val source = MangaSource.HENTAILIB override val source = MangaSource.HENTAILIB
} }*/

@ -1,8 +1,9 @@
package org.koitharu.kotatsu.core.parser.site package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
class MangaChanRepository : ChanRepository() { class MangaChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loaderContext) {
override val defaultDomain = "manga-chan.me" override val defaultDomain = "manga-chan.me"
override val source = MangaSource.MANGACHAN override val source = MangaSource.MANGACHAN

@ -6,9 +6,11 @@ import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.ParseException import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.* import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.* import org.koitharu.kotatsu.utils.ext.*
open class MangaLibRepository : RemoteMangaRepository() { open class MangaLibRepository(loaderContext: MangaLoaderContext) :
RemoteMangaRepository(loaderContext) {
protected open val defaultDomain = "mangalib.me" protected open val defaultDomain = "mangalib.me"
@ -28,6 +30,9 @@ open class MangaLibRepository : RemoteMangaRepository() {
sortOrder: SortOrder?, sortOrder: SortOrder?,
tag: MangaTag? tag: MangaTag?
): List<Manga> { ): List<Manga> {
if (!query.isNullOrEmpty()) {
return search(query)
}
val domain = conf.getDomain(defaultDomain) val domain = conf.getDomain(defaultDomain)
val page = (offset / 60f).toIntUp() val page = (offset / 60f).toIntUp()
val url = buildString { val url = buildString {
@ -107,6 +112,7 @@ open class MangaLibRepository : RemoteMangaRepository() {
) )
) )
} }
chapters.reverse()
break@scripts break@scripts
} }
} }
@ -193,4 +199,25 @@ open class MangaLibRepository : RemoteMangaRepository() {
SortOrder.NEWEST -> "desc&sort=created_at" SortOrder.NEWEST -> "desc&sort=created_at"
else -> "desc&sort=last_chapter_at" else -> "desc&sort=last_chapter_at"
} }
private suspend fun search(query: String): List<Manga> {
val domain = conf.getDomain(defaultDomain)
val json = loaderContext.httpGet("https://$domain/search?query=${query.urlEncoded()}")
.parseJsonArray()
return json.map { jo ->
val url = "https://$domain/${jo.getString("slug")}"
Manga(
id = url.longHashCode(),
url = url,
title = jo.getString("rus_name"),
altTitle = jo.getString("name"),
author = null,
tags = emptySet(),
rating = Manga.NO_RATING,
state = null,
source = source,
coverUrl = "https://$domain/uploads/cover/${jo.getString("slug")}/${jo.getString("cover")}/cover_thumb.jpg"
)
}
}
} }

@ -5,10 +5,11 @@ import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.ParseException import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.* import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.* import org.koitharu.kotatsu.utils.ext.*
import java.util.* import java.util.*
class MangaTownRepository : RemoteMangaRepository() { class MangaTownRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) {
override val source = MangaSource.MANGATOWN override val source = MangaSource.MANGATOWN

@ -1,8 +1,9 @@
package org.koitharu.kotatsu.core.parser.site package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
class MintMangaRepository : GroupleRepository() { class MintMangaRepository(loaderContext: MangaLoaderContext) : GroupleRepository(loaderContext) {
override val source = MangaSource.MINTMANGA override val source = MangaSource.MINTMANGA
override val defaultDomain: String = "mintmanga.live" override val defaultDomain: String = "mintmanga.live"

@ -1,8 +1,9 @@
package org.koitharu.kotatsu.core.parser.site package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
class ReadmangaRepository : GroupleRepository() { class ReadmangaRepository(loaderContext: MangaLoaderContext) : GroupleRepository(loaderContext) {
override val defaultDomain = "readmanga.me" override val defaultDomain = "readmanga.me"
override val source = MangaSource.READMANGA_RU override val source = MangaSource.READMANGA_RU

@ -1,8 +1,9 @@
package org.koitharu.kotatsu.core.parser.site package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
class SelfMangaRepository : GroupleRepository() { class SelfMangaRepository(loaderContext: MangaLoaderContext) : GroupleRepository(loaderContext) {
override val defaultDomain = "selfmanga.ru" override val defaultDomain = "selfmanga.ru"
override val source = MangaSource.SELFMANGA override val source = MangaSource.SELFMANGA

@ -4,11 +4,12 @@ import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.Manga import org.koitharu.kotatsu.core.model.Manga
import org.koitharu.kotatsu.core.model.MangaChapter import org.koitharu.kotatsu.core.model.MangaChapter
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.utils.ext.longHashCode import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.parseHtml import org.koitharu.kotatsu.utils.ext.parseHtml
import org.koitharu.kotatsu.utils.ext.withDomain import org.koitharu.kotatsu.utils.ext.withDomain
class YaoiChanRepository : ChanRepository() { class YaoiChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loaderContext) {
override val source = MangaSource.YAOICHAN override val source = MangaSource.YAOICHAN
override val defaultDomain = "yaoi-chan.me" override val defaultDomain = "yaoi-chan.me"

@ -2,13 +2,19 @@ package org.koitharu.kotatsu.domain
import org.koin.core.KoinComponent import org.koin.core.KoinComponent
import org.koin.core.get import org.koin.core.get
import org.koin.core.inject
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.parser.LocalMangaRepository import org.koitharu.kotatsu.core.parser.LocalMangaRepository
import org.koitharu.kotatsu.core.parser.MangaRepository import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.core.prefs.AppSettings import org.koitharu.kotatsu.core.prefs.AppSettings
import java.lang.ref.WeakReference
import java.util.*
object MangaProviderFactory : KoinComponent { object MangaProviderFactory : KoinComponent {
private val loaderContext by inject<MangaLoaderContext>()
private val cache = EnumMap<MangaSource, WeakReference<MangaRepository>>(MangaSource::class.java)
fun getSources(includeHidden: Boolean): List<MangaSource> { fun getSources(includeHidden: Boolean): List<MangaSource> {
val settings = get<AppSettings>() val settings = get<AppSettings>()
val list = MangaSource.values().toList() - MangaSource.LOCAL val list = MangaSource.values().toList() - MangaSource.LOCAL
@ -27,9 +33,24 @@ object MangaProviderFactory : KoinComponent {
} }
} }
fun createLocal() = LocalMangaRepository() fun createLocal(): LocalMangaRepository =
(cache[MangaSource.LOCAL]?.get() as? LocalMangaRepository)
?: LocalMangaRepository().also {
cache[MangaSource.LOCAL] = WeakReference<MangaRepository>(it)
}
@Throws(Throwable::class)
fun create(source: MangaSource): MangaRepository { fun create(source: MangaSource): MangaRepository {
return source.cls.newInstance() cache[source]?.get()?.let {
return it
}
val instance = try {
source.cls.getDeclaredConstructor(MangaLoaderContext::class.java)
.newInstance(loaderContext)
} catch (e: NoSuchMethodException) {
source.cls.newInstance()
}
cache[source] = WeakReference<MangaRepository>(instance)
return instance
} }
} }

@ -51,13 +51,12 @@ class PageLoader : KoinComponent, CoroutineScope, DisposableHandle {
.cacheControl(CacheUtils.CONTROL_DISABLED) .cacheControl(CacheUtils.CONTROL_DISABLED)
.build() .build()
okHttp.newCall(request).await().use { response -> okHttp.newCall(request).await().use { response ->
val body = response.body!! val body = response.body
val type = body.contentType() checkNotNull(body) {
check(type?.type == "image") { "Null response"
"Unexpected content type ${type?.type}/${type?.subtype}"
} }
cache.put(url) { out -> cache.put(url) { out ->
response.body!!.byteStream().copyTo(out) body.byteStream().copyTo(out)
} }
} }
} }

@ -12,7 +12,6 @@ object CacheUtils {
@JvmStatic @JvmStatic
val CONTROL_DISABLED = CacheControl.Builder() val CONTROL_DISABLED = CacheControl.Builder()
.noCache()
.noStore() .noStore()
.build() .build()

@ -2,6 +2,7 @@ package org.koitharu.kotatsu.utils.ext
import okhttp3.Response import okhttp3.Response
import okhttp3.internal.closeQuietly import okhttp3.internal.closeQuietly
import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
@ -30,6 +31,15 @@ fun Response.parseJson(): JSONObject {
} }
} }
fun Response.parseJsonArray(): JSONArray {
try {
val string = body?.string() ?: throw NullPointerException("Response body is null")
return JSONArray(string)
} finally {
closeQuietly()
}
}
inline fun Elements.findOwnText(predicate: (String) -> Boolean): String? { inline fun Elements.findOwnText(predicate: (String) -> Boolean): String? {
for (x in this) { for (x in this) {
val ownText = x.ownText() val ownText = x.ownText()

@ -10,6 +10,7 @@ import org.junit.runners.Parameterized
import org.koin.core.context.startKoin import org.koin.core.context.startKoin
import org.koin.dsl.module import org.koin.dsl.module
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.parser.UserAgentInterceptor
import org.koitharu.kotatsu.core.prefs.SourceConfig import org.koitharu.kotatsu.core.prefs.SourceConfig
import org.koitharu.kotatsu.domain.MangaLoaderContext import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.domain.MangaProviderFactory import org.koitharu.kotatsu.domain.MangaProviderFactory
@ -88,6 +89,8 @@ class RemoteRepositoryTest(source: MangaSource) {
module { module {
factory { factory {
OkHttpClient.Builder() OkHttpClient.Builder()
.cookieJar(TemporaryCookieJar())
.addInterceptor(UserAgentInterceptor)
.connectTimeout(20, TimeUnit.SECONDS) .connectTimeout(20, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS) .readTimeout(60, TimeUnit.SECONDS)
.writeTimeout(20, TimeUnit.SECONDS) .writeTimeout(20, TimeUnit.SECONDS)

@ -0,0 +1,19 @@
package org.koitharu.kotatsu.parsers
import okhttp3.Cookie
import okhttp3.CookieJar
import okhttp3.HttpUrl
import org.koitharu.kotatsu.core.local.cookies.cache.SetCookieCache
class TemporaryCookieJar : CookieJar {
private val cache = SetCookieCache()
override fun loadForRequest(url: HttpUrl): List<Cookie> {
return cache.toList()
}
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
cache.addAll(cookies)
}
}

@ -1,26 +1,33 @@
package org.koitharu.kotatsu.utils package org.koitharu.kotatsu.utils
import okhttp3.OkHttpClient
import okhttp3.Request
import org.junit.Assert import org.junit.Assert
import org.koin.core.KoinComponent
import org.koin.core.inject
import java.net.HttpURLConnection import java.net.HttpURLConnection
import java.net.URL
object AssertX { object AssertX : KoinComponent {
private val okHttp by inject<OkHttpClient>()
fun assertContentType(url: String, vararg types: String) { fun assertContentType(url: String, vararg types: String) {
Assert.assertFalse("URL is empty", url.isEmpty()) Assert.assertFalse("URL is empty", url.isEmpty())
val cn = URL(url).openConnection() as HttpURLConnection val request = Request.Builder()
cn.requestMethod = "HEAD" .url(url)
cn.connect() .head()
when (val code = cn.responseCode) { .build()
HttpURLConnection.HTTP_MOVED_PERM, val response = okHttp.newCall(request).execute()
when (val code = response.code) {
/*HttpURLConnection.HTTP_MOVED_PERM,
HttpURLConnection.HTTP_MOVED_TEMP -> { HttpURLConnection.HTTP_MOVED_TEMP -> {
assertContentType(cn.getHeaderField("Location"), *types) assertContentType(cn.getHeaderField("Location"), *types)
} }*/
HttpURLConnection.HTTP_OK -> { HttpURLConnection.HTTP_OK -> {
val ct = cn.contentType.substringBeforeLast(';').split("/") val type = response.body!!.contentType()
Assert.assertTrue(types.any { Assert.assertTrue(types.any {
val x = it.split('/') val x = it.split('/')
x[0] == ct[0] && (x[1] == "*" || x[1] == ct[1]) type?.type == x[0] && (x[1] == "*" || type.subtype == x[1])
}) })
} }
else -> Assert.fail("Invalid response code $code at $url") else -> Assert.fail("Invalid response code $code at $url")

Loading…
Cancel
Save