Use relative urls for mangas and change id generation algorythm

pull/26/head
Koitharu 5 years ago
parent c64115a268
commit 4aa1b58109

@ -16,7 +16,7 @@ android {
minSdkVersion 21 minSdkVersion 21
targetSdkVersion 30 targetSdkVersion 30
versionCode gitCommits versionCode gitCommits
versionName '1.0-b3' versionName '1.0-rc1'
kapt { kapt {
arguments { arguments {
@ -68,7 +68,7 @@ dependencies {
implementation 'androidx.core:core-ktx:1.5.0-beta01' implementation 'androidx.core:core-ktx:1.5.0-beta01'
implementation 'androidx.activity:activity-ktx:1.2.0-rc01' implementation 'androidx.activity:activity-ktx:1.2.0-rc01'
implementation 'androidx.fragment:fragment-ktx:1.3.0-rc01' implementation 'androidx.fragment:fragment-ktx:1.3.0-rc02'
implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.3.0-rc01' implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.3.0-rc01'
implementation 'androidx.lifecycle:lifecycle-viewmodel-ktx:2.3.0-rc01' implementation 'androidx.lifecycle:lifecycle-viewmodel-ktx:2.3.0-rc01'
implementation 'androidx.lifecycle:lifecycle-livedata-ktx:2.3.0-rc01' implementation 'androidx.lifecycle:lifecycle-livedata-ktx:2.3.0-rc01'
@ -79,7 +79,7 @@ dependencies {
implementation 'androidx.recyclerview:recyclerview:1.2.0-beta01' implementation 'androidx.recyclerview:recyclerview:1.2.0-beta01'
implementation 'androidx.viewpager2:viewpager2:1.1.0-alpha01' implementation 'androidx.viewpager2:viewpager2:1.1.0-alpha01'
implementation 'androidx.preference:preference-ktx:1.1.1' implementation 'androidx.preference:preference-ktx:1.1.1'
implementation 'androidx.work:work-runtime-ktx:2.5.0-rc01' implementation 'androidx.work:work-runtime-ktx:2.5.0'
implementation 'com.google.android.material:material:1.3.0-rc01' implementation 'com.google.android.material:material:1.3.0-rc01'
//noinspection LifecycleAnnotationProcessorWithJava8 //noinspection LifecycleAnnotationProcessorWithJava8
kapt 'androidx.lifecycle:lifecycle-compiler:2.3.0-rc01' kapt 'androidx.lifecycle:lifecycle-compiler:2.3.0-rc01'

@ -1,6 +1,5 @@
package org.koitharu.kotatsu.core.parser package org.koitharu.kotatsu.core.parser
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.koitharu.kotatsu.base.domain.MangaLoaderContext import org.koitharu.kotatsu.base.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.model.MangaPage import org.koitharu.kotatsu.core.model.MangaPage
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
@ -13,27 +12,60 @@ abstract class RemoteMangaRepository(
protected abstract val source: MangaSource protected abstract val source: MangaSource
protected val conf by lazy { protected abstract val defaultDomain: String
private val conf by lazy {
loaderContext.getSettings(source) loaderContext.getSettings(source)
} }
override val sortOrders: Set<SortOrder> get() = emptySet() override val sortOrders: Set<SortOrder> get() = emptySet()
override suspend fun getPageUrl(page: MangaPage): String = page.url override suspend fun getPageUrl(page: MangaPage): String = page.url.withDomain()
override suspend fun getTags(): Set<MangaTag> = emptySet() override suspend fun getTags(): Set<MangaTag> = emptySet()
abstract fun onCreatePreferences(): Set<String> abstract fun onCreatePreferences(): Set<String>
protected fun getDomain() = conf.getDomain(defaultDomain)
protected fun String.withDomain() = when {
this.startsWith("//") -> buildString {
append("http")
if (conf.isUseSsl(true)) {
append('s')
}
append(":")
append(this@withDomain)
}
this.startsWith("/") -> buildString {
append("http")
if (conf.isUseSsl(true)) {
append('s')
}
append("://")
append(conf.getDomain(defaultDomain))
append(this@withDomain)
}
else -> this
}
protected fun generateUid(url: String): Long { protected fun generateUid(url: String): Long {
val uri = url.toHttpUrl() var h = 1125899906842597L
val x = source.name.hashCode() source.name.forEach { c ->
val y = "${uri.encodedPath}?${uri.query}".hashCode() h = 31 * h + c.toLong()
return (x.toLong() shl 32) or (y.toLong() and 0xffffffffL) }
url.forEach { c ->
h = 31 * h + c.toLong()
}
return h
} }
protected fun generateUid(id: Int): Long { protected fun generateUid(id: Long): Long {
val x = source.name.hashCode() var h = 1125899906842597L
return (x.toLong() shl 32) or (id.toLong() and 0xffffffffL) source.name.forEach { c ->
h = 31 * h + c.toLong()
}
h = 31 * h + id
return h
} }
} }

@ -13,8 +13,6 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
loaderContext loaderContext
) { ) {
protected abstract val defaultDomain: String
override val sortOrders: Set<SortOrder> = EnumSet.of( override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST, SortOrder.NEWEST,
SortOrder.POPULARITY, SortOrder.POPULARITY,
@ -27,7 +25,7 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
sortOrder: SortOrder?, sortOrder: SortOrder?,
tag: MangaTag? tag: MangaTag?
): List<Manga> { ): List<Manga> {
val domain = conf.getDomain(defaultDomain) val domain = getDomain()
val url = when { val url = when {
!query.isNullOrEmpty() -> { !query.isNullOrEmpty() -> {
if (offset != 0) { if (offset != 0) {
@ -44,9 +42,9 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
return root.select("div.content_row").mapNotNull { row -> return root.select("div.content_row").mapNotNull { row ->
val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a")
?: return@mapNotNull null ?: return@mapNotNull null
val href = a.attr("href").withDomain(domain) val href = a.relUrl("href")
Manga( Manga(
id = href.longHashCode(), id = generateUid(href),
url = href, url = href,
altTitle = a.attr("title"), altTitle = a.attr("title"),
title = a.text().substringAfterLast('(').substringBeforeLast(')'), title = a.text().substringAfterLast('(').substringBeforeLast(')'),
@ -55,7 +53,7 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
"/mangaka" "/mangaka"
).firstOrNull()?.text(), ).firstOrNull()?.text(),
coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img")
?.attr("src")?.withDomain(domain).orEmpty(), ?.absUrl("src").orEmpty(),
tags = runCatching { tags = runCatching {
row.selectFirst("div.genre")?.select("a")?.mapToSet { row.selectFirst("div.genre")?.select("a")?.mapToSet {
MangaTag( MangaTag(
@ -72,20 +70,18 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(defaultDomain) val doc = loaderContext.httpGet(manga.url.withDomain()).parseHtml()
val doc = loaderContext.httpGet(manga.url).parseHtml()
val root = val root =
doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root") doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root")
return manga.copy( return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.attr("src")?.withDomain(domain), largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha").flatMap { table -> chapters = root.select("table.table_cha").flatMap { table ->
table.select("div.manga2") table.select("div.manga2")
}.mapNotNull { it.selectFirst("a") }.reversed().mapIndexedNotNull { i, a -> }.map { it.selectFirst("a") }.reversed().mapIndexedNotNull { i, a ->
val href = a.attr("href") val href = a.relUrl("href")
?.withDomain(domain) ?: return@mapIndexedNotNull null
MangaChapter( MangaChapter(
id = href.longHashCode(), id = generateUid(href),
name = a.text().trim(), name = a.text().trim(),
number = i + 1, number = i + 1,
url = href, url = href,
@ -96,7 +92,8 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = loaderContext.httpGet(chapter.url).parseHtml() val fullUrl = chapter.url.withDomain()
val doc = loaderContext.httpGet(fullUrl).parseHtml()
val scripts = doc.select("script") val scripts = doc.select("script")
for (script in scripts) { for (script in scripts) {
val data = script.html() val data = script.html()
@ -106,13 +103,17 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
} }
val json = data.substring(pos).substringAfter('[').substringBefore(';') val json = data.substring(pos).substringAfter('[').substringBefore(';')
.substringBeforeLast(']') .substringBeforeLast(']')
val domain = getDomain()
return json.split(",").mapNotNull { return json.split(",").mapNotNull {
it.trim().removeSurrounding('"', '\'').takeUnless(String::isBlank) it.trim()
.removeSurrounding('"', '\'')
.toRelativeUrl(domain)
.takeUnless(String::isBlank)
}.map { url -> }.map { url ->
MangaPage( MangaPage(
id = url.longHashCode(), id = generateUid(url),
url = url, url = url,
referer = chapter.url, referer = fullUrl,
source = source source = source
) )
} }
@ -121,7 +122,7 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(defaultDomain) val domain = getDomain()
val doc = loaderContext.httpGet("https://$domain/catalog").parseHtml() val doc = loaderContext.httpGet("https://$domain/catalog").parseHtml()
val root = doc.body().selectFirst("div.main_fon").getElementById("side") val root = doc.body().selectFirst("div.main_fon").getElementById("side")
.select("ul").last() .select("ul").last()

@ -14,6 +14,8 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
override val source = MangaSource.DESUME override val source = MangaSource.DESUME
override val defaultDomain = "desu.me"
override val sortOrders: Set<SortOrder> = EnumSet.of( override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED, SortOrder.UPDATED,
SortOrder.POPULARITY, SortOrder.POPULARITY,
@ -27,7 +29,7 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
sortOrder: SortOrder?, sortOrder: SortOrder?,
tag: MangaTag? tag: MangaTag?
): List<Manga> { ): List<Manga> {
val domain = conf.getDomain(DOMAIN) val domain = getDomain()
val url = buildString { val url = buildString {
append("https://") append("https://")
append(domain) append(domain)
@ -51,8 +53,9 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
for (i in 0 until total) { for (i in 0 until total) {
val jo = json.getJSONObject(i) val jo = json.getJSONObject(i)
val cover = jo.getJSONObject("image") val cover = jo.getJSONObject("image")
val id = jo.getLong("id")
list += Manga( list += Manga(
url = jo.getString("url"), url = "/manga/api/$id",
source = MangaSource.DESUME, source = MangaSource.DESUME,
title = jo.getString("russian"), title = jo.getString("russian"),
altTitle = jo.getString("name"), altTitle = jo.getString("name"),
@ -63,7 +66,7 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
else -> null else -> null
}, },
rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f),
id = ID_MASK + jo.getLong("id"), id = generateUid(id),
description = jo.getString("description") description = jo.getString("description")
) )
} }
@ -71,10 +74,10 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(DOMAIN) val url = manga.url.withDomain()
val url = "https://$domain/manga/api/${manga.id - ID_MASK}"
val json = loaderContext.httpGet(url).parseJson().getJSONObject("response") val json = loaderContext.httpGet(url).parseJson().getJSONObject("response")
?: throw ParseException("Invalid response") ?: throw ParseException("Invalid response")
val baseChapterUrl = manga.url + "/chapter/"
return manga.copy( return manga.copy(
tags = json.getJSONArray("genres").mapToSet { tags = json.getJSONArray("genres").mapToSet {
MangaTag( MangaTag(
@ -87,9 +90,9 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
chapters = json.getJSONObject("chapters").getJSONArray("list").mapIndexed { i, it -> chapters = json.getJSONObject("chapters").getJSONArray("list").mapIndexed { i, it ->
val chid = it.getLong("id") val chid = it.getLong("id")
MangaChapter( MangaChapter(
id = ID_MASK + chid, id = generateUid(chid),
source = manga.source, source = manga.source,
url = "$url/chapter/$chid", url = "$baseChapterUrl$chid",
name = it.optString("title", "${manga.title} #${it.getDouble("ch")}"), name = it.optString("title", "${manga.title} #${it.getDouble("ch")}"),
number = i + 1 number = i + 1
) )
@ -98,21 +101,22 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val json = loaderContext.httpGet(chapter.url).parseJson().getJSONObject("response") val fullUrl = chapter.url.withDomain()
?: throw ParseException("Invalid response") val json = loaderContext.httpGet(fullUrl)
return json.getJSONObject("pages").getJSONArray("list").map { .parseJson()
.getJSONObject("response") ?: throw ParseException("Invalid response")
return json.getJSONObject("pages").getJSONArray("list").map { jo ->
MangaPage( MangaPage(
id = it.getLong("id"), id = generateUid(jo.getLong("id")),
referer = chapter.url, referer = fullUrl,
source = chapter.source, source = chapter.source,
url = it.getString("img") url = jo.getString("img")
) )
} }
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(DOMAIN) val doc = loaderContext.httpGet("https://${getDomain()}/manga/").parseHtml()
val doc = loaderContext.httpGet("https://$domain/manga/").parseHtml()
val root = doc.body().getElementById("animeFilter").selectFirst(".catalog-genres") val root = doc.body().getElementById("animeFilter").selectFirst(".catalog-genres")
return root.select("li").mapToSet { return root.select("li").mapToSet {
MangaTag( MangaTag(
@ -133,10 +137,4 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
SortOrder.NEWEST -> "id" SortOrder.NEWEST -> "id"
else -> "updated" else -> "updated"
} }
private companion object {
private const val ID_MASK = 1000
private const val DOMAIN = "desu.me"
}
} }

@ -13,8 +13,6 @@ import java.util.*
abstract class GroupleRepository(loaderContext: MangaLoaderContext) : abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
RemoteMangaRepository(loaderContext) { RemoteMangaRepository(loaderContext) {
protected abstract val defaultDomain: String
override val sortOrders: Set<SortOrder> = EnumSet.of( override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED, SortOrder.UPDATED,
SortOrder.POPULARITY, SortOrder.POPULARITY,
@ -28,13 +26,13 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
sortOrder: SortOrder?, sortOrder: SortOrder?,
tag: MangaTag? tag: MangaTag?
): List<Manga> { ): List<Manga> {
val domain = conf.getDomain(defaultDomain) val domain = getDomain()
val doc = when { val doc = when {
!query.isNullOrEmpty() -> loaderContext.httpPost( !query.isNullOrEmpty() -> loaderContext.httpPost(
"https://$domain/search", "https://$domain/search",
mapOf( mapOf(
"q" to query.urlEncoded(), "q" to query.urlEncoded(),
"offset" to offset.upBy(PAGE_SIZE_SEARCH).toString() "offset" to (offset upBy PAGE_SIZE_SEARCH).toString()
) )
) )
tag == null -> loaderContext.httpGet( tag == null -> loaderContext.httpGet(
@ -42,14 +40,14 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
getSortKey( getSortKey(
sortOrder sortOrder
) )
}&offset=${offset.upBy(PAGE_SIZE)}" }&offset=${offset upBy PAGE_SIZE}"
) )
else -> loaderContext.httpGet( else -> loaderContext.httpGet(
"https://$domain/list/genre/${tag.key}?sortType=${ "https://$domain/list/genre/${tag.key}?sortType=${
getSortKey( getSortKey(
sortOrder sortOrder
) )
}&offset=${offset.upBy(PAGE_SIZE)}" }&offset=${offset upBy PAGE_SIZE}"
) )
}.parseHtml() }.parseHtml()
val root = doc.body().getElementById("mangaBox") val root = doc.body().getElementById("mangaBox")
@ -68,9 +66,10 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text()
?: return@mapNotNull null ?: return@mapNotNull null
val tileInfo = descDiv.selectFirst("div.tile-info") val tileInfo = descDiv.selectFirst("div.tile-info")
val relUrl = href.toRelativeUrl(baseHost)
Manga( Manga(
id = href.longHashCode(), id = generateUid(relUrl),
url = href, url = relUrl,
title = title, title = title,
altTitle = descDiv.selectFirst("h4")?.text(), altTitle = descDiv.selectFirst("h4")?.text(),
coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original").orEmpty(), coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original").orEmpty(),
@ -103,8 +102,7 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(defaultDomain) val doc = loaderContext.httpGet(manga.url.withDomain()).parseHtml()
val doc = loaderContext.httpGet(manga.url).parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?: throw ParseException("Cannot find root") ?: throw ParseException("Cannot find root")
return manga.copy( return manga.copy(
@ -122,11 +120,10 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
) )
}, },
chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") chapters = root.selectFirst("div.chapters-link")?.selectFirst("table")
?.select("a")?.asReversed()?.mapIndexedNotNull { i, a -> ?.select("a")?.asReversed()?.mapIndexed { i, a ->
val href = val href = a.relUrl("href")
a.attr("href")?.withDomain(domain) ?: return@mapIndexedNotNull null
MangaChapter( MangaChapter(
id = href.longHashCode(), id = generateUid(href),
name = a.ownText().removePrefix(manga.title).trim(), name = a.ownText().removePrefix(manga.title).trim(),
number = i + 1, number = i + 1,
url = href, url = href,
@ -137,7 +134,7 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = loaderContext.httpGet(chapter.url + "?mtr=1").parseHtml() val doc = loaderContext.httpGet(chapter.url.withDomain() + "?mtr=1").parseHtml()
val scripts = doc.select("script") val scripts = doc.select("script")
for (script in scripts) { for (script in scripts) {
val data = script.html() val data = script.html()
@ -153,7 +150,7 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
val url = parts[0].value.removeSurrounding('"', '\'') + val url = parts[0].value.removeSurrounding('"', '\'') +
parts[2].value.removeSurrounding('"', '\'') parts[2].value.removeSurrounding('"', '\'')
MangaPage( MangaPage(
id = url.longHashCode(), id = generateUid(url),
url = url, url = url,
referer = chapter.url, referer = chapter.url,
source = source source = source
@ -164,8 +161,7 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(defaultDomain) val doc = loaderContext.httpGet("https://${getDomain()}/list/genres/sort_name").parseHtml()
val doc = loaderContext.httpGet("https://$domain/list/genres/sort_name").parseHtml()
val root = doc.body().getElementById("mangaBox").selectFirst("div.leftContent") val root = doc.body().getElementById("mangaBox").selectFirst("div.leftContent")
.selectFirst("table.table") .selectFirst("table.table")
return root.select("a.element-link").mapToSet { a -> return root.select("a.element-link").mapToSet { a ->

@ -3,10 +3,8 @@ package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.base.domain.MangaLoaderContext import org.koitharu.kotatsu.base.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.exceptions.ParseException import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.* import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.mapToSet import org.koitharu.kotatsu.utils.ext.mapToSet
import org.koitharu.kotatsu.utils.ext.parseHtml import org.koitharu.kotatsu.utils.ext.parseHtml
import org.koitharu.kotatsu.utils.ext.withDomain
class HenChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loaderContext) { class HenChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loaderContext) {
@ -30,14 +28,13 @@ class HenChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(load
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(defaultDomain) val doc = loaderContext.httpGet(manga.url.withDomain()).parseHtml()
val doc = loaderContext.httpGet(manga.url).parseHtml()
val root = val root =
doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root") doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root")
val readLink = manga.url.replace("manga", "online") val readLink = manga.url.replace("manga", "online")
return manga.copy( return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.attr("src")?.withDomain(domain), largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet { tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet {
val a = it.children().last() val a = it.children().last()
MangaTag( MangaTag(
@ -48,7 +45,7 @@ class HenChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(load
} ?: manga.tags, } ?: manga.tags,
chapters = listOf( chapters = listOf(
MangaChapter( MangaChapter(
id = readLink.longHashCode(), id = generateUid(readLink),
url = readLink, url = readLink,
source = source, source = source,
number = 1, number = 1,

@ -17,7 +17,7 @@ import kotlin.collections.ArrayList
open class MangaLibRepository(loaderContext: MangaLoaderContext) : open class MangaLibRepository(loaderContext: MangaLoaderContext) :
RemoteMangaRepository(loaderContext) { RemoteMangaRepository(loaderContext) {
protected open val defaultDomain = "mangalib.me" override val defaultDomain = "mangalib.me"
override val source = MangaSource.MANGALIB override val source = MangaSource.MANGALIB
@ -38,11 +38,10 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
if (!query.isNullOrEmpty()) { if (!query.isNullOrEmpty()) {
return search(query) return search(query)
} }
val domain = conf.getDomain(defaultDomain)
val page = (offset / 60f).toIntUp() val page = (offset / 60f).toIntUp()
val url = buildString { val url = buildString {
append("https://") append("https://")
append(domain) append(getDomain())
append("/manga-list?dir=") append("/manga-list?dir=")
append(getSortKey(sortOrder)) append(getSortKey(sortOrder))
append("&page=") append("&page=")
@ -57,11 +56,11 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
val items = root.selectFirst("div.media-cards-grid").select("div.media-card-wrap") val items = root.selectFirst("div.media-cards-grid").select("div.media-card-wrap")
return items.mapNotNull { card -> return items.mapNotNull { card ->
val a = card.selectFirst("a.media-card") ?: return@mapNotNull null val a = card.selectFirst("a.media-card") ?: return@mapNotNull null
val href = a.attr("href").withDomain(domain) val href = a.relUrl("href")
Manga( Manga(
id = href.longHashCode(), id = generateUid(href),
title = card.selectFirst("h3").text(), title = card.selectFirst("h3").text(),
coverUrl = a.attr("data-src").withDomain(domain), coverUrl = a.absUrl("data-src"),
altTitle = null, altTitle = null,
author = null, author = null,
rating = Manga.NO_RATING, rating = Manga.NO_RATING,
@ -76,11 +75,12 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
override fun onCreatePreferences() = arraySetOf(SourceSettings.KEY_DOMAIN) override fun onCreatePreferences() = arraySetOf(SourceSettings.KEY_DOMAIN)
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val doc = loaderContext.httpGet(manga.url + "?section=info").parseHtml() val fullUrl = manga.url.withDomain()
val doc = loaderContext.httpGet("$fullUrl?section=info").parseHtml()
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found") val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found")
val title = root.selectFirst("div.media-header__wrap")?.children() val title = root.selectFirst("div.media-header__wrap")?.children()
val info = root.selectFirst("div.media-content") val info = root.selectFirst("div.media-content")
val chaptersDoc = loaderContext.httpGet(manga.url + "?section=chapters").parseHtml() val chaptersDoc = loaderContext.httpGet("$fullUrl?section=chapters").parseHtml()
val scripts = chaptersDoc.select("script") val scripts = chaptersDoc.select("script")
var chapters: ArrayList<MangaChapter>? = null var chapters: ArrayList<MangaChapter>? = null
scripts@ for (script in scripts) { scripts@ for (script in scripts) {
@ -109,7 +109,7 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
} }
chapters.add( chapters.add(
MangaChapter( MangaChapter(
id = url.longHashCode(), id = generateUid(url),
url = url, url = url,
source = source, source = source,
number = total - i, number = total - i,
@ -144,7 +144,8 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = loaderContext.httpGet(chapter.url).parseHtml() val fullUrl = chapter.url.withDomain()
val doc = loaderContext.httpGet(fullUrl).parseHtml()
if (doc.location()?.endsWith("/register") == true) { if (doc.location()?.endsWith("/register") == true) {
throw AuthRequiredException("/login".inContextOf(doc)) throw AuthRequiredException("/login".inContextOf(doc))
} }
@ -170,9 +171,9 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
return pages.map { x -> return pages.map { x ->
val pageUrl = "$domain/$url${x.getString("u")}" val pageUrl = "$domain/$url${x.getString("u")}"
MangaPage( MangaPage(
id = pageUrl.longHashCode(), id = generateUid(pageUrl),
url = pageUrl, url = pageUrl,
referer = chapter.url, referer = fullUrl,
source = source source = source
) )
} }
@ -182,8 +183,7 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(defaultDomain) val url = "https://${getDomain()}/manga-list"
val url = "https://$domain/manga-list"
val doc = loaderContext.httpGet(url).parseHtml() val doc = loaderContext.httpGet(url).parseHtml()
val scripts = doc.body().select("script") val scripts = doc.body().select("script")
for (script in scripts) { for (script in scripts) {
@ -215,14 +215,15 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
} }
private suspend fun search(query: String): List<Manga> { private suspend fun search(query: String): List<Manga> {
val domain = conf.getDomain(defaultDomain) val domain = getDomain()
val json = loaderContext.httpGet("https://$domain/search?type=manga&q=$query") val json = loaderContext.httpGet("https://$domain/search?type=manga&q=$query")
.parseJsonArray() .parseJsonArray()
return json.map { jo -> return json.map { jo ->
val url = "https://$domain/${jo.getString("slug")}" val slug = jo.getString("slug")
val url = "https://$domain/$slug"
val covers = jo.getJSONObject("covers") val covers = jo.getJSONObject("covers")
Manga( Manga(
id = url.longHashCode(), id = generateUid(slug),
url = url, url = url,
title = jo.getString("rus_name"), title = jo.getString("rus_name"),
altTitle = jo.getString("name"), altTitle = jo.getString("name"),

@ -15,6 +15,8 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
override val source = MangaSource.MANGATOWN override val source = MangaSource.MANGATOWN
override val defaultDomain = "www.mangatown.com"
override val sortOrders: Set<SortOrder> = EnumSet.of( override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL,
SortOrder.RATING, SortOrder.RATING,
@ -28,9 +30,6 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
sortOrder: SortOrder?, sortOrder: SortOrder?,
tag: MangaTag? tag: MangaTag?
): List<Manga> { ): List<Manga> {
val domain = conf.getDomain(DOMAIN)
val ssl = conf.isUseSsl(false)
val scheme = if (ssl) "https" else "http"
val sortKey = when (sortOrder) { val sortKey = when (sortOrder) {
SortOrder.ALPHABETICAL -> "?name.az" SortOrder.ALPHABETICAL -> "?name.az"
SortOrder.RATING -> "?rating.za" SortOrder.RATING -> "?rating.za"
@ -43,29 +42,28 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
if (offset != 0) { if (offset != 0) {
return emptyList() return emptyList()
} }
"$scheme://$domain/search?name=${query.urlEncoded()}" "/search?name=${query.urlEncoded()}".withDomain()
} }
tag != null -> "$scheme://$domain/directory/${tag.key}/$page.htm$sortKey" tag != null -> "/directory/${tag.key}/$page.htm$sortKey".withDomain()
else -> "$scheme://$domain/directory/$page.htm$sortKey" else -> "/directory/$page.htm$sortKey".withDomain()
} }
val doc = loaderContext.httpGet(url).parseHtml() val doc = loaderContext.httpGet(url).parseHtml()
val root = doc.body().selectFirst("ul.manga_pic_list") val root = doc.body().selectFirst("ul.manga_pic_list")
?: throw ParseException("Root not found") ?: throw ParseException("Root not found")
return root.select("li").mapNotNull { li -> return root.select("li").mapNotNull { li ->
val a = li.selectFirst("a.manga_cover") val a = li.selectFirst("a.manga_cover")
val href = a.attr("href").withDomain(domain, ssl) val href = a.relUrl("href")
val views = li.select("p.view") val views = li.select("p.view")
val status = views.findOwnText { x -> x.startsWith("Status:") } val status = views.findOwnText { x -> x.startsWith("Status:") }
?.substringAfter(':')?.trim()?.toLowerCase(Locale.ROOT) ?.substringAfter(':')?.trim()?.toLowerCase(Locale.ROOT)
Manga( Manga(
id = href.longHashCode(), id = generateUid(href),
title = a.attr("title"), title = a.attr("title"),
coverUrl = a.selectFirst("img").attr("src"), coverUrl = a.selectFirst("img").absUrl("src"),
source = MangaSource.MANGATOWN, source = MangaSource.MANGATOWN,
altTitle = null, altTitle = null,
rating = li.selectFirst("p.score")?.selectFirst("b") rating = li.selectFirst("p.score")?.selectFirst("b")
?.ownText()?.toFloatOrNull()?.div(5f) ?: Manga.NO_RATING, ?.ownText()?.toFloatOrNull()?.div(5f) ?: Manga.NO_RATING,
largeCoverUrl = null,
author = views.findText { x -> x.startsWith("Author:") }?.substringAfter(':') author = views.findText { x -> x.startsWith("Author:") }?.substringAfter(':')
?.trim(), ?.trim(),
state = when (status) { state = when (status) {
@ -86,9 +84,7 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(DOMAIN) val doc = loaderContext.httpGet(manga.url.withDomain()).parseHtml()
val ssl = conf.isUseSsl(false)
val doc = loaderContext.httpGet(manga.url).parseHtml()
val root = doc.body().selectFirst("section.main") val root = doc.body().selectFirst("section.main")
?.selectFirst("div.article_content") ?: throw ParseException("Cannot find root") ?.selectFirst("div.article_content") ?: throw ParseException("Cannot find root")
val info = root.selectFirst("div.detail_info").selectFirst("ul") val info = root.selectFirst("div.detail_info").selectFirst("ul")
@ -106,11 +102,11 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
}.orEmpty(), }.orEmpty(),
description = info.getElementById("show")?.ownText(), description = info.getElementById("show")?.ownText(),
chapters = chaptersList?.mapIndexedNotNull { i, li -> chapters = chaptersList?.mapIndexedNotNull { i, li ->
val href = li.selectFirst("a").attr("href").withDomain(domain, ssl) val href = li.selectFirst("a").relUrl("href")
val name = li.select("span").filter { it.className().isEmpty() } val name = li.select("span").filter { it.className().isEmpty() }
.joinToString(" - ") { it.text() }.trim() .joinToString(" - ") { it.text() }.trim()
MangaChapter( MangaChapter(
id = href.longHashCode(), id = generateUid(href),
url = href, url = href,
source = MangaSource.MANGATOWN, source = MangaSource.MANGATOWN,
number = i + 1, number = i + 1,
@ -121,35 +117,31 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val domain = conf.getDomain(DOMAIN) val fullUrl = chapter.url.withDomain()
val ssl = conf.isUseSsl(false) val doc = loaderContext.httpGet(fullUrl).parseHtml()
val doc = loaderContext.httpGet(chapter.url).parseHtml()
val root = doc.body().selectFirst("div.page_select") val root = doc.body().selectFirst("div.page_select")
?: throw ParseException("Cannot find root") ?: throw ParseException("Cannot find root")
return root.selectFirst("select").select("option").mapNotNull { return root.selectFirst("select").select("option").mapNotNull {
val href = it.attr("value").withDomain(domain, ssl) val href = it.relUrl("value")
if (href.endsWith("featured.html")) { if (href.endsWith("featured.html")) {
return@mapNotNull null return@mapNotNull null
} }
MangaPage( MangaPage(
id = href.longHashCode(), id = generateUid(href),
url = href, url = href,
referer = chapter.url, referer = fullUrl,
source = MangaSource.MANGATOWN source = MangaSource.MANGATOWN
) )
} }
} }
override suspend fun getPageUrl(page: MangaPage): String { override suspend fun getPageUrl(page: MangaPage): String {
val domain = conf.getDomain(DOMAIN) val doc = loaderContext.httpGet(page.url.withDomain()).parseHtml()
val ssl = conf.isUseSsl(false) return doc.getElementById("image").absUrl("src")
val doc = loaderContext.httpGet(page.url).parseHtml()
return doc.getElementById("image").attr("src").withDomain(domain, ssl)
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(DOMAIN) val doc = loaderContext.httpGet("/directory/".withDomain()).parseHtml()
val doc = loaderContext.httpGet("http://$domain/directory/").parseHtml()
val root = doc.body().selectFirst("aside.right") val root = doc.body().selectFirst("aside.right")
.getElementsContainingOwnText("Genres") .getElementsContainingOwnText("Genres")
.first() .first()
@ -180,6 +172,5 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
@Language("RegExp") @Language("RegExp")
val TAG_REGEX = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") val TAG_REGEX = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+")
const val DOMAIN = "www.mangatown.com"
} }
} }

@ -15,6 +15,8 @@ class MangareadRepository(
override val source = MangaSource.MANGAREAD override val source = MangaSource.MANGAREAD
override val defaultDomain = "www.mangaread.org"
override val sortOrders: Set<SortOrder> = EnumSet.of( override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED, SortOrder.UPDATED,
SortOrder.POPULARITY SortOrder.POPULARITY
@ -29,7 +31,6 @@ class MangareadRepository(
if (offset % PAGE_SIZE != 0) { if (offset % PAGE_SIZE != 0) {
return emptyList() return emptyList()
} }
val domain = conf.getDomain(DOMAIN)
val payload = createRequestTemplate() val payload = createRequestTemplate()
payload["page"] = (offset / PAGE_SIZE).toString() payload["page"] = (offset / PAGE_SIZE).toString()
payload["vars[meta_key]"] = when (sortOrder) { payload["vars[meta_key]"] = when (sortOrder) {
@ -40,14 +41,14 @@ class MangareadRepository(
payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() payload["vars[wp-manga-genre]"] = tag?.key.orEmpty()
payload["vars[s]"] = query.orEmpty() payload["vars[s]"] = query.orEmpty()
val doc = loaderContext.httpPost( val doc = loaderContext.httpPost(
"https://${domain}/wp-admin/admin-ajax.php", "https://${getDomain()}/wp-admin/admin-ajax.php",
payload payload
).parseHtml() ).parseHtml()
return doc.select("div.row.c-tabs-item__content").map { div -> return doc.select("div.row.c-tabs-item__content").map { div ->
val href = div.selectFirst("a").absUrl("href") val href = div.selectFirst("a").relUrl("href")
val summary = div.selectFirst(".tab-summary") val summary = div.selectFirst(".tab-summary")
Manga( Manga(
id = href.longHashCode(), id = generateUid(href),
url = href, url = href,
coverUrl = div.selectFirst("img").attr("data-srcset") coverUrl = div.selectFirst("img").attr("data-srcset")
.split(',').firstOrNull()?.substringBeforeLast(' ').orEmpty(), .split(',').firstOrNull()?.substringBeforeLast(' ').orEmpty(),
@ -74,8 +75,7 @@ class MangareadRepository(
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(DOMAIN) val doc = loaderContext.httpGet("https://${getDomain()}/manga/").parseHtml()
val doc = loaderContext.httpGet("https://$domain/manga/").parseHtml()
val root = doc.body().selectFirst("header") val root = doc.body().selectFirst("header")
.selectFirst("ul.second-menu") .selectFirst("ul.second-menu")
return root.select("li").mapNotNullToSet { li -> return root.select("li").mapNotNullToSet { li ->
@ -94,8 +94,8 @@ class MangareadRepository(
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(DOMAIN) val fullUrl = manga.url.withDomain()
val doc = loaderContext.httpGet(manga.url).parseHtml() val doc = loaderContext.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.profile-manga") val root = doc.body().selectFirst("div.profile-manga")
?.selectFirst("div.summary_content") ?.selectFirst("div.summary_content")
?.selectFirst("div.post-content") ?.selectFirst("div.post-content")
@ -107,7 +107,7 @@ class MangareadRepository(
?.attr("data-postid")?.toLongOrNull() ?.attr("data-postid")?.toLongOrNull()
?: throw ParseException("Cannot obtain manga id") ?: throw ParseException("Cannot obtain manga id")
val doc2 = loaderContext.httpPost( val doc2 = loaderContext.httpPost(
"https://${domain}/wp-admin/admin-ajax.php", "https://${getDomain()}/wp-admin/admin-ajax.php",
mapOf( mapOf(
"action" to "manga_get_chapters", "action" to "manga_get_chapters",
"manga" to mangaId.toString() "manga" to mangaId.toString()
@ -129,9 +129,9 @@ class MangareadRepository(
?.joinToString { it.html() }, ?.joinToString { it.html() },
chapters = doc2.select("li").asReversed().mapIndexed { i, li -> chapters = doc2.select("li").asReversed().mapIndexed { i, li ->
val a = li.selectFirst("a") val a = li.selectFirst("a")
val href = a.absUrl("href") val href = a.relUrl("href")
MangaChapter( MangaChapter(
id = href.longHashCode(), id = generateUid(href),
name = a.ownText(), name = a.ownText(),
number = i + 1, number = i + 1,
url = href, url = href,
@ -142,17 +142,18 @@ class MangareadRepository(
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = loaderContext.httpGet(chapter.url).parseHtml() val fullUrl = chapter.url.withDomain()
val doc = loaderContext.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.main-col-inner") val root = doc.body().selectFirst("div.main-col-inner")
?.selectFirst("div.reading-content") ?.selectFirst("div.reading-content")
?: throw ParseException("Root not found") ?: throw ParseException("Root not found")
return root.select("div.page-break").map { div -> return root.select("div.page-break").map { div ->
val img = div.selectFirst("img") val img = div.selectFirst("img")
val url = img.absUrl("data-src") val url = img.relUrl("data-src")
MangaPage( MangaPage(
id = url.longHashCode(), id = generateUid(url),
url = url, url = url,
referer = chapter.url, referer = fullUrl,
source = MangaSource.MANGAREAD source = MangaSource.MANGAREAD
) )
} }
@ -163,7 +164,6 @@ class MangareadRepository(
private companion object { private companion object {
private const val PAGE_SIZE = 12 private const val PAGE_SIZE = 12
private const val DOMAIN = "www.mangaread.org"
private fun createRequestTemplate() = private fun createRequestTemplate() =
"action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5Borderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5Borderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border%5D=desc&vars%5Bmanga_archives_item_layout%5D=default"

@ -1,7 +1,6 @@
package org.koitharu.kotatsu.core.parser.site package org.koitharu.kotatsu.core.parser.site
import androidx.collection.arraySetOf import androidx.collection.arraySetOf
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONException import org.json.JSONException
import org.json.JSONObject import org.json.JSONObject
@ -18,6 +17,8 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
override val source = MangaSource.REMANGA override val source = MangaSource.REMANGA
override val defaultDomain = "remanga.org"
override val sortOrders: Set<SortOrder> = EnumSet.of( override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY, SortOrder.POPULARITY,
SortOrder.RATING, SortOrder.RATING,
@ -32,7 +33,7 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
sortOrder: SortOrder?, sortOrder: SortOrder?,
tag: MangaTag? tag: MangaTag?
): List<Manga> { ): List<Manga> {
val domain = conf.getDomain(DEFAULT_DOMAIN) val domain = getDomain()
val urlBuilder = StringBuilder() val urlBuilder = StringBuilder()
.append("https://api.") .append("https://api.")
.append(domain) .append(domain)
@ -40,20 +41,21 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
urlBuilder.append("/api/search/?query=") urlBuilder.append("/api/search/?query=")
.append(query.urlEncoded()) .append(query.urlEncoded())
} else { } else {
urlBuilder.append("/api/search/catalog/?page=") urlBuilder.append("/api/search/catalog/?ordering=")
.append("&ordering=")
.append(getSortKey(sortOrder)) .append(getSortKey(sortOrder))
if (tag != null) { if (tag != null) {
urlBuilder.append("&genres=" + tag.key) urlBuilder.append("&genres=" + tag.key)
} }
} }
urlBuilder.append((offset / PAGE_SIZE) + 1) urlBuilder
.append("&page=")
.append((offset / PAGE_SIZE) + 1)
.append("&count=") .append("&count=")
.append(PAGE_SIZE) .append(PAGE_SIZE)
val content = loaderContext.httpGet(urlBuilder.toString()).parseJson() val content = loaderContext.httpGet(urlBuilder.toString()).parseJson()
.getJSONArray("content") .getJSONArray("content")
return content.map { jo -> return content.map { jo ->
val url = "https://$domain/manga/${jo.getString("dir")}" val url = "/manga/${jo.getString("dir")}"
val img = jo.getJSONObject("img") val img = jo.getJSONObject("img")
Manga( Manga(
id = generateUid(url), id = generateUid(url),
@ -77,8 +79,9 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(DEFAULT_DOMAIN) val domain = getDomain()
val slug = manga.url.toHttpUrl().pathSegments.last() val slug = manga.url.find(LAST_URL_PATH_REGEX)
?: throw ParseException("Cannot obtain slug from ${manga.url}")
val data = loaderContext.httpGet( val data = loaderContext.httpGet(
url = "https://api.$domain/api/titles/$slug/" url = "https://api.$domain/api/titles/$slug/"
).parseJson() ).parseJson()
@ -110,7 +113,7 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
val id = jo.getLong("id") val id = jo.getLong("id")
val name = jo.getString("name") val name = jo.getString("name")
MangaChapter( MangaChapter(
id = generateUid(id.toInt()), id = generateUid(id),
url = "https://api.$domain/api/titles/chapters/$id/", url = "https://api.$domain/api/titles/chapters/$id/",
number = chapters.length() - i, number = chapters.length() - i,
name = buildString { name = buildString {
@ -128,8 +131,8 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val referer = "https://${conf.getDomain(DEFAULT_DOMAIN)}/" val referer = "https://${getDomain()}/"
val content = loaderContext.httpGet(chapter.url).parseJson() val content = loaderContext.httpGet(chapter.url.withDomain()).parseJson()
.getJSONObject("content").getJSONArray("pages") .getJSONObject("content").getJSONArray("pages")
val pages = ArrayList<MangaPage>(content.length()) val pages = ArrayList<MangaPage>(content.length())
for (i in 0 until content.length()) { for (i in 0 until content.length()) {
@ -143,7 +146,7 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(DEFAULT_DOMAIN) val domain = getDomain()
val content = loaderContext.httpGet("https://api.$domain/api/forms/titles/?get=genres") val content = loaderContext.httpGet("https://api.$domain/api/forms/titles/?get=genres")
.parseJson().getJSONObject("content").getJSONArray("genres") .parseJson().getJSONObject("content").getJSONArray("genres")
return content.mapToSet { jo -> return content.mapToSet { jo ->
@ -166,7 +169,7 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
} }
private fun parsePage(jo: JSONObject, referer: String) = MangaPage( private fun parsePage(jo: JSONObject, referer: String) = MangaPage(
id = generateUid(jo.getLong("id").toInt()), id = generateUid(jo.getLong("id")),
url = jo.getString("link"), url = jo.getString("link"),
referer = referer, referer = referer,
source = source source = source
@ -175,9 +178,10 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
private companion object { private companion object {
const val PAGE_SIZE = 30 const val PAGE_SIZE = 30
const val DEFAULT_DOMAIN = "remanga.org"
const val STATUS_ONGOING = 1 const val STATUS_ONGOING = 1
const val STATUS_FINISHED = 0 const val STATUS_FINISHED = 0
val LAST_URL_PATH_REGEX = Regex("/[^/]+/?$")
} }
} }

@ -5,9 +5,8 @@ import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.Manga import org.koitharu.kotatsu.core.model.Manga
import org.koitharu.kotatsu.core.model.MangaChapter import org.koitharu.kotatsu.core.model.MangaChapter
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.utils.ext.longHashCode
import org.koitharu.kotatsu.utils.ext.parseHtml import org.koitharu.kotatsu.utils.ext.parseHtml
import org.koitharu.kotatsu.utils.ext.withDomain import org.koitharu.kotatsu.utils.ext.relUrl
class YaoiChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loaderContext) { class YaoiChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loaderContext) {
@ -15,20 +14,18 @@ class YaoiChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(loa
override val defaultDomain = "yaoi-chan.me" override val defaultDomain = "yaoi-chan.me"
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val domain = conf.getDomain(defaultDomain) val doc = loaderContext.httpGet(manga.url.withDomain()).parseHtml()
val doc = loaderContext.httpGet(manga.url).parseHtml()
val root = val root =
doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root") doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root")
return manga.copy( return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.attr("src")?.withDomain(domain), largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha").flatMap { table -> chapters = root.select("table.table_cha").flatMap { table ->
table.select("div.manga") table.select("div.manga")
}.mapNotNull { it.selectFirst("a") }.reversed().mapIndexedNotNull { i, a -> }.mapNotNull { it.selectFirst("a") }.reversed().mapIndexed { i, a ->
val href = a.attr("href") val href = a.relUrl("href")
?.withDomain(domain) ?: return@mapIndexedNotNull null
MangaChapter( MangaChapter(
id = href.longHashCode(), id = generateUid(href),
name = a.text().trim(), name = a.text().trim(),
number = i + 1, number = i + 1,
url = href, url = href,

@ -217,7 +217,7 @@ class ReaderViewModel(
private suspend fun loadChapter(chapterId: Long): List<ReaderPage> { private suspend fun loadChapter(chapterId: Long): List<ReaderPage> {
val manga = checkNotNull(mangaData.value) { "Manga is null" } val manga = checkNotNull(mangaData.value) { "Manga is null" }
val chapter = checkNotNull(chapters.get(chapterId)) { "Chapter $chapterId not found" } val chapter = checkNotNull(chapters[chapterId]) { "Requested chapter not found" }
val repo = manga.source.repository val repo = manga.source.repository
return repo.getPages(chapter).mapIndexed { index, page -> return repo.getPages(chapter).mapIndexed { index, page ->
ReaderPage.from(page, index, chapterId) ReaderPage.from(page, index, chapterId)

@ -7,6 +7,7 @@ import org.json.JSONObject
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.internal.StringUtil import org.jsoup.internal.StringUtil
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.nodes.Node import org.jsoup.nodes.Node
import org.jsoup.select.Elements import org.jsoup.select.Elements
@ -70,3 +71,24 @@ fun String.inContextOf(node: Node): String {
StringUtil.resolve(node.baseUri(), this) StringUtil.resolve(node.baseUri(), this)
} }
} }
fun String.toRelativeUrl(domain: String): String {
if (isEmpty() || startsWith("/")) {
return this
}
return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/")
}
fun Element.relUrl(attributeKey: String): String {
val attr = attr(attributeKey)
if (attr.isEmpty()) {
return ""
}
if (attr.startsWith("/")) {
return attr
}
val baseUrl = REGEX_URL_BASE.find(baseUri())?.value ?: return attr
return attr.removePrefix(baseUrl.dropLast(1))
}
private val REGEX_URL_BASE = Regex("^[^/]{2,6}://[^/]+/", RegexOption.IGNORE_CASE)

@ -33,7 +33,7 @@ fun Float.toIntUp(): Int {
} }
} }
fun Int.upBy(step: Int): Int { infix fun Int.upBy(step: Int): Int {
val mod = this % step val mod = this % step
return if (mod == this) { return if (mod == this) {
this this

@ -16,28 +16,6 @@ fun String.longHashCode(): Long {
return h return h
} }
@Deprecated("Use String.inContextOf")
fun String.withDomain(domain: String, ssl: Boolean = true) = when {
this.startsWith("//") -> buildString {
append("http")
if (ssl) {
append('s')
}
append(":")
append(this@withDomain)
}
this.startsWith("/") -> buildString {
append("http")
if (ssl) {
append('s')
}
append("://")
append(domain)
append(this@withDomain)
}
else -> this
}
fun String.removeSurrounding(vararg chars: Char): String { fun String.removeSurrounding(vararg chars: Char): String {
if (length == 0) { if (length == 0) {
return this return this
@ -125,3 +103,5 @@ fun String.substringBetween(from: String, to: String, fallbackValue: String): St
substring(fromIndex + from.length, toIndex) substring(fromIndex + from.length, toIndex)
} }
} }
fun String.find(regex: Regex) = regex.find(this)?.value

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<PreferenceScreen xmlns:android="http://schemas.android.com/apk/res/android" <PreferenceScreen
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"> xmlns:app="http://schemas.android.com/apk/res-auto">
<EditTextPreference <EditTextPreference
@ -8,6 +9,7 @@
app:iconSpaceReserved="false" /> app:iconSpaceReserved="false" />
<SwitchPreferenceCompat <SwitchPreferenceCompat
android:defaultValue="true"
android:key="ssl" android:key="ssl"
android:title="@string/use_ssl" android:title="@string/use_ssl"
app:iconSpaceReserved="false" /> app:iconSpaceReserved="false" />

@ -15,6 +15,7 @@ import org.koin.test.get
import org.koitharu.kotatsu.base.domain.MangaLoaderContext import org.koitharu.kotatsu.base.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.network.UserAgentInterceptor import org.koitharu.kotatsu.core.network.UserAgentInterceptor
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.core.prefs.SourceSettings import org.koitharu.kotatsu.core.prefs.SourceSettings
import org.koitharu.kotatsu.utils.AssertX import org.koitharu.kotatsu.utils.AssertX
import org.koitharu.kotatsu.utils.ext.isDistinctBy import org.koitharu.kotatsu.utils.ext.isDistinctBy
@ -28,7 +29,7 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
.newInstance(get<MangaLoaderContext>()) .newInstance(get<MangaLoaderContext>())
} catch (e: NoSuchMethodException) { } catch (e: NoSuchMethodException) {
source.cls.newInstance() source.cls.newInstance()
} } as RemoteMangaRepository
@Test @Test
fun list() { fun list() {
@ -36,13 +37,9 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
Assert.assertFalse("List is empty", list.isEmpty()) Assert.assertFalse("List is empty", list.isEmpty())
Assert.assertTrue("Mangas are not distinct", list.isDistinctBy { it.id }) Assert.assertTrue("Mangas are not distinct", list.isDistinctBy { it.id })
val item = list.random() val item = list.random()
AssertX.assertUrlRelative("Url is not relative", item.url)
AssertX.assertUrlAbsolute("Url is not absolute", item.coverUrl)
AssertX.assertContentType("Bad cover at ${item.url}", item.coverUrl, "image/*") AssertX.assertContentType("Bad cover at ${item.url}", item.coverUrl, "image/*")
AssertX.assertContentType(
"Wrong content type at ${item.url}",
item.url,
"text/html",
"application/json"
)
Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank()) Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank())
} }
@ -52,13 +49,8 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
Assert.assertFalse("List is empty", list.isEmpty()) Assert.assertFalse("List is empty", list.isEmpty())
Assert.assertTrue("Mangas are not distinct", list.isDistinctBy { it.id }) Assert.assertTrue("Mangas are not distinct", list.isDistinctBy { it.id })
val item = list.random() val item = list.random()
AssertX.assertUrlRelative("Url is not relative", item.url)
AssertX.assertContentType("Bad cover at ${item.url}", item.coverUrl, "image/*") AssertX.assertContentType("Bad cover at ${item.url}", item.coverUrl, "image/*")
AssertX.assertContentType(
"Wrong content type at ${item.url}",
item.url,
"text/html",
"application/json"
)
Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank()) Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank())
} }
@ -72,13 +64,8 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
val list = runBlocking { repo.getList(0, tag = tag) } val list = runBlocking { repo.getList(0, tag = tag) }
Assert.assertFalse("List is empty", list.isEmpty()) Assert.assertFalse("List is empty", list.isEmpty())
val item = list.random() val item = list.random()
AssertX.assertUrlRelative("Url is not relative", item.url)
AssertX.assertContentType("Bad cover at ${item.coverUrl}", item.coverUrl, "image/*") AssertX.assertContentType("Bad cover at ${item.coverUrl}", item.coverUrl, "image/*")
AssertX.assertContentType(
"Wrong response from ${item.url}",
item.url,
"text/html",
"application/json"
)
Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank()) Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank())
} }
@ -86,7 +73,7 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
fun details() { fun details() {
val manga = runBlocking { repo.getList(0) }.random() val manga = runBlocking { repo.getList(0) }.random()
val details = runBlocking { repo.getDetails(manga) } val details = runBlocking { repo.getDetails(manga) }
Assert.assertFalse("Chapter is empty at ${details.url}", details.chapters.isNullOrEmpty()) Assert.assertFalse("No chapters at ${details.url}", details.chapters.isNullOrEmpty())
Assert.assertFalse( Assert.assertFalse(
"Description is empty at ${details.url}", "Description is empty at ${details.url}",
details.description.isNullOrEmpty() details.description.isNullOrEmpty()
@ -95,16 +82,11 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
"Chapters are not distinct", "Chapters are not distinct",
details.chapters.orEmpty().isDistinctBy { it.id }) details.chapters.orEmpty().isDistinctBy { it.id })
val chapter = details.chapters?.randomOrNull() ?: return val chapter = details.chapters?.randomOrNull() ?: return
AssertX.assertUrlRelative("Url is not relative", chapter.url)
Assert.assertFalse( Assert.assertFalse(
"Chapter name missing at ${details.url}:${chapter.number}", "Chapter name missing at ${details.url}:${chapter.number}",
chapter.name.isBlank() chapter.name.isBlank()
) )
AssertX.assertContentType(
"Chapter response wrong at ${chapter.url}",
chapter.url,
"text/html",
"application/json"
)
} }
@Test @Test

@ -6,6 +6,7 @@ import org.junit.Assert
import org.koin.core.component.KoinComponent import org.koin.core.component.KoinComponent
import org.koin.core.component.inject import org.koin.core.component.inject
import java.net.HttpURLConnection import java.net.HttpURLConnection
import java.net.URI
object AssertX : KoinComponent { object AssertX : KoinComponent {
@ -30,4 +31,12 @@ object AssertX : KoinComponent {
} }
} }
fun assertUrlRelative(message: String, url: String) {
Assert.assertFalse(message, URI(url).isAbsolute)
}
fun assertUrlAbsolute(message: String, url: String) {
Assert.assertTrue(message, URI(url).isAbsolute)
}
} }
Loading…
Cancel
Save