From 56b529331eb9b73d2ca0e00f49d21be2770ddc18 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 1 Mar 2020 20:48:59 +0200 Subject: [PATCH] Mangachan parsers --- .../kotatsu/core/model/MangaSource.kt | 9 +- .../core/parser/site/ChanRepository.kt | 137 ++++++++++++++++++ .../core/parser/site/GroupleRepository.kt | 2 +- .../core/parser/site/HenChanRepository.kt | 10 ++ .../core/parser/site/MangaChanRepository.kt | 10 ++ .../core/parser/site/YaoiChanRepository.kt | 10 ++ .../kotatsu/ui/details/ChaptersFragment.kt | 2 +- .../ui/details/MangaDetailsActivity.kt | 2 +- .../ui/details/MangaDetailsFragment.kt | 2 +- .../ui/details/MangaDetailsPresenter.kt | 2 +- .../kotatsu/ui/details/MangaDetailsView.kt | 2 +- .../koitharu/kotatsu/utils/ext/StringExt.kt | 6 +- 12 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/HenChanRepository.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaChanRepository.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/YaoiChanRepository.kt diff --git a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt index d8b5ebcfc..a8e992914 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt @@ -4,9 +4,7 @@ import android.os.Parcelable import kotlinx.android.parcel.Parcelize import org.koitharu.kotatsu.core.parser.LocalMangaRepository import org.koitharu.kotatsu.core.parser.MangaRepository -import org.koitharu.kotatsu.core.parser.site.MintMangaRepository -import org.koitharu.kotatsu.core.parser.site.ReadmangaRepository -import org.koitharu.kotatsu.core.parser.site.SelfMangaRepository +import org.koitharu.kotatsu.core.parser.site.* @Suppress("SpellCheckingInspection") @Parcelize @@ -18,5 +16,8 @@ enum class MangaSource( LOCAL("Local", null, LocalMangaRepository::class.java), READMANGA_RU("ReadManga", "ru", ReadmangaRepository::class.java), MINTMANGA("MintManga", "ru", MintMangaRepository::class.java), - SELFMANGA("SelfManga", "ru", SelfMangaRepository::class.java) + SELFMANGA("SelfManga", "ru", SelfMangaRepository::class.java), + MANGACHAN("Манга-тян", "ru", MangaChanRepository::class.java), + HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java), + YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java) } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt new file mode 100644 index 000000000..2b5f963bd --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt @@ -0,0 +1,137 @@ +package org.koitharu.kotatsu.core.parser.site + +import org.koitharu.kotatsu.core.exceptions.ParseException +import org.koitharu.kotatsu.core.model.* +import org.koitharu.kotatsu.core.parser.BaseMangaRepository +import org.koitharu.kotatsu.domain.MangaLoaderContext +import org.koitharu.kotatsu.utils.ext.* + +abstract class ChanRepository( + private val source: MangaSource, + loaderContext: MangaLoaderContext +) : BaseMangaRepository(loaderContext) { + + protected abstract val domain: String + + override val sortOrders = setOf(SortOrder.NEWEST, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) + + override suspend fun getList( + offset: Int, + query: String?, + sortOrder: SortOrder?, + tag: MangaTag? + ): List { + val url = when { + query != null -> "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" + tag != null -> "https://$domain/tags/${tag.key}&n=${getSortKey2(sortOrder)}?offset=$offset" + else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" + } + val doc = loaderContext.get(url).parseHtml() + val root = doc.body().selectFirst("div.main_fon").getElementById("content") + ?: throw ParseException("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("a.title_link") + ?: return@mapNotNull null + val href = a.attr("href").withDomain(domain) + Manga( + id = href.longHashCode(), + url = href, + altTitle = a.attr("title"), + title = a.text().substringAfterLast('(').substringBeforeLast(')'), + author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka" + ).firstOrNull()?.text(), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.attr("src")?.withDomain(domain).orEmpty(), + tags = safe { + row.selectFirst("div.genre")?.select("a")?.map { + MangaTag( + title = it.text(), + key = it.attr("href").substringAfterLast('/').urlEncoded(), + source = source + ) + }?.toSet() + }.orEmpty(), + source = source + ) + } + } + + + override suspend fun getDetails(manga: Manga): Manga { + val doc = loaderContext.get(manga.url).parseHtml() + val root = + doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + table.select("div.manga2") + }.mapNotNull { it.selectFirst("a") }.mapIndexedNotNull { i, a -> + val href = a.attr("href") + ?.withDomain(domain) ?: return@mapIndexedNotNull null + MangaChapter( + id = href.longHashCode(), + name = a.text().trim(), + number = i + 1, + url = href, + source = source + ) + } + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = loaderContext.get(chapter.url).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + return json.split(",").map { + val url = it.trim().removeSurrounding('"') + MangaPage( + id = url.longHashCode(), + url = url, + source = source + ) + } + } + throw ParseException("Pages list not found at ${chapter.url}") + } + + override suspend fun getTags(): Set { + val doc = loaderContext.get("https://$domain/catalog").parseHtml() + val root = doc.body().selectFirst("div.main_fon").getElementById("side") + .select("ul").last() + return root.select("li.sidetag").map { li -> + val a = li.children().last() + MangaTag( + title = a.text().capitalize(), + key = a.attr("href").substringAfterLast('/'), + source = source + ) + }.toSet() + } + + private fun getSortKey(sortOrder: SortOrder?) = + when (sortOrder ?: sortOrders.minBy { it.ordinal }) { + SortOrder.ALPHABETICAL -> "catalog" + SortOrder.POPULARITY -> "mostfavorites" + SortOrder.NEWEST -> "manga/new" + else -> "mostfavorites" + } + + private fun getSortKey2(sortOrder: SortOrder?) = + when (sortOrder ?: sortOrders.minBy { it.ordinal }) { + SortOrder.ALPHABETICAL -> "abcasc" + SortOrder.POPULARITY -> "favdesc" + SortOrder.NEWEST -> "datedesc" + else -> "favdesc" + } +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt index 3997b0fbd..eb7df7140 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt @@ -87,7 +87,7 @@ abstract class GroupleRepository( override suspend fun getDetails(manga: Manga): Manga { val doc = loaderContext.get(manga.url).parseHtml() - val root = doc.body().getElementById("mangaBox") + val root = doc.body().getElementById("mangaBox") ?: throw ParseException("Cannot find root") return manga.copy( description = root.selectFirst("div.manga-description").firstChild()?.html(), largeCoverUrl = root.selectFirst("div.subject-cower")?.selectFirst("img")?.attr( diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/HenChanRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/HenChanRepository.kt new file mode 100644 index 000000000..7994f38ff --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/HenChanRepository.kt @@ -0,0 +1,10 @@ +package org.koitharu.kotatsu.core.parser.site + +import org.koitharu.kotatsu.core.model.MangaSource +import org.koitharu.kotatsu.domain.MangaLoaderContext + +class HenChanRepository(loaderContext: MangaLoaderContext) : + ChanRepository(MangaSource.HENCHAN, loaderContext) { + + override val domain: String = "h-chan.me" +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaChanRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaChanRepository.kt new file mode 100644 index 000000000..30984d55f --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaChanRepository.kt @@ -0,0 +1,10 @@ +package org.koitharu.kotatsu.core.parser.site + +import org.koitharu.kotatsu.core.model.MangaSource +import org.koitharu.kotatsu.domain.MangaLoaderContext + +class MangaChanRepository(loaderContext: MangaLoaderContext) : + ChanRepository(MangaSource.MANGACHAN, loaderContext) { + + override val domain: String = "manga-chan.me" +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/YaoiChanRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/YaoiChanRepository.kt new file mode 100644 index 000000000..9d44ca31c --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/YaoiChanRepository.kt @@ -0,0 +1,10 @@ +package org.koitharu.kotatsu.core.parser.site + +import org.koitharu.kotatsu.core.model.MangaSource +import org.koitharu.kotatsu.domain.MangaLoaderContext + +class YaoiChanRepository(loaderContext: MangaLoaderContext) : + ChanRepository(MangaSource.YAOICHAN, loaderContext) { + + override val domain: String = "yaoi-chan.me" +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/ui/details/ChaptersFragment.kt b/app/src/main/java/org/koitharu/kotatsu/ui/details/ChaptersFragment.kt index e22837d4f..df3bec7c1 100644 --- a/app/src/main/java/org/koitharu/kotatsu/ui/details/ChaptersFragment.kt +++ b/app/src/main/java/org/koitharu/kotatsu/ui/details/ChaptersFragment.kt @@ -49,7 +49,7 @@ class ChaptersFragment : BaseFragment(R.layout.fragment_chapters), MangaDetailsV progressBar.isVisible = isLoading } - override fun onError(e: Exception) = Unit //handled in activity + override fun onError(e: Throwable) = Unit //handled in activity override fun onMangaRemoved(manga: Manga) = Unit //handled in activity diff --git a/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsActivity.kt b/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsActivity.kt index fa633bbf8..378488cd6 100644 --- a/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsActivity.kt +++ b/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsActivity.kt @@ -68,7 +68,7 @@ class MangaDetailsActivity : BaseActivity(), MangaDetailsView { finish() } - override fun onError(e: Exception) { + override fun onError(e: Throwable) { Snackbar.make(pager, e.getDisplayMessage(resources), Snackbar.LENGTH_LONG).show() } diff --git a/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsFragment.kt b/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsFragment.kt index 411b64cdb..3744d21f6 100644 --- a/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsFragment.kt +++ b/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsFragment.kt @@ -82,7 +82,7 @@ class MangaDetailsFragment : BaseFragment(R.layout.fragment_details), MangaDetai progressBar.isVisible = isLoading } - override fun onError(e: Exception) = Unit //handled in activity + override fun onError(e: Throwable) = Unit //handled in activity override fun onMangaRemoved(manga: Manga) = Unit //handled in activity diff --git a/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsPresenter.kt b/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsPresenter.kt index 5d3aeb06b..3376c25dd 100644 --- a/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsPresenter.kt +++ b/app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsPresenter.kt @@ -52,7 +52,7 @@ class MangaDetailsPresenter private constructor() : BasePresenter maxLength) { this.take(maxLength - 1) + Typography.ellipsis -} else this \ No newline at end of file +} else this + +fun String.urlEncoded(): String = URLEncoder.encode(this, Charsets.UTF_8.name()) \ No newline at end of file