From 83dec28ff07f2196dc1818bd1808556f524774b2 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sat, 21 May 2022 01:25:52 +0300 Subject: [PATCH 1/3] [New source] Nicovideo Seiga --- .../parsers/site/NicovideoSeigaParser.kt | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt new file mode 100644 index 00000000..3f324b27 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -0,0 +1,132 @@ +package org.koitharu.kotatsu.parsers.site + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("NICOVIDEOSEIGA", "Nicovideo Seiga", "ja") +class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.NICOVIDEOSEIGA), + MangaParserAuthProvider { + + override val authUrl: String + get() = "https://account.nicovideo.jp/login?site=seiga" + + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(getDomain()).any { + it.name == "user_session" + } + } + + override suspend fun getUsername(): String { + val body = context.httpGet("https://nicovideo.jp/my").parseHtml().body() + return "Nicovideo User" // TODO Figure out + } + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("seiga.nicovideo.jp", null) + + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder?, + ): List { + val page = (offset / 20f).toIntUp().inc() + val url = "/manga/list?page=$page&sort=${getSortKey(sortOrder)}".withDomain() + val doc = context.httpGet(url).parseHtml() + val comicList = doc.body().select("#comic_list > ul > li") ?: parseFailed("Container not found") + val items = comicList.select("div > .description > div > div") + return items.mapNotNull { item -> + val href = item.select(".comic_icon > div > a").attr("href") ?: return@mapNotNull null + val statusText = item.select(".mg_description_header > .mg_icon > .content_status > span").text() + Manga( + id = generateUid(href), + title = item.select(".mg_body > .title > a").text() ?: return@mapNotNull null, + coverUrl = item.select(".comic_icon > div > a > img").attr("src"), + altTitle = null, + author = item.select(".mg_description_header > .mg_author > a").text() ?: return@mapNotNull null, + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = emptySet(), + state = when (statusText) { + "連載" -> MangaState.ONGOING + "完結" -> MangaState.FINISHED + else -> null + }, + publicUrl = href.toAbsoluteUrl(item.host ?: getDomain()), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = context.httpGet(manga.url.withDomain()).parseHtml() + val contents = doc.body().select("#contents") ?: parseFailed("Cannot find root") + val statusText = contents.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span").text() + return manga.copy( + description = contents.select("div.mg_work_detail > div > div.row > div.description_text").text(), + largeCoverUrl = contents.select("div.primaries > div.main_visual > a > img").attr("src"), + state = when (statusText) { + "連載" -> MangaState.ONGOING + "完結" -> MangaState.FINISHED + else -> null + }, + chapters = contents.select("#episode_list > ul > li").mapIndexedNotNull { i, li -> + val href = li.select("div > div.description > div.title > a").attr("href").withDomain() + MangaChapter( + id = generateUid(href), + name = li.select("div > div.description > div.title > a").text(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = 0, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.withDomain() + val doc = context.httpGet(fullUrl).parseHtml() + if (!doc.select("#login_manga").isEmpty()) + throw AuthRequiredException(source) + val root = doc.body().select("#page_contents > li") + ?: throw ParseException("Root not found") + return root.map { li -> + val url = li.select("div > img").attr("data-original") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + return emptySet() + } + + private fun getSortKey(sortOrder: SortOrder?) = + when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) { + SortOrder.POPULARITY -> "manga_view" + SortOrder.UPDATED -> "manga_updated" + else -> "rate" + } +} \ No newline at end of file From 5b35441bf2db45abb596ce5ae1dd811cbb1d147a Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sat, 21 May 2022 15:57:47 +0300 Subject: [PATCH 2/3] Some adjustments Co-authored-by: Koitharu --- .../parsers/site/NicovideoSeigaParser.kt | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt index 3f324b27..8c95e216 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -6,11 +6,13 @@ import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.AuthRequiredException -import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.util.* +private const val STATUS_ONGOING = "連載" +private const val STATUS_FINISHED = "完結" + @MangaSourceParser("NICOVIDEOSEIGA", "Nicovideo Seiga", "ja") class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.NICOVIDEOSEIGA), MangaParserAuthProvider { @@ -26,8 +28,8 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars } override suspend fun getUsername(): String { - val body = context.httpGet("https://nicovideo.jp/my").parseHtml().body() - return "Nicovideo User" // TODO Figure out + val body = context.httpGet("https://app.nicovideo.jp/my/apps").parseHtml().body() + return body.selectFirst("#userinfo > div > div > strong")?.text() ?: parseFailed("Cannot get username") } override val sortOrders: Set = EnumSet.of( @@ -44,26 +46,29 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars sortOrder: SortOrder?, ): List { val page = (offset / 20f).toIntUp().inc() + if (!query.isNullOrEmpty()) { + return if (offset == 0) getSearchList(query, page) else emptyList() + } val url = "/manga/list?page=$page&sort=${getSortKey(sortOrder)}".withDomain() val doc = context.httpGet(url).parseHtml() val comicList = doc.body().select("#comic_list > ul > li") ?: parseFailed("Container not found") val items = comicList.select("div > .description > div > div") return items.mapNotNull { item -> - val href = item.select(".comic_icon > div > a").attr("href") ?: return@mapNotNull null - val statusText = item.select(".mg_description_header > .mg_icon > .content_status > span").text() + val href = item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() Manga( id = generateUid(href), - title = item.select(".mg_body > .title > a").text() ?: return@mapNotNull null, - coverUrl = item.select(".comic_icon > div > a > img").attr("src"), + title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), altTitle = null, - author = item.select(".mg_description_header > .mg_author > a").text() ?: return@mapNotNull null, + author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), rating = RATING_UNKNOWN, url = href, isNsfw = false, tags = emptySet(), state = when (statusText) { - "連載" -> MangaState.ONGOING - "完結" -> MangaState.FINISHED + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED else -> null }, publicUrl = href.toAbsoluteUrl(item.host ?: getDomain()), @@ -74,18 +79,18 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars override suspend fun getDetails(manga: Manga): Manga { val doc = context.httpGet(manga.url.withDomain()).parseHtml() - val contents = doc.body().select("#contents") ?: parseFailed("Cannot find root") + val contents = doc.body().selectFirst("#contents") ?: parseFailed("Cannot find root") val statusText = contents.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span").text() return manga.copy( - description = contents.select("div.mg_work_detail > div > div.row > div.description_text").text(), - largeCoverUrl = contents.select("div.primaries > div.main_visual > a > img").attr("src"), + description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), + largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img")?.attrAsAbsoluteUrlOrNull("src"), state = when (statusText) { - "連載" -> MangaState.ONGOING - "完結" -> MangaState.FINISHED + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED else -> null }, chapters = contents.select("#episode_list > ul > li").mapIndexedNotNull { i, li -> - val href = li.select("div > div.description > div.title > a").attr("href").withDomain() + val href = li.selectFirst("div > div.description > div.title > a")?.attrAsRelativeUrl("href") ?: parseFailed() MangaChapter( id = generateUid(href), name = li.select("div > div.description > div.title > a").text(), @@ -106,7 +111,6 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars if (!doc.select("#login_manga").isEmpty()) throw AuthRequiredException(source) val root = doc.body().select("#page_contents > li") - ?: throw ParseException("Root not found") return root.map { li -> val url = li.select("div > img").attr("data-original") MangaPage( @@ -123,10 +127,35 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars return emptySet() } + private suspend fun getSearchList(query: String, page: Int): List { + val domain = getDomain() + val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() + val root = doc.body().select(".search_result__item") + return root.mapNotNull { item -> + val href = item.selectFirst(".search_result__item__thumbnail > a")?.attrAsRelativeUrl("href") ?: parseFailed() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(item.host ?: getDomain()), + title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") + ?.text()?.trim() ?: return@mapNotNull null, + altTitle = null, + author = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") + ?.attrAsAbsoluteUrl("data-original").orEmpty(), + ) + } + } + private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) { SortOrder.POPULARITY -> "manga_view" SortOrder.UPDATED -> "manga_updated" - else -> "rate" + else -> "manga_view" } } \ No newline at end of file From b5c830e10e389ab3040bdb843074e8ce67cc35f9 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Tue, 14 Jun 2022 19:52:18 +0300 Subject: [PATCH 3/3] Final touches --- .../parsers/site/NicovideoSeigaParser.kt | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt index 8c95e216..603d4f19 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -29,7 +29,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars override suspend fun getUsername(): String { val body = context.httpGet("https://app.nicovideo.jp/my/apps").parseHtml().body() - return body.selectFirst("#userinfo > div > div > strong")?.text() ?: parseFailed("Cannot get username") + return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) } override val sortOrders: Set = EnumSet.of( @@ -46,10 +46,13 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars sortOrder: SortOrder?, ): List { val page = (offset / 20f).toIntUp().inc() - if (!query.isNullOrEmpty()) { - return if (offset == 0) getSearchList(query, page) else emptyList() + val url = when { + !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() + tags.isNullOrEmpty() -> "https://${getDomain()}/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + tags.size == 1 -> "https://${getDomain()}${tags.first().key}?page=$page&sort=${getSortKey(sortOrder)}" + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") + else -> "https://${getDomain()}/manga/list?page=$page&sort=${getSortKey(sortOrder)}" } - val url = "/manga/list?page=$page&sort=${getSortKey(sortOrder)}".withDomain() val doc = context.httpGet(url).parseHtml() val comicList = doc.body().select("#comic_list > ul > li") ?: parseFailed("Container not found") val items = comicList.select("div > .description > div > div") @@ -65,7 +68,13 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars rating = RATING_UNKNOWN, url = href, isNsfw = false, - tags = emptySet(), + tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('='), + title = a.ownText().trim(), + source = source, + ) + }, state = when (statusText) { STATUS_ONGOING -> MangaState.ONGOING STATUS_FINISHED -> MangaState.FINISHED @@ -89,6 +98,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars STATUS_FINISHED -> MangaState.FINISHED else -> null }, + isNsfw = contents.select(".icon_adult").isNotEmpty(), chapters = contents.select("#episode_list > ul > li").mapIndexedNotNull { i, li -> val href = li.selectFirst("div > div.description > div.title > a")?.attrAsRelativeUrl("href") ?: parseFailed() MangaChapter( @@ -124,7 +134,16 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars } override suspend fun getTags(): Set { - return emptySet() + val doc = context.httpGet("https://${getDomain()}/manga/list").parseHtml() + val root = doc.body().select("#mg_category_list > ul > li") ?: parseFailed("Cannot find tags") + return root.mapToSet { li -> + val a = li.selectFirst("a") ?: parseFailed("a is null") + MangaTag( + title = a.text(), + key = a.attrAsRelativeUrlOrNull("href").orEmpty(), + source = source, + ) + } } private suspend fun getSearchList(query: String, page: Int): List {