diff --git a/docs/parser_classes.gaphor b/docs/parser_classes.gaphor new file mode 100644 index 000000000..9762866bf --- /dev/null +++ b/docs/parser_classes.gaphor @@ -0,0 +1,1111 @@ + + + + + +Новая модель + + + + + + + + + + + + + + + + + + + + + + + + +Новая диаграмма + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +1 + + +LegacyMangaParser + + + + + + + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 107.50391387939453, 388.8671875) + + +(0.0, 0.0) + + +147.0 + + +60.0 + + + + + +0 + + +0 + + + + + + + + + + + + + + + + + +1 + + +AbstractMangaParser + + + + + + + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 405.16796875, 388.8671875) + + +(0.0, 0.0) + + +158.0 + + +60.0 + + + + + +0 + + +0 + + + + + + + + + + + + + + + + + +1 + + +PagedMangaParser + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 476.3368367667698, 525.76953125) + + +(0.0, 0.0) + + +142.0 + + +60.0 + + + + + +0 + + +0 + + + + + + + + + + + + + + + + + +1 + + +SinglePageMangaParser + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 405.16796875, 627.46875) + + +(0.0, 0.0) + + +175.0 + + +60.0 + + + + + +0 + + +0 + + + + + + + + + + +0 + + +0 + + + + + +(1.0, 0.0, 0.0, 1.0, 499.2109069824219, 463.45703125) + + +[(28.486861756586336, 62.3125), (25.111328125, -14.58984375)] + + + + + + + + + + + + + + + + + + + + + + + +MangaParser + + + + + + + + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 278.00391387939453, 232.92578125) + + +(0.0, 0.0) + + +105.0 + + +80.0 + + + + + +0 + + +0 + + + + + +0 + + + + + + + +0 + + +0 + + + + + +(1.0, 0.0, 0.0, 1.0, 306.1445007324219, 270.0625) + + +[(55.866059373910275, 42.86328125), (164.5765002560883, 118.8046875)] + + + + + + + + + + + + + + + + + + + + + + + + + + +0 + + +0 + + + + + +(1.0, 0.0, 0.0, 1.0, 289.0820617675781, 276.40625) + + +[(-0.2702028783513697, 36.51953125), (-89.14763215970096, 112.4609375)] + + + + + + + + + + + + + + + + + + + + + + + + + + + + +LegacyPagedMangaParser + + + + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 159.50391387939453, 521.75390625) + + +(0.0, 0.0) + + +190.0 + + +60.0 + + + + + +0 + + +0 + + + + + + + + + + + + +LegacySinglePageMangaParser + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 107.50391387939453, 627.46875) + + +(0.0, 0.0) + + +223.0 + + +60.0 + + + + + +0 + + +0 + + + + + + + + + + +0 + + +0 + + + + + +(1.0, 0.0, 0.0, 1.0, 175.87969633151002, 433.1640319824219) + + +[(20.67503694485717, 88.58987426757812), (20.67503694485717, 15.703155517578125)] + + + + + + + + + + + + + + + + + + + + + + + + + + +0 + + +0 + + + + + +(1.0, 0.0, 0.0, 1.0, 148.88671875, 427.8085632324219) + + +[(0.5527361628988388, 199.66018676757812), (0.5527361628988459, 21.058624267578125)] + + + + + + + + + + + + + + + + + + + + + + + + + + +0 + + +0 + + + + + +(1.0, 0.0, 0.0, 1.0, 436.2929382324219, 439.1913757324219) + + +[(20.37646032737257, 188.27737426757812), (18.488327026367188, 9.675811767578125)] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +MangaParserWrapper + + + + + + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 452.6727742667698, 236.1809574894652) + + +(0.0, 0.0) + + +158.0 + + +60.0 + + + + + +0 + + +0 + + + + + + + + + + +0 + + +0 + + + + + +(1.0, 0.0, 0.0, 1.0, 306.0585632324219, 249.69920349121094) + + +[(76.94535064697266, 12.615185431614634), (146.61421103434793, 16.200927328075977)] + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Used for providing external api. Do not use this class directly + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 445.4868312390082, 128.1331358519086) + + +(0.0, 0.0) + + +183.21868896484375 + + +91.23829650878906 + + + + + + + + + + + + + +0 + + +0 + + +(1.0, 0.0, 0.0, 1.0, 550.2876124890082, 278.05499559311954) + + +[(0.8567782509883415, -41.87403810365436), (0.5, -58.683563232421875)] + + + + + + + + + + + + + + + +Extend this class if your manga source provides standard limit-offset based lists (get manga list by offset) + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 673.0610499890082, 367.0515553989646) + + +(0.0, 0.0) + + +228.8028016098773 + + +88.0 + + + + + + + + + + + + + + + +Extend this class if your manga source provides paged-based lists (get manga list by page number) + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 666.7924311664914, 507.7539062499999) + + +(0.0, 0.0) + + +214.34368896484375 + + +88.0 + + + + + + + + + + + + + + + +Extend this class if your manga source does not provide pagination (all manga provided in one list) + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 666.7924311664914, 560.9671898788581) + + +(0.0, 58.00435704705592) + + +263.9307954323941 + + +78.01706672440287 + + + + + + + + + + + + + +0 + + +0 + + +(1.0, 0.0, 0.0, 1.0, 561.8951626340418, 549.6101338901756) + + +[(56.44167413272805, 7.038279316310902), (104.89726853244963, 8.304008355003589)] + + + + + + + + + + + + + +0 + + +0 + + +(1.0, 0.0, 0.0, 1.0, 559.3873501340418, 413.0007588901755) + + +[(3.7806186159582467, 0.0), (113.67369985496646, 1.6012844908540842)] + + + + + + + + + + + + + +0 + + +0 + + +(1.0, 0.0, 0.0, 1.0, 522.3600063840418, 652.6882588901756) + + +[(57.80796236595825, 5.29182139794003), (144.43242478244963, 5.657840086725969)] + + + + + + + + + + + + + + + +Do not use these classes for new parsers + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 59.50391387939453, 301.16000020170736) + + +(-58.93360137939453, 0.0) + + +144.78774075904175 + + +68.0 + + + + + + + + + + + + + +0 + + +0 + + +(1.0, 0.0, 0.0, 1.0, 65.93617825904175, 335.96078145170736) + + +[(40.6686968734994, 33.19921875), (64.01299389673568, 52.90640604829264)] + + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 375.05802564891326, 349.05453145170736) + + +(0.0, 3.15625) + + +590.6594026101285 + + +368.44140625 + + + + + + + +To create your own parser you have to extends one of these classes + + + + + + + + + +(1.0, 0.0, 0.0, 1.0, 756.725301794198, 225.57697659840966) + + +(0.0, 0.0) + + +208.99212646484375 + + +73.47482464883183 + + + + + + + + + + + + + +0 + + +0 + + +(1.0, 0.0, 0.0, 1.0, 943.6141683885666, 419.2772168262177) + + +[(-27.404961772030788, -67.06643537451032), (-27.404961772030788, -120.2254155789762)] + + + + + + + + + \ No newline at end of file diff --git a/docs/parser_classes.png b/docs/parser_classes.png new file mode 100644 index 000000000..6f7251c56 Binary files /dev/null and b/docs/parser_classes.png differ diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index d2fa0fa65..28f87c4a9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -28,6 +28,13 @@ public interface MangaParser : Interceptor { public val config: MangaSourceConfig + /** + * Provide default domain and available alternatives, if any. + * + * Never hardcode domain in requests, use [domain] instead. + */ + public val configKeyDomain: ConfigKey.Domain + public val domain: String public suspend fun getList(query: MangaSearchQuery): List diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/core/AbstractMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/core/AbstractMangaParser.kt index 28e58ba7a..4b09db57a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/core/AbstractMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/core/AbstractMangaParser.kt @@ -22,7 +22,7 @@ import java.util.* @InternalParsersApi public abstract class AbstractMangaParser @InternalParsersApi constructor( @property:InternalParsersApi public val context: MangaLoaderContext, - public override val source: MangaParserSource, + public final override val source: MangaParserSource, ) : MangaParser { public override val config: MangaSourceConfig by lazy { context.getConfig(source) } @@ -30,16 +30,18 @@ public abstract class AbstractMangaParser @InternalParsersApi constructor( public open val sourceLocale: Locale get() = if (source.locale.isEmpty()) Locale.ROOT else Locale(source.locale) - /** - * Provide default domain and available alternatives, if any. - * - * Never hardcode domain in requests, use [domain] instead. - */ - @InternalParsersApi - public abstract val configKeyDomain: ConfigKey.Domain - protected open val userAgentKey: ConfigKey.UserAgent = ConfigKey.UserAgent(context.getDefaultUserAgent()) + protected val sourceContentRating: ContentRating? + get() = if (source.contentType == ContentType.HENTAI) { + ContentRating.ADULT + } else { + null + } + + final override val domain: String + get() = config[configKeyDomain] + @Deprecated("Override intercept() instead") override fun getRequestHeaders(): Headers = Headers.Builder() .add("User-Agent", config[userAgentKey]) @@ -54,9 +56,6 @@ public abstract class AbstractMangaParser @InternalParsersApi constructor( return SortOrder.entries.first { it in supported } } - override val domain: String - get() = config[configKeyDomain] - @JvmField protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/core/LegacyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/core/LegacyMangaParser.kt index 18895d2f7..440a27fba 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/core/LegacyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/core/LegacyMangaParser.kt @@ -22,7 +22,7 @@ import java.util.* @InternalParsersApi public abstract class LegacyMangaParser @InternalParsersApi constructor( @property:InternalParsersApi public val context: MangaLoaderContext, - public override val source: MangaParserSource, + public final override val source: MangaParserSource, ) : MangaParser { public final override val searchQueryCapabilities: MangaSearchQueryCapabilities @@ -37,14 +37,6 @@ public abstract class LegacyMangaParser @InternalParsersApi constructor( protected val isNsfwSource: Boolean = source.contentType == ContentType.HENTAI - /** - * Provide default domain and available alternatives, if any. - * - * Never hardcode domain in requests, use [domain] instead. - */ - @InternalParsersApi - public abstract val configKeyDomain: ConfigKey.Domain - protected open val userAgentKey: ConfigKey.UserAgent = ConfigKey.UserAgent(context.getDefaultUserAgent()) override fun getRequestHeaders(): Headers = Headers.Builder() @@ -60,7 +52,7 @@ public abstract class LegacyMangaParser @InternalParsersApi constructor( return SortOrder.entries.first { it in supported } } - override val domain: String + final override val domain: String get() = config[configKeyDomain] @JvmField diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/core/MangaParserWrapper.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/core/MangaParserWrapper.kt index 4c13da50d..54eb442b8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/core/MangaParserWrapper.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/core/MangaParserWrapper.kt @@ -14,11 +14,11 @@ internal class MangaParserWrapper( private val delegate: MangaParser, ) : MangaParser by delegate { - override suspend fun getList(searchQuery: MangaSearchQuery): List = withContext(Dispatchers.Default) { - if (!searchQuery.skipValidation) { - searchQueryCapabilities.validate(searchQuery) + override suspend fun getList(query: MangaSearchQuery): List = withContext(Dispatchers.Default) { + if (!query.skipValidation) { + searchQueryCapabilities.validate(query) } - delegate.getList(searchQuery) + delegate.getList(query) } override suspend fun getDetails(manga: Manga): Manga = withContext(Dispatchers.Default) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/MangaboxParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/MangaboxParser.kt index ca6b0bd14..ac8dc3d2f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/MangaboxParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/MangaboxParser.kt @@ -180,7 +180,7 @@ internal abstract class MangaboxParser( authors = emptySet(), state = null, source = source, - contentRating = if (source.contentType == ContentType.HENTAI) ContentRating.ADULT else ContentRating.SAFE, + contentRating = sourceContentRating, ) } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/ManhwaIndoParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/ManhwaIndoParser.kt index fb9e350a5..f204d7275 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/ManhwaIndoParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/id/ManhwaIndoParser.kt @@ -29,7 +29,7 @@ internal class ManhwaIndoParser(context: MangaLoaderContext) : } private suspend fun fetchPage(img: Element): MangaPage? = runCatchingCancellable { - val url = img.requireSrc().toAbsoluteUrl(domain) ?: return@runCatchingCancellable null + val url = img.requireSrc().toAbsoluteUrl(domain) webClient.httpHead(url).use { response -> if (response.mimeType?.startsWith("image/") == true) { MangaPage( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt index 71bd55533..a2f0d022a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt @@ -50,7 +50,7 @@ internal class Hentai18VN(context: MangaLoaderContext) : parseMangaSearch(response) } - !filter.tags.isNullOrEmpty() -> { + filter.tags.isNotEmpty() -> { val tag = filter.tags.first() val url = buildString { append("https://") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/HentaiVnBuzz.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/HentaiVnBuzz.kt index a49094959..f1a3f2535 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/HentaiVnBuzz.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/HentaiVnBuzz.kt @@ -52,7 +52,7 @@ internal class HentaiVnBuzz(context: MangaLoaderContext) : } } - !filter.tags.isNullOrEmpty() -> { + filter.tags.isNotEmpty() -> { val tag = filter.tags.first() buildString { append("/the-loai/") @@ -110,7 +110,7 @@ internal class HentaiVnBuzz(context: MangaLoaderContext) : val doc = webClient.httpGet(fullUrl).parseHtml() return when { !filter.query.isNullOrEmpty() -> parseSearchManga(doc) - !filter.tags.isNullOrEmpty() -> parseSearchManga(doc) + filter.tags.isNotEmpty() -> parseSearchManga(doc) else -> parseListManga(doc) } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt index b11fa12de..e611f350b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt @@ -39,7 +39,7 @@ internal class TruyenHentaiVN(context: MangaLoaderContext) : append(domain) when { - !filter.tags.isNullOrEmpty() -> { + filter.tags.isNotEmpty() -> { val tag = filter.tags.first() append(tag.key) if (page > 1) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/DocTruyen3Q.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/DocTruyen3Q.kt index 1db3e26df..ac9f3fc79 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/DocTruyen3Q.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/DocTruyen3Q.kt @@ -222,7 +222,7 @@ internal class DocTruyen3Q(context: MangaLoaderContext) : val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() return doc.select("div.page-chapter img").mapNotNull { img -> - val url = img.attr("src")?.toAbsoluteUrl(domain) ?: return@mapNotNull null + val url = img.attrAsAbsoluteUrlOrNull("src") ?: return@mapNotNull null MangaPage( id = generateUid(url), url = url, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NewTruyen.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NewTruyen.kt index ac5338452..e1b2a6516 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NewTruyen.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NewTruyen.kt @@ -3,19 +3,18 @@ package org.koitharu.kotatsu.parsers.site.wpcomics.vi import androidx.collection.ArraySet import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.site.wpcomics.WpComicsParser import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.site.wpcomics.WpComicsParser +import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NEWTRUYEN", "NewTruyen", "vi") internal class NewTruyen(context: MangaLoaderContext) : - WpComicsParser(context, MangaParserSource.NEWTRUYEN, "newtruyen2.com", 36) { + WpComicsParser(context, MangaParserSource.NEWTRUYEN, "newtruyen2.com", 36) { override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = getAvailableTags(), @@ -26,14 +25,14 @@ internal class NewTruyen(context: MangaLoaderContext) : val fullUrl = manga.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() val storyID = doc.selectFirst("input#storyID")?.attr("value") - ?: throw ParseException("Story ID not found", fullUrl) + ?: throw ParseException("Story ID not found", fullUrl) val chaptersDeferred = async { getChapterList(storyID) } val tagsElement = doc.select("p.col-xs-12 a.tr-theloai") val mangaTags = tagsElement.map { MangaTag( title = it.text(), key = it.attr("href").substringAfterLast('/'), - source = source + source = source, ) }.toSet() val author = doc.body().select(selectAut).textOrNull() @@ -54,7 +53,7 @@ internal class NewTruyen(context: MangaLoaderContext) : } private suspend fun getChapterList(storyID: String): List { - val url = "/Story/ListChapterByStoryID?storyID=" + storyID + val url = "/Story/ListChapterByStoryID?storyID=$storyID" val fullUrl = url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() return doc.select("div.col-xs-5.chapter").mapChapters(reversed = true) { i, li -> diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Collection.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Collection.kt index 68bd660e7..38d0dfa7b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Collection.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Collection.kt @@ -87,3 +87,15 @@ public inline fun Collection.mapToArray(transform: (T) -> R): forEachIndexed { index, t -> result[index] = transform(t) } return result as Array } + +public fun Array.toArraySet(): Set = when (size) { + 0 -> emptySet() + 1 -> setOf(first()) + else -> ArraySet(this) +} + +public fun Collection.toArraySet(): Set = when (size) { + 0 -> emptySet() + 1 -> setOf(first()) + else -> ArraySet(this) +}