KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove unessery code

Browse Source 
Remove old chapter manga and change last source for new manga chapter
Simplify parseChapterDate
Fix fetchAvailableTags on some source need to override
This commit is contained in:
devi
2024-09-20 20:37:36 +02:00
parent 336c4a4d49
commit 600eab20a1
50 changed files with 448 additions and 739 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt
View File

@@ -38,28 +38,6 @@ public class MangaChapter(
@JvmField public val source: MangaSource,
) {
@Deprecated(message = "Consider using constructor with volume value")
internal constructor(
id: Long,
name: String,
number: Int,
url: String,
scanlator: String?,
uploadDate: Long,
branch: String?,
source: MangaSource,
) : this(
id = id,
name = name,
number = number.toFloat(),
volume = 0,
url = url,
scanlator = scanlator,
uploadDate = uploadDate,
branch = branch,
source = source,
)
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false

6
src/main/kotlin/org/koitharu/kotatsu/parsers/model/WordSet.kt
View File

@@ -5,7 +5,7 @@ import org.koitharu.kotatsu.parsers.InternalParsersApi
@InternalParsersApi
public class WordSet(private vararg val words: String) {
public fun anyWordIn(dateString: String): Boolean = words.any {
dateString.contains(it, ignoreCase = true)
}
public fun anyWordIn(dateString: String): Boolean = words.any { dateString.contains(it, ignoreCase = true) }
public fun startsWith(dateString: String): Boolean = words.any { dateString.startsWith(it, ignoreCase = true) }
public fun endsWith(dateString: String): Boolean = words.any { dateString.endsWith(it, ignoreCase = true) }
}

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/BatoToParser.kt
View File

@@ -71,8 +71,6 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.allOf(MangaState::class.java),
availableContentRating = EnumSet.of(ContentRating.SAFE),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = setOf(
Locale.CHINESE, Locale.ENGLISH, Locale.US, Locale.FRENCH, Locale.GERMAN, Locale.ITALIAN, Locale.JAPANESE,
Locale("af"), Locale("ar"), Locale("az"), Locale("eu"), Locale("be"),

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ComickFunParser.kt
View File

@@ -47,7 +47,6 @@ internal class ComickFunParser(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED),
availableContentRating = emptySet(),
availableContentTypes = EnumSet.of(
ContentType.MANGA,
ContentType.MANHWA,
@@ -55,7 +54,6 @@ internal class ComickFunParser(context: MangaLoaderContext) :
ContentType.OTHER,
),
availableDemographics = EnumSet.allOf(Demographic::class.java),
availableLocales = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

86
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
View File

@@ -482,14 +482,14 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
title = doc.selectFirstOrThrow("h1").text(),
url = id.toString(),
coverUrl =
"https:" +
doc.selectFirstOrThrow("picture > source")
.attr("data-srcset")
.substringBefore(" "),
"https:" +
doc.selectFirstOrThrow("picture > source")
.attr("data-srcset")
.substringBefore(" "),
publicUrl =
doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href")
.toAbsoluteUrl(domain),
doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href")
.toAbsoluteUrl(domain),
author = null,
tags = emptySet(),
isNsfw = true,
@@ -512,37 +512,37 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
return manga.copy(
title = json.getString("title"),
largeCoverUrl =
json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash")
val commonId = commonImageId()
val imageId = imageIdFromHash(hash)
val subDomain = 'a' + subdomainOffset(imageId)
json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash")
val commonId = commonImageId()
val imageId = imageIdFromHash(hash)
val subDomain = 'a' + subdomainOffset(imageId)
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
},
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
},
author =
json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(),
json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(),
publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain),
tags =
buildSet {
json.optJSONArray("characters")
?.mapToTags("character")
?.let(::addAll)
json.optJSONArray("tags")
?.mapToTags("tag")
?.let(::addAll)
json.optJSONArray("artists")
?.mapToTags("artist")
?.let(::addAll)
json.optJSONArray("parodys")
?.mapToTags("parody")
?.let(::addAll)
json.optJSONArray("groups")
?.mapToTags("group")
?.let(::addAll)
},
buildSet {
json.optJSONArray("characters")
?.mapToTags("character")
?.let(::addAll)
json.optJSONArray("tags")
?.mapToTags("tag")
?.let(::addAll)
json.optJSONArray("artists")
?.mapToTags("artist")
?.let(::addAll)
json.optJSONArray("parodys")
?.mapToTags("parody")
?.let(::addAll)
json.optJSONArray("groups")
?.mapToTags("group")
?.let(::addAll)
},
chapters = listOf(
MangaChapter(
id = generateUid(manga.url),
@@ -566,15 +566,15 @@ internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context
mapJSON {
MangaTag(
title =
it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title ♀"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
"$title ♂"
} else {
title
}
},
it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title ♀"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
"$title ♂"
} else {
title
}
},
key = it.getString("url").tagUrlToTag(),
source = source,
).let(tags::add)

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt
View File

@@ -69,7 +69,6 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
MangaState.ABANDONED,
),
availableContentRating = EnumSet.allOf(ContentRating::class.java),
availableContentTypes = emptySet(),
availableDemographics = EnumSet.allOf(Demographic::class.java),
availableLocales = localesDeferred.await(),
)

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaFireParser.kt
View File

@@ -81,10 +81,6 @@ internal abstract class MangaFireParser(
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = tags.get().values.toSet(),
availableStates = EnumSet.allOf(MangaState::class.java),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaPark.kt
View File

@@ -32,8 +32,6 @@ internal class MangaPark(context: MangaLoaderContext) :
availableTags = tagsMap.get().values.toSet(),
availableStates = EnumSet.allOf(MangaState::class.java),
availableContentRating = EnumSet.of(ContentRating.SAFE),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = setOf(
Locale("af"), Locale("sq"), Locale("am"), Locale("ar"), Locale("hy"),
Locale("az"), Locale("be"), Locale("bn"), Locale("zh_hk"), Locale("zh_tw"),

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaReaderToParser.kt
View File

@@ -77,10 +77,6 @@ internal class MangaReaderToParser(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = tags.get().values.toSet(),
availableStates = EnumSet.allOf(MangaState::class.java),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineMangaParser.kt
View File

@@ -53,10 +53,6 @@ internal abstract class NineMangaParser(
MangaState.ONGOING,
MangaState.FINISHED,
),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override fun intercept(chain: Interceptor.Chain): Response {

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/FlixScans.kt
View File

@@ -31,9 +31,6 @@ internal class FlixScans(context: MangaLoaderContext) : PagedMangaParser(context
availableTags = fetchAvailableTags(),
availableStates = EnumSet.allOf(MangaState::class.java),
availableContentRating = EnumSet.of(ContentRating.ADULT),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/TeamXNovel.kt
View File

@@ -33,10 +33,6 @@ internal class TeamXNovel(context: MangaLoaderContext) : PagedMangaParser(contex
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/cupfox/CupFoxParser.kt
View File

@@ -35,10 +35,6 @@ internal abstract class CupFoxParser(
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/AsuraScansParser.kt
View File

@@ -36,10 +36,6 @@ internal class AsuraScansParser(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = getOrCreateTagMap().values.toSet(),
availableStates = EnumSet.allOf(MangaState::class.java),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ComicExtra.kt
View File

@@ -29,7 +29,6 @@ internal class ComicExtra(context: MangaLoaderContext) : PagedMangaParser(contex
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED),
availableContentRating = emptySet(),
)
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/MangaTownParser.kt
View File

@@ -39,7 +39,6 @@ internal class MangaTownParser(context: MangaLoaderContext) :
MangaState.ONGOING,
MangaState.FINISHED,
),
availableContentRating = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Mangaowl.kt
View File

@@ -39,10 +39,6 @@ internal class Mangaowl(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Manhwa18Parser.kt
View File

@@ -45,7 +45,6 @@ internal class Manhwa18Parser(context: MangaLoaderContext) :
MangaState.FINISHED,
MangaState.PAUSED,
),
availableContentRating = emptySet(),
)
override suspend fun getFavicons(): Favicons {

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt
View File

@@ -32,7 +32,6 @@ internal class ManhwasMen(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED),
availableContentRating = emptySet(),
)
override suspend fun getListPage(

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/VyManga.kt
View File

@@ -40,7 +40,6 @@ internal class VyManga(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED),
availableContentRating = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

87
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt
View File

@@ -78,10 +78,6 @@ internal abstract class FmreaderParser(
MangaState.FINISHED,
MangaState.ABANDONED,
),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
@@ -253,31 +249,19 @@ internal abstract class FmreaderParser(
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") ||
d.endsWith(" atrás") ||
// short Hours
d.endsWith(" h") ||
// short Day
d.endsWith(" d") -> parseRelativeDate(date)
// Handle 'yesterday' and 'today', using midnight
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet(" ago", " atrás", " h", " d").endsWith(d) -> { parseRelativeDate(d) }
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet("today").startsWith(d) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
@@ -291,49 +275,24 @@ internal abstract class FmreaderParser(
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes", "minuto", "minutos").anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("hour", "hours", "hora", "horas", "h").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet("day", "days", "día", "dia").anyWordIn(date) -> cal.apply {
add(
Calendar.DAY_OF_MONTH,
-number,
)
}.timeInMillis
WordSet("week", "weeks", "semana", "semanas").anyWordIn(date) -> cal.apply {
add(
Calendar.WEEK_OF_YEAR,
-number,
)
}.timeInMillis
WordSet("month", "months", "mes", "meses").anyWordIn(date) -> cal.apply {
add(
Calendar.MONTH,
-number,
)
}.timeInMillis
WordSet("year", "año", "años").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
WordSet("second")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes", "minuto", "minutos")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours", "hora", "horas", "h")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days", "día", "dia")
.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("week", "weeks", "semana", "semanas")
.anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis
WordSet("month", "months", "mes", "meses")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year", "año", "años")
.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/BentomangaParser.kt
View File

@@ -52,7 +52,6 @@ internal class BentomangaParser(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED),
availableContentRating = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/LegacyScansParser.kt
View File

@@ -28,7 +28,6 @@ internal class LegacyScansParser(context: MangaLoaderContext) :
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.PAUSED),
availableContentRating = emptySet(),
)
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

5
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/LugnicaScans.kt
View File

@@ -33,12 +33,7 @@ internal class LugnicaScans(context: MangaLoaderContext) :
)
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = emptySet(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/MangaMana.kt
View File

@@ -42,7 +42,6 @@ internal class MangaMana(context: MangaLoaderContext) : PagedMangaParser(context
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED),
availableContentRating = emptySet(),
)
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fuzzydoodle/FuzzyDoodleParser.kt
View File

@@ -76,7 +76,6 @@ internal abstract class FuzzyDoodleParser(
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED),
availableContentRating = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/HeanCms.kt
View File

@@ -54,10 +54,6 @@ internal abstract class HeanCms(
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = fetchAvailableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED),
availableContentRating = emptySet(),
availableContentTypes = emptySet(),
availableDemographics = emptySet(),
availableLocales = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ja/NicovideoSeigaParser.kt
View File

@@ -149,7 +149,7 @@ internal class NicovideoSeigaParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga"))
val doc = webClient.httpGet(fullUrl).parseHtml()
if (!doc.select("#login_manga").isEmpty)
if (!doc.select("#login_manga").isEmpty())
throw AuthRequiredException(source)
val root = doc.body().select("#page_contents > li")
return root.map { li ->

46
src/main/kotlin/org/koitharu/kotatsu/parsers/site/keyoapp/KeyoappParser.kt
View File

@@ -266,22 +266,17 @@ internal abstract class KeyoappParser(
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") -> parseRelativeDate(date)
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1)
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet(" ago").endsWith(d) -> { parseRelativeDate(d) }
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet("today").startsWith(d) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
@@ -299,17 +294,18 @@ internal abstract class KeyoappParser(
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("minute", "minutes").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
WordSet("second")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("minute", "minutes")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days")
.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year")
.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}

90
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt
View File

@@ -746,56 +746,46 @@ internal abstract class MadaraParser(
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") || d.endsWith(" atrás") || // Handle translated 'ago' in Portuguese.
d.startsWith("há ") || // other translated 'ago' in Portuguese.
d.endsWith(" hace") || // other translated 'ago' in Spanish
d.endsWith(" publicado") ||
d.endsWith(" назад") || // other translated 'ago' in Russian
d.endsWith(" önce") || // Handle translated 'ago' in Turkish.
d.endsWith(" trước") || // Handle translated 'ago' in Viêt Nam.
d.endsWith("مضت") || // Handle translated 'ago' in Arabic
d.startsWith("منذ") ||
d.startsWith("il y a") || // Handle translated 'ago' in French.
//If there is no ago but just a motion of time
// short Hours
d.endsWith(" h") ||
// short Day
d.endsWith(" d") ||
// Day in Portuguese
d.endsWith(" días") || d.endsWith(" día") ||
// Day in French
d.endsWith(" jour") || d.endsWith(" jours") ||
// Hours in Portuguese
d.endsWith(" horas") || d.endsWith(" hora") ||
// Hours in french
d.endsWith(" heure") || d.endsWith(" heures") ||
// Minutes in English
d.endsWith(" mins") ||
// Minutes in Portuguese
d.endsWith(" minutos") || d.endsWith(" minuto") ||
//Minutes in French
d.endsWith(" minute") || d.endsWith(" minutes") ||
//month in French
d.endsWith(" mois") -> parseRelativeDate(date)
// Handle 'yesterday' and 'today', using midnight
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet(" ago", "atrás", " hace", " publicado"," назад", " önce", " trước", "مضت",
" h", " d", " días", " jour", " horas", " heure", " mins", " minutos", " minute", " mois").endsWith(d) -> {
parseRelativeDate(d)
}
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet("há ", "منذ", "il y a" ).startsWith(d) -> {
parseRelativeDate(d)
}
WordSet("yesterday", "يوم واحد").startsWith(d) -> {
Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
WordSet("today").startsWith(d) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
WordSet("يومين").startsWith(d) -> {
Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -2) // day before yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
@@ -812,27 +802,19 @@ internal abstract class MadaraParser(
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("detik", "segundo", "second", "ثوان")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("menit", "dakika", "min", "minute", "minutes", "minuto", "mins", "phút", "минут", "دقيقة")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("jam", "saat", "heure", "hora", "horas", "hour", "hours", "h", "ساعات", "ساعة")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "d", "день")
.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months", "أشهر", "mois")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year")
.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}

280
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/en/AdultWebtoon.kt
View File

@@ -15,159 +15,159 @@ import java.text.SimpleDateFormat
@MangaSourceParser("ADULT_WEBTOON", "AdultWebtoon", "en", ContentType.HENTAI)
internal class AdultWebtoon(context: MangaLoaderContext) :
MadaraParser(context, MangaParserSource.ADULT_WEBTOON, "adultwebtoon.com") {
override val tagPrefix = "adult-webtoon-genre/"
override val listUrl = "adult-webtoon/"
override val postReq = true
override val withoutAjax = true
MadaraParser(context, MangaParserSource.ADULT_WEBTOON, "adultwebtoon.com") {
override val tagPrefix = "adult-webtoon-genre/"
override val listUrl = "adult-webtoon/"
override val postReq = true
override val withoutAjax = true
override suspend fun getFilterOptions() = super.getFilterOptions().copy(
availableStates = emptySet(),
availableContentRating = emptySet(),
)
override suspend fun getFilterOptions() = super.getFilterOptions().copy(
availableStates = emptySet(),
availableContentRating = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val pages = page + 1
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val pages = page + 1
val url = buildString {
append("https://")
append(domain)
when {
!filter.query.isNullOrEmpty() -> {
if (pages > 1) {
append("/page/")
append(pages.toString())
}
append("/?s=")
append(filter.query.urlEncoded())
append("&post_type=wp-manga")
}
val url = buildString {
append("https://")
append(domain)
when {
!filter.query.isNullOrEmpty() -> {
if (pages > 1) {
append("/page/")
append(pages.toString())
}
append("/?s=")
append(filter.query.urlEncoded())
append("&post_type=wp-manga")
}
else -> {
else -> {
if (filter.tags.isNotEmpty()) {
filter.tags.oneOrThrowIfMany()?.let {
append('/')
append(tagPrefix)
append(it.key)
append('/')
}
} else {
append('/')
append(listUrl)
}
if (filter.tags.isNotEmpty()) {
filter.tags.oneOrThrowIfMany()?.let {
append('/')
append(tagPrefix)
append(it.key)
append('/')
}
} else {
append('/')
append(listUrl)
}
if (pages > 1) {
append("page/")
append(pages)
append('/')
}
if (pages > 1) {
append("page/")
append(pages)
append('/')
}
append("?m_orderby=")
when (order) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("latest")
SortOrder.NEWEST -> append("new-manga")
SortOrder.ALPHABETICAL -> append("alphabet")
SortOrder.RATING -> append("rating")
else -> append("latest")
}
}
}
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
append("?m_orderby=")
when (order) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("latest")
SortOrder.NEWEST -> append("new-manga")
SortOrder.ALPHABETICAL -> append("alphabet")
SortOrder.RATING -> append("rating")
else -> append("latest")
}
}
}
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val body = doc.body()
val chaptersDeferred = async { loadChapters(manga.url, doc) }
val desc = body.select(selectDesc).html()
val stateDiv = if (selectState.isEmpty()) {
body.selectFirst("div.post-content_item:contains(Status)")?.selectLast("div.summary-content")
} else {
body.selectFirst(selectState)
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val body = doc.body()
val chaptersDeferred = async { loadChapters(manga.url, doc) }
val desc = body.select(selectDesc).html()
val stateDiv = if (selectState.isEmpty()) {
body.selectFirst("div.post-content_item:contains(Status)")?.selectLast("div.summary-content")
} else {
body.selectFirst(selectState)
}
val state = stateDiv?.let {
when (it.text()) {
in ongoing -> MangaState.ONGOING
in finished -> MangaState.FINISHED
in abandoned -> MangaState.ABANDONED
in paused -> MangaState.PAUSED
else -> null
}
}
val state = stateDiv?.let {
when (it.text()) {
in ongoing -> MangaState.ONGOING
in finished -> MangaState.FINISHED
in abandoned -> MangaState.ABANDONED
in paused -> MangaState.PAUSED
else -> null
}
}
val alt =
doc.body().select(".post-content_item:contains(Alt) .summary-content").firstOrNull()?.tableValue()?.text()
?.trim()
val alt =
doc.body().select(".post-content_item:contains(Alt) .summary-content").firstOrNull()?.tableValue()?.text()
?.trim()
manga.copy(
tags = doc.body().select(selectGenre).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
altTitle = alt,
state = state,
chapters = chaptersDeferred.await(),
)
}
manga.copy(
tags = doc.body().select(selectGenre).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
altTitle = alt,
state = state,
chapters = chaptersDeferred.await(),
)
}
override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> {
val mangaId = document.select("div#manga-chapters-holder").attr("data-id")
val url = "https://$domain/wp-admin/admin-ajax.php"
val postData = "post_id=$mangaId&action=ajax_chap"
val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build()
val doc = makeRequest(url, postData.toRequestBody(), headers)
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectChapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
val link = href + stylePage
val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text()
val name = a.selectFirst("p")?.text() ?: a.ownText()
MangaChapter(
id = generateUid(href),
url = link,
name = name,
number = i + 1f,
volume = 0,
branch = null,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
scanlator = null,
source = source,
)
}
}
override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> {
val mangaId = document.select("div#manga-chapters-holder").attr("data-id")
val url = "https://$domain/wp-admin/admin-ajax.php"
val postData = "post_id=$mangaId&action=ajax_chap"
val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build()
val doc = makeRequest(url, postData.toRequestBody(), headers)
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectChapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
val link = href + stylePage
val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text()
val name = a.selectFirst("p")?.text() ?: a.ownText()
MangaChapter(
id = generateUid(href),
url = link,
name = name,
number = i + 1f,
volume = 0,
branch = null,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
scanlator = null,
source = source,
)
}
}
private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document {
var retryCount = 0
val backoffDelay = 2000L // Initial delay (milliseconds)
val request = Request.Builder().url(url).post(payload).headers(headers).build()
while (true) {
try {
return context.httpClient.newCall(request).execute().parseHtml()
private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document {
var retryCount = 0
val backoffDelay = 2000L // Initial delay (milliseconds)
val request = Request.Builder().url(url).post(payload).headers(headers).build()
while (true) {
try {
return context.httpClient.newCall(request).execute().parseHtml()
} catch (e: Exception) {
// Log or handle the exception as needed
if (++retryCount <= 5) {
withContext(Dispatchers.Default) {
delay(backoffDelay)
}
} else {
throw e
}
}
}
}
} catch (e: Exception) {
// Log or handle the exception as needed
if (++retryCount <= 5) {
withContext(Dispatchers.Default) {
delay(backoffDelay)
}
} else {
throw e
}
}
}
}
}

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/en/MmScans.kt
View File

@@ -13,4 +13,4 @@ internal class MmScans(context: MangaLoaderContext) :
override val selectChapter = "li.chapter-li"
override val selectDesc = "div.summary-text"
override val withoutAjax = true
}
}

106
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/DragonTranslationParser.kt
View File

@@ -12,66 +12,66 @@ import java.util.*
@MangaSourceParser("DRAGONTRANSLATION", "Dragon Translation", "es")
internal class DragonTranslationParser(context: MangaLoaderContext) :
MadaraParser(context, MangaParserSource.DRAGONTRANSLATION, "dragontranslation.net", 30) {
MadaraParser(context, MangaParserSource.DRAGONTRANSLATION, "dragontranslation.net", 30) {
override val selectPage = "div#chapter_imgs img"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val selectPage = "div#chapter_imgs img"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
override suspend fun getFilterOptions() = super.getFilterOptions().copy(
availableStates = emptySet(),
availableContentRating = emptySet(),
)
override suspend fun getFilterOptions() = super.getFilterOptions().copy(
availableStates = emptySet(),
availableContentRating = emptySet(),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when {
!filter.query.isNullOrEmpty() -> {
append("/mangas?buscar=")
append(filter.query.urlEncoded())
append("&page=")
append(page.toString())
}
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when {
!filter.query.isNullOrEmpty() -> {
append("/mangas?buscar=")
append(filter.query.urlEncoded())
append("&page=")
append(page.toString())
}
else -> {
else -> {
append("/mangas?page=")
append(page.toString())
append("/mangas?page=")
append(page.toString())
val tag = filter.tags.oneOrThrowIfMany()
if (filter.tags.isNotEmpty()) {
append("&tag=")
append(tag?.key.orEmpty())
}
}
}
}
val tag = filter.tags.oneOrThrowIfMany()
if (filter.tags.isNotEmpty()) {
append("&tag=")
append(tag?.key.orEmpty())
}
}
}
}
val doc = webClient.httpGet(url).parseHtml()
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.video-bg div.col-6 ").map { div ->
val href =
div.selectFirst("a.series-link")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img.thumb-img")?.src().orEmpty(),
title = div.selectFirst("div.series-box h5")?.text().orEmpty(),
altTitle = null,
rating = div.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
return doc.select("div.video-bg div.col-6 ").map { div ->
val href =
div.selectFirst("a.series-link")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img.thumb-img")?.src().orEmpty(),
title = div.selectFirst("div.series-box h5")?.text().orEmpty(),
altTitle = null,
rating = div.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
}

76
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/MadthemeParser.kt
View File

@@ -271,30 +271,19 @@ internal abstract class MadthemeParser(
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") ||
// short Hours
d.endsWith(" h") ||
// short Day
d.endsWith(" d") -> parseRelativeDate(date)
// Handle 'yesterday' and 'today', using midnight
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet(" ago", " h", " d").endsWith(d) -> { parseRelativeDate(d) }
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet("today").startsWith(d) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
@@ -308,45 +297,22 @@ internal abstract class MadthemeParser(
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet(
"day",
"days",
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("hour", "hours", "h").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet(
"min",
"minute",
"minutes",
).anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("second").anyWordIn(date) -> cal.apply {
add(
Calendar.SECOND,
-number,
)
}.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
WordSet("second")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours", "h")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days")
.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year")
.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}

8
src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/zh/Hanman18.kt
View File

@@ -5,8 +5,8 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.ContentType
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions
import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.site.manga18.Manga18Parser
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
import org.koitharu.kotatsu.parsers.util.generateUid
@@ -17,6 +17,10 @@ import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
internal class Hanman18(context: MangaLoaderContext) :
Manga18Parser(context, MangaParserSource.HANMAN18, "hanman18.com") {
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = emptySet()
)
override suspend fun getChapters(doc: Document): List<MangaChapter> {
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a")
@@ -34,6 +38,4 @@ internal class Hanman18(context: MangaLoaderContext) :
)
}
}
private suspend fun fetchAvailableTags(): Set<MangaTag> = emptySet() // search by tag does not work
}

79
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/MangaboxParser.kt
View File

@@ -146,7 +146,7 @@ internal abstract class MangaboxParser(
protected open val selectTagMap = "div.panel-genres-list a:not(.genres-select)"
private suspend fun fetchAvailableTags(): Set<MangaTag> {
protected open suspend fun fetchAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
val tags = doc.select(selectTagMap).drop(1) // remove all tags
return tags.mapNotNullToSet { a ->
@@ -263,30 +263,18 @@ internal abstract class MangaboxParser(
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") ||
// short Hours
d.endsWith(" h") ||
// short Day
d.endsWith(" d") -> parseRelativeDate(date)
WordSet(" ago", " h", " d").endsWith(d) -> { parseRelativeDate(d) }
// Handle 'yesterday' and 'today', using midnight
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet("today").startsWith(d) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
@@ -300,45 +288,22 @@ internal abstract class MangaboxParser(
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet(
"day",
"days",
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("hour", "hours", "h").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet(
"min",
"minute",
"minutes",
).anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("second").anyWordIn(date) -> cal.apply {
add(
Calendar.SECOND,
-number,
)
}.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
WordSet("second")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours", "h")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days")
.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year")
.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/en/Mangairo.kt
View File

@@ -108,7 +108,7 @@ internal class Mangairo(context: MangaLoaderContext) :
}
}
private suspend fun fetchAvailableTags(): Set<MangaTag> {
override suspend fun fetchAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl/type-latest/ctg-all/state-all/page-1").parseHtml()
return doc.select("div.panel_category a:not(.ctg_select)").mapNotNullToSet { a ->
val key = a.attr("href").substringAfterLast("ctg-").substringBefore("/")

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/en/Mangakakalot.kt
View File

@@ -97,7 +97,7 @@ internal class Mangakakalot(context: MangaLoaderContext) :
}
}
private suspend fun fetchAvailableTags(): Set<MangaTag> {
override suspend fun fetchAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
val tags = doc.select("ul.tag li a").drop(1)
return tags.mapNotNullToSet { a ->

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangabox/en/MangakakalotTv.kt
View File

@@ -128,7 +128,7 @@ internal class MangakakalotTv(context: MangaLoaderContext) :
override val selectTagMap = "ul.tag li a"
private suspend fun fetchAvailableTags(): Set<MangaTag> {
override suspend fun fetchAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
return doc.select(selectTagMap).mapNotNullToSet { a ->
MangaTag(

1
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/MmrcmsParser.kt
View File

@@ -66,7 +66,6 @@ internal abstract class MmrcmsParser(
override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities(
isMultipleTagsSupported = false,
isSearchSupported = true,
)

59
src/main/kotlin/org/koitharu/kotatsu/parsers/site/otakusanctuary/OtakuSanctuaryParser.kt
View File

@@ -241,61 +241,30 @@ internal abstract class OtakuSanctuaryParser(
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") || d.endsWith(" atrás") || d.startsWith("cách đây ") -> parseRelativeDate(date)
WordSet(" ago", " atrás").endsWith(d) -> { parseRelativeDate(d) }
WordSet("cách đây ").startsWith(d) -> { parseRelativeDate(d) }
else -> dateFormat.tryParse(date)
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet(
"day",
"days",
"d",
"ngày",
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet(
"tiếng",
"hour",
"hours",
).anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet(
"min",
"minute",
"minutes",
"phút",
).anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("second", "giây").anyWordIn(date) -> cal.apply {
add(
Calendar.SECOND,
-number,
)
}.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
WordSet("second", "giây")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes", "phút")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("tiếng", "hour", "hours")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days", "d", "ngày")
.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year")
.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}

24
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/MangaWtfParser.kt
View File

@@ -71,7 +71,7 @@ internal class MangaWtfParser(
SortOrder.POPULARITY -> "viewsCount,desc"
SortOrder.RATING -> "likesCount,desc"
SortOrder.NEWEST -> "createdAt,desc"
else -> throw IllegalArgumentException("Unsupported ${order}")
else -> throw IllegalArgumentException("Unsupported $order")
},
)
if (filter.tags.isNotEmpty()) {
@@ -137,13 +137,13 @@ internal class MangaWtfParser(
tags = jo.getJSONArray("labels").mapJSONToSet { it.toMangaTag() },
state = jo.getStringOrNull("status")?.toMangaState(),
author =
jo.getJSONArray("relations").toJSONList().firstNotNullOfOrNull {
if (it.getStringOrNull("type") == "AUTHOR") {
it.getJSONObject("publisher").getStringOrNull("name")
} else {
null
}
},
jo.getJSONArray("relations").toJSONList().firstNotNullOfOrNull {
if (it.getStringOrNull("type") == "AUTHOR") {
it.getJSONObject("publisher").getStringOrNull("name")
} else {
null
}
},
source = source,
largeCoverUrl = null,
description = jo.getString("description").nl2br(),
@@ -209,10 +209,10 @@ internal class MangaWtfParser(
MangaChapter(
id = generateUid(jo.getString("id")),
name =
jo.getStringOrNull("name") ?: buildString {
if (volume > 0) append("Том ").append(volume).append(' ')
if (number > 0) append("Глава ").append(number) else append("Без имени")
},
jo.getStringOrNull("name") ?: buildString {
if (volume > 0) append("Том ").append(volume).append(' ')
if (number > 0) append("Глава ").append(number) else append("Без имени")
},
number = number,
volume = volume,
url = jo.getString("id"),

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/RemangaParser.kt
View File

@@ -174,7 +174,8 @@ internal class RemangaParser(
MangaChapter(
id = generateUid(id),
url = "/api/titles/chapters/$id/",
number = jo.getIntOrDefault("index", chapters.size - i),
number = jo.getIntOrDefault("index", chapters.size - i).toFloat(),
volume = 0,
name = buildString {
append("Том ")
append(jo.optString("tome", "0"))

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/multichan/ChanParser.kt
View File

@@ -216,7 +216,7 @@ internal abstract class ChanParser(
when (order) {
SortOrder.RATING,
SortOrder.POPULARITY,
-> "favdesc"
-> "favdesc"
SortOrder.ALPHABETICAL -> "abcasc"
else -> "" // SortOrder.NEWEST

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/rulib/LibSocialParser.kt
View File

@@ -95,7 +95,7 @@ internal abstract class LibSocialParser(
SortOrder.NEWEST -> "created_at"
SortOrder.ALPHABETICAL,
SortOrder.ALPHABETICAL_DESC,
-> "rus_name"
-> "rus_name"
else -> null
},
@@ -108,7 +108,7 @@ internal abstract class LibSocialParser(
SortOrder.RATING,
SortOrder.NEWEST,
SortOrder.ALPHABETICAL_DESC,
-> "desc"
-> "desc"
SortOrder.ALPHABETICAL -> "asc"
else -> null

5
src/main/kotlin/org/koitharu/kotatsu/parsers/site/scan/fr/MangaFr.kt
View File

@@ -12,8 +12,9 @@ internal class MangaFr(context: MangaLoaderContext) :
ScanParser(context, MangaParserSource.MANGAFR, "www.mangafr.org") {
override val listUrl = "/series"
private suspend fun fetchAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = emptySet()
)
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val dateFormat = SimpleDateFormat("MM-dd-yyyy", sourceLocale)

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/MangaInUaParser.kt
View File

@@ -124,7 +124,8 @@ internal class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
prevChapterName = name
name
},
number = i,
number = i.toFloat(),
volume = 0,
url = href,
scanlator = null,
branch = if (isAlternative) {

57
src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/WpComicsParser.kt
View File

@@ -279,24 +279,17 @@ internal abstract class WpComicsParser(
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") ||
d.endsWith(" trước")
-> parseRelativeDate(date)
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1)
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet(" ago", " trước").endsWith(d) -> { parseRelativeDate(d) }
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet("today").startsWith(d) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
@@ -314,28 +307,16 @@ internal abstract class WpComicsParser(
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second", "giây").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes", "mins", "phút").anyWordIn(date) -> cal.apply {
add(Calendar.MINUTE, -number)
}.timeInMillis
WordSet("jam", "saat", "heure", "hora", "horas", "hour", "hours", "h", "giờ").anyWordIn(date) -> cal.apply {
add(Calendar.HOUR, -number)
}.timeInMillis
WordSet("day", "days", "d", "ngày").anyWordIn(date) -> cal.apply {
add(Calendar.DAY_OF_MONTH, -number)
}.timeInMillis
WordSet("month", "months", "tháng").anyWordIn(date) -> cal.apply {
add(
Calendar.MONTH,
-number,
)
}.timeInMillis
WordSet("second", "giây")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes", "mins", "phút")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("jam", "saat", "heure", "hora", "horas", "hour", "hours", "h", "giờ")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days", "d", "ngày")
.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months", "tháng")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year", "năm").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/en/XoxoComics.kt
View File

@@ -110,7 +110,7 @@ internal class XoxoComics(context: MangaLoaderContext) :
source = source,
)
}
val result = list.associateByTo(ArrayMap<String, MangaTag>(list.size)) { it.title }
val result = list.associateByTo(ArrayMap(list.size)) { it.title }
tagCache = result
result
}

75
src/main/kotlin/org/koitharu/kotatsu/parsers/site/zmanga/ZMangaParser.kt
View File

@@ -250,30 +250,18 @@ internal abstract class ZMangaParser(
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") ||
// short Hours
d.endsWith(" h") ||
// short Day
d.endsWith(" d") -> parseRelativeDate(date)
WordSet(" ago", " h", " d").endsWith(d) -> { parseRelativeDate(d) }
// Handle 'yesterday' and 'today', using midnight
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
WordSet("today").startsWith(d) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
@@ -287,45 +275,22 @@ internal abstract class ZMangaParser(
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet(
"day",
"days",
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("hour", "hours", "h").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet(
"min",
"minute",
"minutes",
).anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("second").anyWordIn(date) -> cal.apply {
add(
Calendar.SECOND,
-number,
)
}.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
WordSet("second")
.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes")
.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours", "h")
.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months")
.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year")
.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}