KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Batch sources fix

Browse Source
This commit is contained in:
Koitharu
2023-06-28 12:05:41 +03:00
parent da4566f82f
commit 06d2976bb5
17 changed files with 64 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
View File

@@ -5,6 +5,7 @@ import okhttp3.Headers
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.FaviconParser
import org.koitharu.kotatsu.parsers.util.domain
@@ -31,12 +32,14 @@ abstract class MangaParser @InternalParsersApi constructor(
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [getDomain] instead.
* Never hardcode domain in requests, use [domain] instead.
*/
@InternalParsersApi
abstract val configKeyDomain: ConfigKey.Domain
open val headers: Headers? = null
open val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_MOBILE)
.build()
/**
* Used as fallback if value of `sortOrder` passed to [getList] is null

12
src/main/kotlin/org/koitharu/kotatsu/parsers/network/UserAgents.kt Normal file
View File

@@ -0,0 +1,12 @@
package org.koitharu.kotatsu.parsers.network
object UserAgents {
const val CHROME_MOBILE =
"Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.196 Mobile Safari/537.36"
const val CHROME_DESKTOP =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
const val KOTATSU = "Kotatsu/5.3 (Android 12;;; en)"
}

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt
View File

@@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault
import java.util.*
@@ -29,7 +30,7 @@ internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser(
override val configKeyDomain = ConfigKey.Domain("bentomanga.com", "www.bentomanga.com")
override val headers: Headers = Headers.Builder()
.add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0")
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
init {

6
src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt
View File

@@ -1,7 +1,6 @@
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import okhttp3.Headers
import okhttp3.Interceptor
import okhttp3.Response
import org.json.JSONArray
@@ -36,11 +35,6 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context,
private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources"
override val headers
get() = Headers.Builder()
.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0")
.build()
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("honey-manga.com.ua")

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt
View File

@@ -9,6 +9,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
@@ -23,7 +24,7 @@ internal class JapScanParser(context: MangaLoaderContext) : PagedMangaParser(con
override val configKeyDomain = ConfigKey.Domain("www.japscan.lol", "japscan.ws")
override val headers: Headers = Headers.Builder()
.add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0")
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(

6
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt
View File

@@ -1,6 +1,5 @@
package org.koitharu.kotatsu.parsers.site
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.Response
@@ -25,8 +24,9 @@ internal abstract class NineMangaParser(
context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes")
}
override val headers = Headers.Builder().add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0").build()
override val headers = super.headers.newBuilder()
.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.build()
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt
View File

@@ -12,6 +12,7 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.net.URLDecoder
@@ -29,7 +30,7 @@ internal class RemangaParser(
) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
private val baseHeaders = Headers.Builder()
.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0")
.add("User-Agent", UserAgents.CHROME_MOBILE)
.build()
override val headers

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/AstralManga.kt
View File

@@ -14,7 +14,7 @@ import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
@MangaSourceParser("ASTRALMANGA", "AstralManga", "fr")
internal class AstralManga(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.ASTRALMANGA, "astral-manga.fr", pageSize = 10) {
Madara6Parser(context, MangaSource.ASTRALMANGA, "astral-manga.fr", pageSize = 12) {
override val datePattern = "dd/MM/yyyy"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt
View File

@@ -13,7 +13,7 @@ import java.util.*
@MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en")
internal class Hentai4Free(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") {
MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net", pageSize = 24) {
override val tagPrefix = "hentai-tag/"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentaiteca.kt
View File

@@ -14,7 +14,7 @@ import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
@MangaSourceParser("HENTAITECA", "Hentaiteca", "pt")
internal class Hentaiteca(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.HENTAITECA, "hentaiteca.net") {
Madara6Parser(context, MangaSource.HENTAITECA, "hentaiteca.net", pageSize = 10) {
override val datePattern = "MM/dd/yyyy"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentaizone.kt
View File

@@ -15,7 +15,7 @@ import java.util.*
@MangaSourceParser("HENTAIZONE", "Hentaizone", "fr")
internal class Hentaizone(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.HENTAIZONE, "hentaizone.xyz") {
Madara6Parser(context, MangaSource.HENTAIZONE, "hentaizone.xyz", pageSize = 10) {
override val datePattern = "MMM d, yyyy"
override val sourceLocale: Locale = Locale.FRENCH

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hipercool.kt
View File

@@ -15,7 +15,7 @@ import java.util.*
@MangaSourceParser("HIPERCOOL", "Hipercool", "pt")
internal class Hipercool(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.HIPERCOOL, "hipercool.xyz") {
Madara6Parser(context, MangaSource.HIPERCOOL, "hipercool.xyz", pageSize = 20) {
override val datePattern = "MMMM d, yyyy"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt
View File

@@ -12,7 +12,7 @@ import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("MANGALINK_AR", "Mangalink", "ar")
internal class MangalinkParser(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") {
MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online", pageSize = 10) {
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt
View File

@@ -7,7 +7,7 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "in")
@MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "id")
internal class ReaperScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") {

62
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt
View File

@@ -58,18 +58,14 @@ internal abstract class MangaReaderParser(
open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List<MangaChapter>): Manga {
val mangaInfo =
docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?:
docs.selectFirst("div.seriestucontentr") ?:
docs.selectFirst("div.seriestucon")
docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?: docs.selectFirst("div.seriestucontentr")
?: docs.selectFirst("div.seriestucon")
val state_select =
docs.selectFirst(".tsinfo div:contains(Status)") ?:
docs.selectFirst(".tsinfo div:contains(Statut)") ?:
docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?:
docs.selectFirst(".tsinfo div:contains(Estado)") ?:
docs.selectFirst(".tsinfo div:contains(สถานะ)") ?:
docs.selectFirst(".tsinfo div:contains(Stato )") ?:
docs.selectFirst(".tsinfo div:contains(Durum)")
docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)")
?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)")
?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )")
?: docs.selectFirst(".tsinfo div:contains(Durum)")
val mangaState = state_select?.lastElementChild()?.let {
when (it.text()) {
@@ -86,8 +82,9 @@ internal abstract class MangaReaderParser(
"Publishing",
"Devam Ediyor",
"Em Andamento",
"In Corso"
"In Corso",
-> MangaState.ONGOING
"Completed",
"Completo",
"Complété",
@@ -99,8 +96,9 @@ internal abstract class MangaReaderParser(
"Завершено",
"Finished",
"Finalizado",
"Completata"
"Completata",
-> MangaState.FINISHED
else -> null
}
}
@@ -120,13 +118,10 @@ internal abstract class MangaReaderParser(
open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List<MangaChapter>): Manga {
val state_select =
docs.selectFirst(".tsinfo div:contains(Status)") ?:
docs.selectFirst(".tsinfo div:contains(Statut)") ?:
docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?:
docs.selectFirst(".tsinfo div:contains(Estado)") ?:
docs.selectFirst(".tsinfo div:contains(สถานะ)") ?:
docs.selectFirst(".tsinfo div:contains(Stato )") ?:
docs.selectFirst(".tsinfo div:contains(Durum)")
docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)")
?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)")
?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )")
?: docs.selectFirst(".tsinfo div:contains(Durum)")
val mangaState = state_select?.lastElementChild()?.let {
when (it.text()) {
@@ -143,8 +138,9 @@ internal abstract class MangaReaderParser(
"Publishing",
"Devam Ediyor",
"Em Andamento",
"In Corso"
"In Corso",
-> MangaState.ONGOING
"Completed",
"Completo",
"Complété",
@@ -156,8 +152,9 @@ internal abstract class MangaReaderParser(
"Завершено",
"Finished",
"Finalizado",
"Completata"
"Completata",
-> MangaState.FINISHED
else -> null
}
}
@@ -170,21 +167,20 @@ internal abstract class MangaReaderParser(
state = mangaState,
author =
docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.text() ?:
docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.text() ?:
docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.text() ?:
docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.text() ,
docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.text()
?: docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.text()
?: docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.text()
?: docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.text(),
isNsfw = manga.isNsfw
|| docs.selectFirst(".info-right .alr") != null
|| docs.selectFirst(".postbody .alr") != null,
|| docs.selectFirst(".info-right .alr") != null
|| docs.selectFirst(".postbody .alr") != null,
tags = tags,
chapters = chapters,
)
}
override suspend fun getListPage(
page: Int,
query: String?,
@@ -766,7 +762,7 @@ internal abstract class MangaReaderParser(
// Fr site //
@MangaSourceParser("PHENIXSCANS", "Phenixscans", "fr")
class PhenixscansParser(context: MangaLoaderContext) :
class PhenixscansParser(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.PHENIXSCANS, pageSize = 20, searchPageSize = 10) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("phenixscans.fr")
@@ -810,7 +806,7 @@ internal abstract class MangaReaderParser(
}
@MangaSourceParser("EPSILONSCAN", "Epsilonscan", "fr")
class EpsilonscanParser(context: MangaLoaderContext) :
class EpsilonscanParser(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.EPSILONSCAN, pageSize = 20, searchPageSize = 10) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("epsilonscan.fr")
@@ -819,13 +815,13 @@ internal abstract class MangaReaderParser(
get() = "/manga"
override val tableMode: Boolean
get() = false
override val isNsfwSource: Boolean = true
override val isNsfwSource: Boolean = true
override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.FRENCH)
}
@MangaSourceParser("LEGACY_SCANS", "Legacy Scans", "fr")
class LegacyScansParser(context: MangaLoaderContext) :
class LegacyScansParser(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.LEGACY_SCANS, pageSize = 20, searchPageSize = 10) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("legacy-scans.com")
@@ -833,7 +829,7 @@ internal abstract class MangaReaderParser(
override val listUrl: String
get() = "/manga"
override val tableMode: Boolean
get() = false
get() = false
override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.FRENCH)
}

8
src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt
View File

@@ -7,12 +7,9 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.mergeWith
private const val HEADER_USER_AGENT = "User-Agent"
private const val HEADER_REFERER = "Referer"
internal class CommonHeadersInterceptor(
private val userAgent: String,
) : Interceptor {
internal class CommonHeadersInterceptor() : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
@@ -23,9 +20,6 @@ internal class CommonHeadersInterceptor(
if (sourceHeaders != null) {
headersBuilder.mergeWith(sourceHeaders, replaceExisting = false)
}
if (headersBuilder[HEADER_USER_AGENT] == null) {
headersBuilder[HEADER_USER_AGENT] = userAgent
}
if (headersBuilder[HEADER_REFERER] == null && parser != null) {
headersBuilder[HEADER_REFERER] = "https://${parser.domain}/"
}

11
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt
View File

@@ -11,20 +11,11 @@ import java.util.concurrent.TimeUnit
internal object MangaLoaderContextMock : MangaLoaderContext() {
private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
/*BuildConfig.VERSION_NAME*/ "4.4",
/*Build.VERSION.RELEASE*/ "r",
/*Build.MODEL*/ "",
/*Build.BRAND*/ "",
/*Build.DEVICE*/ "",
/*Locale.getDefault().language*/ "en",
)
override val cookieJar = InMemoryCookieJar()
override val httpClient: OkHttpClient = OkHttpClient.Builder()
.cookieJar(cookieJar)
.addInterceptor(CommonHeadersInterceptor(userAgent))
.addInterceptor(CommonHeadersInterceptor())
.addInterceptor(CloudFlareInterceptor())
.connectTimeout(20, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)