Batch sources fix

src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt

@@ -5,6 +5,7 @@ import okhttp3.Headers
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
+import org.koitharu.kotatsu.parsers.network.UserAgents
 import org.koitharu.kotatsu.parsers.network.WebClient
 import org.koitharu.kotatsu.parsers.util.FaviconParser
 import org.koitharu.kotatsu.parsers.util.domain
@@ -31,12 +32,14 @@ abstract class MangaParser @InternalParsersApi constructor(
 	/**
 	 * Provide default domain and available alternatives, if any.
 	 *
-	 * Never hardcode domain in requests, use [getDomain] instead.
+	 * Never hardcode domain in requests, use [domain] instead.
 	 */
 	@InternalParsersApi
 	abstract val configKeyDomain: ConfigKey.Domain
 
-	open val headers: Headers? = null
+	open val headers: Headers = Headers.Builder()
+		.add("User-Agent", UserAgents.CHROME_MOBILE)
+		.build()
 
 	/**
 	 * Used as fallback if value of `sortOrder` passed to [getList] is null

src/main/kotlin/org/koitharu/kotatsu/parsers/network/UserAgents.kt

@@ -0,0 +1,12 @@
+package org.koitharu.kotatsu.parsers.network
+
+object UserAgents {
+
+	const val CHROME_MOBILE =
+		"Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.196 Mobile Safari/537.36"
+
+	const val CHROME_DESKTOP =
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
+
+	const val KOTATSU = "Kotatsu/5.3 (Android 12;;; en)"
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt

@@ -11,6 +11,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.network.UserAgents
 import org.koitharu.kotatsu.parsers.util.*
 import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault
 import java.util.*
@@ -29,7 +30,7 @@ internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser(
 	override val configKeyDomain = ConfigKey.Domain("bentomanga.com", "www.bentomanga.com")
 
 	override val headers: Headers = Headers.Builder()
-		.add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0")
+		.add("User-Agent", UserAgents.CHROME_DESKTOP)
 		.build()
 
 	init {

src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt

@@ -1,7 +1,6 @@
 package org.koitharu.kotatsu.parsers.site
 
 import androidx.collection.ArraySet
-import okhttp3.Headers
 import okhttp3.Interceptor
 import okhttp3.Response
 import org.json.JSONArray
@@ -36,11 +35,6 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context,
 
 	private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources"
 
-	override val headers
-		get() = Headers.Builder()
-			.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0")
-			.build()
-
 	override val configKeyDomain: ConfigKey.Domain
 		get() = ConfigKey.Domain("honey-manga.com.ua")
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt

@@ -9,6 +9,7 @@ import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.network.UserAgents
 import org.koitharu.kotatsu.parsers.util.*
 import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
 import org.koitharu.kotatsu.parsers.util.json.mapJSON
@@ -23,7 +24,7 @@ internal class JapScanParser(context: MangaLoaderContext) : PagedMangaParser(con
 	override val configKeyDomain = ConfigKey.Domain("www.japscan.lol", "japscan.ws")
 
 	override val headers: Headers = Headers.Builder()
-		.add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0")
+		.add("User-Agent", UserAgents.CHROME_DESKTOP)
 		.build()
 
 	override suspend fun getListPage(

src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt

@@ -1,6 +1,5 @@
 package org.koitharu.kotatsu.parsers.site
 
-import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.Interceptor
 import okhttp3.Response
@@ -25,8 +24,9 @@ internal abstract class NineMangaParser(
 		context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes")
 	}
 
-	override val headers = Headers.Builder().add("Accept-Language", "en-US;q=0.7,en;q=0.3")
-		.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0").build()
+	override val headers = super.headers.newBuilder()
+		.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
+		.build()
 
 	override val sortOrders: Set<SortOrder> = Collections.singleton(
 		SortOrder.POPULARITY,

src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt

@@ -12,6 +12,7 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.network.UserAgents
 import org.koitharu.kotatsu.parsers.util.*
 import org.koitharu.kotatsu.parsers.util.json.*
 import java.net.URLDecoder
@@ -29,7 +30,7 @@ internal class RemangaParser(
 ) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
 
 	private val baseHeaders = Headers.Builder()
-		.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0")
+		.add("User-Agent", UserAgents.CHROME_MOBILE)
 		.build()
 
 	override val headers

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/AstralManga.kt

@@ -14,7 +14,7 @@ import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
 
 @MangaSourceParser("ASTRALMANGA", "AstralManga", "fr")
 internal class AstralManga(context: MangaLoaderContext) :
-	Madara6Parser(context, MangaSource.ASTRALMANGA, "astral-manga.fr", pageSize = 10) {
+	Madara6Parser(context, MangaSource.ASTRALMANGA, "astral-manga.fr", pageSize = 12) {
 
 	override val datePattern = "dd/MM/yyyy"
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt

@@ -13,7 +13,7 @@ import java.util.*
 
 @MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en")
 internal class Hentai4Free(context: MangaLoaderContext) :
-	MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") {
+	MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net", pageSize = 24) {
 
 	override val tagPrefix = "hentai-tag/"
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentaiteca.kt

@@ -14,7 +14,7 @@ import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
 
 @MangaSourceParser("HENTAITECA", "Hentaiteca", "pt")
 internal class Hentaiteca(context: MangaLoaderContext) :
-	Madara6Parser(context, MangaSource.HENTAITECA, "hentaiteca.net") {
+	Madara6Parser(context, MangaSource.HENTAITECA, "hentaiteca.net", pageSize = 10) {
 
 	override val datePattern = "MM/dd/yyyy"
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentaizone.kt

@@ -15,7 +15,7 @@ import java.util.*
 
 @MangaSourceParser("HENTAIZONE", "Hentaizone", "fr")
 internal class Hentaizone(context: MangaLoaderContext) :
-	Madara6Parser(context, MangaSource.HENTAIZONE, "hentaizone.xyz") {
+	Madara6Parser(context, MangaSource.HENTAIZONE, "hentaizone.xyz", pageSize = 10) {
 
 	override val datePattern = "MMM d, yyyy"
 	override val sourceLocale: Locale = Locale.FRENCH

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hipercool.kt

@@ -15,7 +15,7 @@ import java.util.*
 
 @MangaSourceParser("HIPERCOOL", "Hipercool", "pt")
 internal class Hipercool(context: MangaLoaderContext) :
-	Madara6Parser(context, MangaSource.HIPERCOOL, "hipercool.xyz") {
+	Madara6Parser(context, MangaSource.HIPERCOOL, "hipercool.xyz", pageSize = 20) {
 
 	override val datePattern = "MMMM d, yyyy"
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt

@@ -12,7 +12,7 @@ import org.koitharu.kotatsu.parsers.util.*
 
 @MangaSourceParser("MANGALINK_AR", "Mangalink", "ar")
 internal class MangalinkParser(context: MangaLoaderContext) :
-	MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") {
+	MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online", pageSize = 10) {
 
 	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
 		val fullUrl = manga.url.toAbsoluteUrl(domain)

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt

@@ -7,7 +7,7 @@ import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
 import java.util.*
 
-@MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "in")
+@MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "id")
 internal class ReaperScansParser(context: MangaLoaderContext) :
 	Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") {
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt

@@ -58,18 +58,14 @@ internal abstract class MangaReaderParser(
 
 	open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List<MangaChapter>): Manga {
 		val mangaInfo =
-			docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?:
-			docs.selectFirst("div.seriestucontentr") ?:
-			docs.selectFirst("div.seriestucon")
+			docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?: docs.selectFirst("div.seriestucontentr")
+			?: docs.selectFirst("div.seriestucon")
 
 		val state_select =
-			docs.selectFirst(".tsinfo div:contains(Status)") ?:
-			docs.selectFirst(".tsinfo div:contains(Statut)") ?:
-			docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?:
-			docs.selectFirst(".tsinfo div:contains(Estado)") ?:
-			docs.selectFirst(".tsinfo div:contains(สถานะ)") ?:
-			docs.selectFirst(".tsinfo div:contains(Stato )") ?:
-			docs.selectFirst(".tsinfo div:contains(Durum)")
+			docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)")
+			?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)")
+			?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )")
+			?: docs.selectFirst(".tsinfo div:contains(Durum)")
 
 		val mangaState = state_select?.lastElementChild()?.let {
 			when (it.text()) {
@@ -86,8 +82,9 @@ internal abstract class MangaReaderParser(
 				"Publishing",
 				"Devam Ediyor",
 				"Em Andamento",
-				"In Corso"
+				"In Corso",
 				-> MangaState.ONGOING
+
 				"Completed",
 				"Completo",
 				"Complété",
@@ -99,8 +96,9 @@ internal abstract class MangaReaderParser(
 				"Завершено",
 				"Finished",
 				"Finalizado",
-				"Completata"
+				"Completata",
 				-> MangaState.FINISHED
+
 				else -> null
 			}
 		}
@@ -120,13 +118,10 @@ internal abstract class MangaReaderParser(
 	open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List<MangaChapter>): Manga {
 
 		val state_select =
-			docs.selectFirst(".tsinfo div:contains(Status)") ?:
-			docs.selectFirst(".tsinfo div:contains(Statut)") ?:
-			docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?:
-			docs.selectFirst(".tsinfo div:contains(Estado)") ?:
-			docs.selectFirst(".tsinfo div:contains(สถานะ)") ?:
-			docs.selectFirst(".tsinfo div:contains(Stato )") ?:
-			docs.selectFirst(".tsinfo div:contains(Durum)")
+			docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)")
+			?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)")
+			?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )")
+			?: docs.selectFirst(".tsinfo div:contains(Durum)")
 
 		val mangaState = state_select?.lastElementChild()?.let {
 			when (it.text()) {
@@ -143,8 +138,9 @@ internal abstract class MangaReaderParser(
 				"Publishing",
 				"Devam Ediyor",
 				"Em Andamento",
-				"In Corso"
+				"In Corso",
 				-> MangaState.ONGOING
+
 				"Completed",
 				"Completo",
 				"Complété",
@@ -156,8 +152,9 @@ internal abstract class MangaReaderParser(
 				"Завершено",
 				"Finished",
 				"Finalizado",
-				"Completata"
+				"Completata",
 				-> MangaState.FINISHED
+
 				else -> null
 			}
 		}
@@ -170,21 +167,20 @@ internal abstract class MangaReaderParser(
 
 			state = mangaState,
 			author =
-			docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.text() ?:
-			docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.text() ?:
-			docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.text() ?:
-			docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.text() ,
+			docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.text()
+				?: docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.text()
+				?: docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.text()
+				?: docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.text(),
 
 			isNsfw = manga.isNsfw
-					|| docs.selectFirst(".info-right .alr") != null
-					|| docs.selectFirst(".postbody .alr") != null,
+				|| docs.selectFirst(".info-right .alr") != null
+				|| docs.selectFirst(".postbody .alr") != null,
 			tags = tags,
 			chapters = chapters,
 		)
 	}
 
 
-
 	override suspend fun getListPage(
 		page: Int,
 		query: String?,
@@ -766,7 +762,7 @@ internal abstract class MangaReaderParser(
 	// Fr site //
 
 	@MangaSourceParser("PHENIXSCANS", "Phenixscans", "fr")
-    class PhenixscansParser(context: MangaLoaderContext) :
+	class PhenixscansParser(context: MangaLoaderContext) :
 		MangaReaderParser(context, MangaSource.PHENIXSCANS, pageSize = 20, searchPageSize = 10) {
 		override val configKeyDomain: ConfigKey.Domain
 			get() = ConfigKey.Domain("phenixscans.fr")
@@ -810,7 +806,7 @@ internal abstract class MangaReaderParser(
 	}
 
 	@MangaSourceParser("EPSILONSCAN", "Epsilonscan", "fr")
-    class EpsilonscanParser(context: MangaLoaderContext) :
+	class EpsilonscanParser(context: MangaLoaderContext) :
 		MangaReaderParser(context, MangaSource.EPSILONSCAN, pageSize = 20, searchPageSize = 10) {
 		override val configKeyDomain: ConfigKey.Domain
 			get() = ConfigKey.Domain("epsilonscan.fr")
@@ -819,13 +815,13 @@ internal abstract class MangaReaderParser(
 			get() = "/manga"
 		override val tableMode: Boolean
 			get() = false
-        override val isNsfwSource: Boolean = true    
+		override val isNsfwSource: Boolean = true
 
 		override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.FRENCH)
 	}
 
 	@MangaSourceParser("LEGACY_SCANS", "Legacy Scans", "fr")
-    class LegacyScansParser(context: MangaLoaderContext) :
+	class LegacyScansParser(context: MangaLoaderContext) :
 		MangaReaderParser(context, MangaSource.LEGACY_SCANS, pageSize = 20, searchPageSize = 10) {
 		override val configKeyDomain: ConfigKey.Domain
 			get() = ConfigKey.Domain("legacy-scans.com")
@@ -833,7 +829,7 @@ internal abstract class MangaReaderParser(
 		override val listUrl: String
 			get() = "/manga"
 		override val tableMode: Boolean
-			get() = false  
+			get() = false
 
 		override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.FRENCH)
 	}

src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt

@@ -7,12 +7,9 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
 import org.koitharu.kotatsu.parsers.util.domain
 import org.koitharu.kotatsu.parsers.util.mergeWith
 
-private const val HEADER_USER_AGENT = "User-Agent"
 private const val HEADER_REFERER = "Referer"
 
-internal class CommonHeadersInterceptor(
-	private val userAgent: String,
-) : Interceptor {
+internal class CommonHeadersInterceptor() : Interceptor {
 
 	override fun intercept(chain: Interceptor.Chain): Response {
 		val request = chain.request()
@@ -23,9 +20,6 @@ internal class CommonHeadersInterceptor(
 		if (sourceHeaders != null) {
 			headersBuilder.mergeWith(sourceHeaders, replaceExisting = false)
 		}
-		if (headersBuilder[HEADER_USER_AGENT] == null) {
-			headersBuilder[HEADER_USER_AGENT] = userAgent
-		}
 		if (headersBuilder[HEADER_REFERER] == null && parser != null) {
 			headersBuilder[HEADER_REFERER] = "https://${parser.domain}/"
 		}

src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt

@@ -11,20 +11,11 @@ import java.util.concurrent.TimeUnit
 
 internal object MangaLoaderContextMock : MangaLoaderContext() {
 
-	private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
-		/*BuildConfig.VERSION_NAME*/ "4.4",
-		/*Build.VERSION.RELEASE*/ "r",
-		/*Build.MODEL*/ "",
-		/*Build.BRAND*/ "",
-		/*Build.DEVICE*/ "",
-		/*Locale.getDefault().language*/ "en",
-	)
-
 	override val cookieJar = InMemoryCookieJar()
 
 	override val httpClient: OkHttpClient = OkHttpClient.Builder()
 		.cookieJar(cookieJar)
-		.addInterceptor(CommonHeadersInterceptor(userAgent))
+		.addInterceptor(CommonHeadersInterceptor())
 		.addInterceptor(CloudFlareInterceptor())
 		.connectTimeout(20, TimeUnit.SECONDS)
 		.readTimeout(60, TimeUnit.SECONDS)