Merge remote-tracking branch 'origin/master'

src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ComickFunParser.kt

@@ -188,7 +188,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
 					chap?.let { append("Chap ").append(it) }
 					jo.getStringOrNull("title")?.let { append(": ").append(it) }
 				},
-				number = branchedChapters[branch]?.size?.plus(1) ?: 0,
+				number = branchedChapters[branch]?.size?.plus(1) ?: 1,
 				url = jo.getString("hid"),
 				scanlator = jo.optJSONArray("group_name")?.asIterable<String>()?.joinToString()
 					?.takeUnless { it.isBlank() },

src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/RemangaParser.kt

@@ -1,6 +1,9 @@
 package org.koitharu.kotatsu.parsers.site.ru
 
 import okhttp3.Headers
+import okhttp3.Interceptor
+import okhttp3.Response
+import okhttp3.internal.closeQuietly
 import org.json.JSONArray
 import org.json.JSONException
 import org.json.JSONObject
@@ -23,11 +26,12 @@ import java.util.*
 private const val PAGE_SIZE = 30
 private const val STATUS_ONGOING = 1
 private const val STATUS_FINISHED = 0
+private const val TOO_MANY_REQUESTS = 429
 
 @MangaSourceParser("REMANGA", "ReManga", "ru")
 internal class RemangaParser(
 	context: MangaLoaderContext,
-) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
+) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor {
 
 	private val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE)
 
@@ -59,6 +63,16 @@ internal class RemangaParser(
 
 	private val regexLastUrlPath = Regex("/[^/]+/?$")
 
+	override fun intercept(chain: Interceptor.Chain): Response {
+		val response = chain.proceed(chain.request())
+		if (response.code == TOO_MANY_REQUESTS) {
+			response.closeQuietly()
+			Thread.sleep(1000)
+			return chain.proceed(chain.request().newBuilder().build())
+		}
+		return response
+	}
+
 	override suspend fun getListPage(
 		page: Int,
 		query: String?,
@@ -266,7 +280,7 @@ internal class RemangaParser(
 		var page = 1
 		while (true) {
 			val content = webClient.httpGet(
-				url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100",
+				url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=500",
 			).parseJson().getJSONArray("content")
 			val len = content.length()
 			if (len == 0) {

src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/multichan/ChanParser.kt

@@ -1,5 +1,6 @@
 package org.koitharu.kotatsu.parsers.site.ru.multichan
 
+import okhttp3.HttpUrl
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
@@ -18,6 +19,7 @@ internal abstract class ChanParser(
 		SortOrder.NEWEST,
 		SortOrder.POPULARITY,
 		SortOrder.ALPHABETICAL,
+		SortOrder.RATING,
 	)
 
 	override val authUrl: String
@@ -33,23 +35,7 @@ internal abstract class ChanParser(
 		sortOrder: SortOrder,
 	): List<Manga> {
 		val domain = domain
-		val url = when {
+		val doc = webClient.httpGet(buildUrl(offset, query, tags, sortOrder)).parseHtml()
-			!query.isNullOrEmpty() -> {
-				if (offset != 0) {
-					return emptyList()
-				}
-				"https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}"
-			}
-
-			!tags.isNullOrEmpty() -> tags.joinToString(
-				prefix = "https://$domain/tags/",
-				postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset",
-				separator = "+",
-			) { tag -> tag.key }
-
-			else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset"
-		}
-		val doc = webClient.httpGet(url).parseHtml()
 		val root = doc.body().selectFirst("div.main_fon")?.getElementById("content")
 			?: doc.parseFailed("Cannot find root")
 		return root.select("div.content_row").mapNotNull { row ->
@@ -191,21 +177,54 @@ internal abstract class ChanParser(
 		}
 	}
 
-	private fun getSortKey(sortOrder: SortOrder) =
+	protected open fun buildUrl(
-		when (sortOrder) {
+		offset: Int,
-			SortOrder.ALPHABETICAL -> "catalog"
+		query: String?,
-			SortOrder.POPULARITY -> "mostfavorites"
+		tags: Set<MangaTag>?,
-			SortOrder.NEWEST -> "manga/new"
+		sortOrder: SortOrder,
-			else -> "mostfavorites"
+	): HttpUrl {
-		}
+		val builder = urlBuilder()
+		builder.addQueryParameter("offset", offset.toString())
+		when {
+			!query.isNullOrEmpty() -> {
+				builder.addQueryParameter("do", "search")
+				builder.addQueryParameter("subaction", "search")
+				builder.addQueryParameter("search_start", ((offset / 40) + 1).toString())
+				builder.addQueryParameter("full_search", "0")
+				builder.addQueryParameter("result_from", (offset + 1).toString())
+				builder.addQueryParameter("result_num", "40")
+				builder.addQueryParameter("story", query)
+				builder.addQueryParameter("need_sort_date", "false")
+			}
 
-	private fun getSortKey2(sortOrder: SortOrder) =
+			!tags.isNullOrEmpty() -> {
-		when (sortOrder) {
+				builder.addPathSegment("tags")
-			SortOrder.ALPHABETICAL -> "abcasc"
+				builder.addPathSegment(tags.joinToString("+") { it.key })
-			SortOrder.POPULARITY -> "favdesc"
+				builder.addQueryParameter(
-			SortOrder.NEWEST -> "datedesc"
+					"n",
-			else -> "favdesc"
+					when (sortOrder) {
+						SortOrder.RATING,
+						SortOrder.POPULARITY,
+						-> "favdesc"
+
+						SortOrder.ALPHABETICAL -> "abcasc"
+						else -> "" // SortOrder.NEWEST
+					},
+				)
+			}
+
+			else -> when (sortOrder) {
+				SortOrder.POPULARITY -> builder.addPathSegment("mostviews")
+				SortOrder.ALPHABETICAL -> builder.addPathSegment("catalog")
+				SortOrder.RATING -> builder.addPathSegment("mostfavorites")
+				else -> { // SortOrder.NEWEST
+					builder.addPathSegment("manga")
+					builder.addPathSegment("new")
+				}
+			}
 		}
+		return builder.build()
+	}
 
 	private fun String.toTagName() = replace('_', ' ').toTitleCase()
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/multichan/HenChanParser.kt

@@ -1,15 +1,19 @@
 package org.koitharu.kotatsu.parsers.site.ru.multichan
 
+import okhttp3.HttpUrl
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
+import java.util.*
 
 @MangaSourceParser("HENCHAN", "Хентай-тян", "ru", type = ContentType.HENTAI)
 internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.HENCHAN) {
 
 	override val configKeyDomain = ConfigKey.Domain(
+		"x.henchan.pro",
+		"xxx.henchan.pro",
 		"y.hentaichan.live",
 		"xxx.hentaichan.live",
 		"xx.hentaichan.live",
@@ -17,6 +21,12 @@ internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context,
 		"hentaichan.pro",
 	)
 
+	override val sortOrders: Set<SortOrder> = EnumSet.of(
+		SortOrder.NEWEST,
+		SortOrder.POPULARITY,
+		SortOrder.RATING,
+	)
+
 	override suspend fun getDetails(manga: Manga): Manga {
 		val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
 		val root = doc.body().requireElementById("dle-content")
@@ -46,4 +56,28 @@ internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context,
 			),
 		)
 	}
+
+	override fun buildUrl(offset: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): HttpUrl {
+		if (query.isNullOrEmpty() && tags.isNullOrEmpty()) {
+			val builder = urlBuilder().addQueryParameter("offset", offset.toString())
+			when (sortOrder) {
+				SortOrder.POPULARITY -> {
+					builder.addPathSegment("mostviews")
+					builder.addQueryParameter("sort", "manga")
+				}
+
+				SortOrder.RATING -> {
+					builder.addPathSegment("mostfavorites")
+					builder.addQueryParameter("sort", "manga")
+				}
+
+				else -> { // SortOrder.NEWEST
+					builder.addPathSegment("manga")
+					builder.addPathSegment("newest")
+				}
+			}
+			return builder.build()
+		}
+		return super.buildUrl(offset, query, tags, sortOrder)
+	}
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/multichan/YaoiChanParser.kt

@@ -6,12 +6,18 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaChapter
 import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.model.SortOrder
 import org.koitharu.kotatsu.parsers.util.*
 
 @MangaSourceParser("YAOICHAN", "Яой-тян", "ru")
 internal class YaoiChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.YAOICHAN) {
 
-	override val configKeyDomain = ConfigKey.Domain("yaoi-chan.me")
+	override val configKeyDomain = ConfigKey.Domain(
+		"v1.yaoi-chan.me",
+		"yaoi-chan.me",
+	)
+
+	override val sortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST)
 
 	override suspend fun getDetails(manga: Manga): Manga {
 		val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()

src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/rulib/MangaLibParser.kt

@@ -1,5 +1,6 @@
 package org.koitharu.kotatsu.parsers.site.ru.rulib
 
+import androidx.collection.ArrayMap
 import androidx.collection.ArraySet
 import kotlinx.coroutines.withTimeoutOrNull
 import okhttp3.Response
@@ -17,10 +18,7 @@ import org.koitharu.kotatsu.parsers.exception.NotFoundException
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
-import org.koitharu.kotatsu.parsers.util.json.JSONIterator
+import org.koitharu.kotatsu.parsers.util.json.*
-import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
-import org.koitharu.kotatsu.parsers.util.json.mapJSON
-import org.koitharu.kotatsu.parsers.util.json.values
 import java.text.SimpleDateFormat
 import java.util.*
 
@@ -102,10 +100,15 @@ internal open class MangaLibParser(
 				if (line.startsWith("window.__DATA__")) {
 					val json = JSONObject(line.substringAfter('=').substringBeforeLast(';'))
 					val list = json.getJSONObject("chapters").getJSONArray("list")
+					val branches = json.getJSONObject("chapters").getJSONArray("branches").toJSONList()
+						.associate { x ->
+							x.getInt("id") to x.getJSONArray("teams").toJSONList().joinToString { it.getString("name") }
+						}
 					val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not"
 					val total = list.length()
 					chapters = ChaptersListBuilder(total)
-					for (i in 0 until total) {
+					val counters = ArrayMap<Int, Int>(branches.size)
+					for (i in (0 until total).reversed()) {
 						val item = list.getJSONObject(i)
 						val chapterId = item.getLong("chapter_id")
 						val scanlator = item.getStringOrNull("username")
@@ -130,22 +133,22 @@ internal open class MangaLibParser(
 						val volume = item.getInt("chapter_volume")
 						val number = item.getString("chapter_number")
 						val fullNameChapter = "Том $volume. Глава $number"
+						val branchId = item.getIntOrDefault("branch_id", 0)
 						chapters.add(
 							MangaChapter(
 								id = generateUid(chapterId),
 								url = url,
 								source = source,
-								number = total - i,
+								number = counters.incrementAndGet(branchId),
 								uploadDate = dateFormat.tryParse(
 									item.getString("chapter_created_at").substringBefore(" "),
 								),
 								scanlator = scanlator,
-								branch = null,
+								branch = branches[branchId],
 								name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter",
 							),
 						)
 					}
-					chapters.reverse()
 					break@scripts
 				}
 			}
@@ -301,7 +304,7 @@ internal open class MangaLibParser(
 		return json.mapJSON { jo ->
 			val slug = jo.getString("slug")
 			val url = "/$slug"
-			val covers = jo.getJSONObject("covers")
+			val cover = jo.getJSONObject("covers").getString("default").toAbsoluteUrl(domain)
 			val title = jo.getString("rus_name").ifEmpty { jo.getString("name") }
 			Manga(
 				id = generateUid(url),
@@ -315,8 +318,8 @@ internal open class MangaLibParser(
 				state = null,
 				isNsfw = false,
 				source = source,
-				coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain),
+				coverUrl = cover,
-				largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain),
+				largeCoverUrl = null,
 			)
 		}
 	}
@@ -343,6 +346,13 @@ internal open class MangaLibParser(
 		return isSuccessful && mimeType?.startsWith("image/") == true && headersContentLength() >= 1024L
 	}
 
+	private fun MutableMap<Int, Int>.incrementAndGet(key: Int): Int {
+		var v = getOrDefault(key, 0)
+		v++
+		put(key, v)
+		return v
+	}
+
 	@MangaSourceParser("MANGALIB", "MangaLib", "ru")
 	class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB)