[Grouple] Selecting the fastest pages server

pull/154/head
Koitharu 3 years ago
parent e51b33c74a
commit 454b24ec88
No known key found for this signature in database
GPG Key ID: 8E861F8CE6E7CE27

@ -1,5 +1,7 @@
package org.koitharu.kotatsu.parsers.site.grouple
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
@ -27,6 +29,9 @@ internal abstract class GroupleParser(
private val siteId: Int,
) : MangaParser(context, source), MangaParserAuthProvider {
@Volatile
private var cachedPagesServer: String? = null
override val headers = Headers.Builder()
.add("User-Agent", userAgent)
.build()
@ -226,14 +231,19 @@ internal abstract class GroupleParser(
val parts = page.url.split('|')
val path = parts.last()
val servers = parts.dropLast(1).toSet()
for (server in servers) {
val url = server + path
if (tryHead(url)) {
return url
val cachedServer = cachedPagesServer
if (cachedServer != null && cachedServer in servers && tryHead(cachedServer + path)) {
return cachedServer + path
}
val server = coroutineScope {
servers.map { server ->
async {
if (tryHead(server + path)) server else null
}
}.awaitFirst { it != null }
}
val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url)
return fallbackServer + path
cachedPagesServer = server
return checkNotNull(server + path)
}
override suspend fun getTags(): Set<MangaTag> {

@ -0,0 +1,40 @@
package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Job
import kotlinx.coroutines.selects.select
import kotlin.coroutines.cancellation.CancellationException
fun Iterable<Job>.cancelAll(cause: CancellationException? = null) {
forEach { it.cancel(cause) }
}
suspend fun <T> Iterable<Deferred<T>>.awaitFirst(): T = select<T> {
for (async in this@awaitFirst) {
async.onAwait { it }
}
}.also { this@awaitFirst.cancelAll() }
suspend fun <T> Collection<Deferred<T>>.awaitFirst(condition: (T) -> Boolean): T {
var result: Any? = NULL
var counter = size
while (result === NULL && counter > 0) {
val candidate = select<T> {
for (async in this@awaitFirst) {
async.onAwait { it }
}
}
if (condition(candidate)) {
result = candidate
}
counter--
}
cancelAll()
if (result === NULL) {
throw NoSuchElementException()
}
@Suppress("UNCHECKED_CAST")
return result as T
}
private val NULL = Any()
Loading…
Cancel
Save