feat: enhance user-agent identification and implement backoff strategy for university source requests
- Added PSU_CONTACT environment variable to docker-compose for owner contact in User-Agent. - Updated source.js to include user-agent construction with service version and contact information. - Implemented exponential backoff strategy for failed requests to the university, with retry logic based on HTTP response status. - Modified status.js to display backoff information in the status response. - Adjusted watch.js to reduce request frequency during university downtime. - Enhanced test coverage with new backoff.test.js to validate backoff behavior and user-agent format.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { parseWeeksDocx, parseBellsDocx } from "./calendar.js";
|
||||
import { apiError, sourceError } from "./errors.js";
|
||||
@@ -43,6 +44,39 @@ const backingOff = (what) => Date.now() - failedAt[what] < MIN_REFRESH;
|
||||
|
||||
const TIMEOUT = Number(process.env.PSU_SOURCE_TIMEOUT_MS) || 25000;
|
||||
const MAX_SIZE = 10 * 1024 * 1024;
|
||||
|
||||
// Every request says who asks and how to reach the owner, so the university's administrators
|
||||
// see a known reader rather than an anonymous robot: "PSU-Schedule/0.1.0 (+https://…; mailto:…)".
|
||||
const VERSION = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
||||
const contact = [process.env.PSU_PUBLIC_URL && `+${process.env.PSU_PUBLIC_URL}`, process.env.PSU_CONTACT].filter(Boolean);
|
||||
export const USER_AGENT = `PSU-Schedule/${VERSION}${contact.length ? ` (${contact.join("; ")})` : ""}`;
|
||||
|
||||
// While the university is down, nothing reaches it until the pause is over: the watcher, a
|
||||
// forced refresh and a document opened by a student all get the saved copy or an error at
|
||||
// once. The pause doubles with every failed attempt (5, 10, 20, 40 minutes…) up to an hour and
|
||||
// ends with the first answer. A failing document alone (HTTP 404, 500) or a slow download does
|
||||
// not pause the rest; the server's whole answering does: no connection, 502–504, 429.
|
||||
const BACKOFF = process.env.PSU_SOURCE_BACKOFF_MS ? Number(process.env.PSU_SOURCE_BACKOFF_MS) : 5 * 60 * 1000;
|
||||
const BACKOFF_MAX = Number(process.env.PSU_SOURCE_BACKOFF_MAX_MS) || 60 * 60 * 1000;
|
||||
const outage = { failures: 0, until: 0, error: null };
|
||||
const DOWN = new Set([429, 502, 503, 504]);
|
||||
/** Seconds or a date in Retry-After, as milliseconds from now; 0 without one. */
|
||||
const retryAfter = (value) => {
|
||||
if (!value) return 0;
|
||||
const ms = /^\d+$/.test(value) ? Number(value) * 1000 : Date.parse(value) - Date.now();
|
||||
return Number.isFinite(ms) ? Math.min(Math.max(ms, 0), 6 * BACKOFF_MAX) : 0;
|
||||
};
|
||||
function sourceDown(error, wait = 0) {
|
||||
outage.failures++;
|
||||
outage.error = error;
|
||||
if (BACKOFF) outage.until = Date.now() + Math.max(Math.min(BACKOFF * 2 ** (outage.failures - 1), BACKOFF_MAX), wait);
|
||||
}
|
||||
/** The pause after failed attempts, for /status: null while the university answers. */
|
||||
export const sourceBackoff = () =>
|
||||
outage.failures
|
||||
? { failures: outage.failures, retryAt: outage.until > Date.now() ? new Date(outage.until).toISOString() : null }
|
||||
: null;
|
||||
|
||||
export async function request(path, method = "GET") {
|
||||
// Only the university share is reachable; listings and catalog entries supply the paths.
|
||||
if (
|
||||
@@ -52,16 +86,33 @@ export async function request(path, method = "GET") {
|
||||
!["GET", "PROPFIND"].includes(method)
|
||||
)
|
||||
throw new Error("Недопустимый путь документа.");
|
||||
if (Date.now() < outage.until) throw sourceError(outage.error);
|
||||
// Node's fetch keeps connections to the university open between requests, so a catalog
|
||||
// crawl of a hundred listings pays for one TLS handshake, not a hundred. Certificates are
|
||||
// checked, redirects are refused, and the answer is cut off past 10 MB or 25 seconds.
|
||||
try {
|
||||
const response = await fetch(ORIGIN + path, {
|
||||
method,
|
||||
headers: method === "PROPFIND" ? { Depth: "1" } : {},
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(TIMEOUT),
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(ORIGIN + path, {
|
||||
method,
|
||||
headers: { "User-Agent": USER_AGENT, ...(method === "PROPFIND" && { Depth: "1" }) },
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(TIMEOUT),
|
||||
});
|
||||
} catch (error) {
|
||||
// No connection at all, or a listing (small and quick when all is well) timed out.
|
||||
if (error?.name !== "TimeoutError" || method === "PROPFIND") sourceDown(error);
|
||||
throw error;
|
||||
}
|
||||
if (DOWN.has(response.status)) {
|
||||
await response.body?.cancel();
|
||||
const error = new Error(`HTTP ${response.status}`);
|
||||
sourceDown(error, retryAfter(response.headers.get("retry-after")));
|
||||
throw error;
|
||||
}
|
||||
// Any other answer, a missing or broken document included: the university is back.
|
||||
outage.failures = 0;
|
||||
outage.until = 0;
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
@@ -478,6 +529,8 @@ export async function sourceStatus(today = todayIso()) {
|
||||
// The last check: how many listings it cost and how many folders came from the cache.
|
||||
lastCrawl: status.lastCrawl,
|
||||
failingSince: status.catalogErrorSince,
|
||||
// While it is down: how many attempts failed and when the next one may reach it.
|
||||
backoff: sourceBackoff(),
|
||||
}
|
||||
: null,
|
||||
calendar: calendar
|
||||
|
||||
Reference in New Issue
Block a user