feat: enhance user-agent identification and implement backoff strategy for university source requests

- Added PSU_CONTACT environment variable to docker-compose for owner contact in User-Agent.
- Updated source.js to include user-agent construction with service version and contact information.
- Implemented exponential backoff strategy for failed requests to the university, with retry logic based on HTTP response status.
- Modified status.js to display backoff information in the status response.
- Adjusted watch.js to reduce request frequency during university downtime.
- Enhanced test coverage with new backoff.test.js to validate backoff behavior and user-agent format.
This commit is contained in:
2026-10-22 01:17:54 +03:00
parent 10c85e3bb9
commit c0487e4c2a
9 changed files with 82503 additions and 81477 deletions

View File

@@ -1,5 +1,6 @@
import { XMLParser } from "fast-xml-parser";
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { readFileSync } from "node:fs";
import { createHash } from "node:crypto";
import { parseWeeksDocx, parseBellsDocx } from "./calendar.js";
import { apiError, sourceError } from "./errors.js";
@@ -43,6 +44,39 @@ const backingOff = (what) => Date.now() - failedAt[what] < MIN_REFRESH;
const TIMEOUT = Number(process.env.PSU_SOURCE_TIMEOUT_MS) || 25000;
const MAX_SIZE = 10 * 1024 * 1024;
// Every request says who asks and how to reach the owner, so the university's administrators
// see a known reader rather than an anonymous robot: "PSU-Schedule/0.1.0 (+https://…; mailto:…)".
const VERSION = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
const contact = [process.env.PSU_PUBLIC_URL && `+${process.env.PSU_PUBLIC_URL}`, process.env.PSU_CONTACT].filter(Boolean);
export const USER_AGENT = `PSU-Schedule/${VERSION}${contact.length ? ` (${contact.join("; ")})` : ""}`;
// While the university is down, nothing reaches it until the pause is over: the watcher, a
// forced refresh and a document opened by a student all get the saved copy or an error at
// once. The pause doubles with every failed attempt (5, 10, 20, 40 minutes…) up to an hour and
// ends with the first answer. A failing document alone (HTTP 404, 500) or a slow download does
// not pause the rest; the server's whole answering does: no connection, 502–504, 429.
const BACKOFF = process.env.PSU_SOURCE_BACKOFF_MS ? Number(process.env.PSU_SOURCE_BACKOFF_MS) : 5 * 60 * 1000;
const BACKOFF_MAX = Number(process.env.PSU_SOURCE_BACKOFF_MAX_MS) || 60 * 60 * 1000;
const outage = { failures: 0, until: 0, error: null };
const DOWN = new Set([429, 502, 503, 504]);
/** Seconds or a date in Retry-After, as milliseconds from now; 0 without one. */
const retryAfter = (value) => {
if (!value) return 0;
const ms = /^\d+$/.test(value) ? Number(value) * 1000 : Date.parse(value) - Date.now();
return Number.isFinite(ms) ? Math.min(Math.max(ms, 0), 6 * BACKOFF_MAX) : 0;
};
function sourceDown(error, wait = 0) {
outage.failures++;
outage.error = error;
if (BACKOFF) outage.until = Date.now() + Math.max(Math.min(BACKOFF * 2 ** (outage.failures - 1), BACKOFF_MAX), wait);
}
/** The pause after failed attempts, for /status: null while the university answers. */
export const sourceBackoff = () =>
outage.failures
? { failures: outage.failures, retryAt: outage.until > Date.now() ? new Date(outage.until).toISOString() : null }
: null;
export async function request(path, method = "GET") {
// Only the university share is reachable; listings and catalog entries supply the paths.
if (
@@ -52,16 +86,33 @@ export async function request(path, method = "GET") {
!["GET", "PROPFIND"].includes(method)
)
throw new Error("Недопустимый путь документа.");
if (Date.now() < outage.until) throw sourceError(outage.error);
// Node's fetch keeps connections to the university open between requests, so a catalog
// crawl of a hundred listings pays for one TLS handshake, not a hundred. Certificates are
// checked, redirects are refused, and the answer is cut off past 10 MB or 25 seconds.
try {
const response = await fetch(ORIGIN + path, {
method,
headers: method === "PROPFIND" ? { Depth: "1" } : {},
redirect: "manual",
signal: AbortSignal.timeout(TIMEOUT),
});
let response;
try {
response = await fetch(ORIGIN + path, {
method,
headers: { "User-Agent": USER_AGENT, ...(method === "PROPFIND" && { Depth: "1" }) },
redirect: "manual",
signal: AbortSignal.timeout(TIMEOUT),
});
} catch (error) {
// No connection at all, or a listing (small and quick when all is well) timed out.
if (error?.name !== "TimeoutError" || method === "PROPFIND") sourceDown(error);
throw error;
}
if (DOWN.has(response.status)) {
await response.body?.cancel();
const error = new Error(`HTTP ${response.status}`);
sourceDown(error, retryAfter(response.headers.get("retry-after")));
throw error;
}
// Any other answer, a missing or broken document included: the university is back.
outage.failures = 0;
outage.until = 0;
if (!response.ok) {
await response.body?.cancel();
throw new Error(`HTTP ${response.status}`);
@@ -478,6 +529,8 @@ export async function sourceStatus(today = todayIso()) {
// The last check: how many listings it cost and how many folders came from the cache.
lastCrawl: status.lastCrawl,
failingSince: status.catalogErrorSince,
// While it is down: how many attempts failed and when the next one may reach it.
backoff: sourceBackoff(),
}
: null,
calendar: calendar