feat: Implement Telegram alerts for catalog updates and failures
This commit is contained in:
137
server/source.js
137
server/source.js
@@ -16,21 +16,25 @@ const SHARE = "/public.php/dav/files/no5zKbMrns6j5jQ/";
|
||||
const YEAR_FOLDER = /^(\d{4})-(\d{4})$/;
|
||||
const parser = new XMLParser({ removeNSPrefix: true, ignoreAttributes: false });
|
||||
const arr = (value) => (value ? (Array.isArray(value) ? value : [value]) : []);
|
||||
// A check of an unchanged catalog costs two listings (see crawl), so it can run every 5 minutes.
|
||||
const DISCOVERY_TTL = 15 * 60 * 1000,
|
||||
CATALOG_TTL = 15 * 60 * 1000,
|
||||
CATALOG_TTL = 5 * 60 * 1000,
|
||||
CALENDAR_TTL = 6 * 60 * 60 * 1000;
|
||||
// Even with unchanged ETags the whole catalog is listed again this often, in case the university's
|
||||
// server ever fails to pass a change up to the parent folders.
|
||||
const FULL_CRAWL = 6 * 60 * 60 * 1000;
|
||||
// A forced refresh (?refresh=1, the PWA's "Обновить") reaches the university at most this often;
|
||||
// sooner, the copy just read is the answer. Anyone can send the request, and one catalog crawl
|
||||
// sooner, the copy just read is the answer. Anyone can send the request, and a full catalog crawl
|
||||
// is up to 180 listings of the university's Nextcloud.
|
||||
export const MIN_REFRESH = 2 * 60 * 1000;
|
||||
export const MIN_REFRESH = process.env.PSU_MIN_REFRESH_MS ? Number(process.env.PSU_MIN_REFRESH_MS) : 2 * 60 * 1000;
|
||||
const CALENDAR_MIN_REFRESH = 10 * 60 * 1000;
|
||||
/** Milliseconds since an ISO time; Infinity for a missing or broken one. */
|
||||
const ageOf = (iso) => {
|
||||
const age = Date.now() - new Date(iso).getTime();
|
||||
return Number.isFinite(age) ? age : Infinity;
|
||||
};
|
||||
let catalogue, pending, calendar, calendarPending, sources, sourcesPending;
|
||||
const status = { catalogError: null, calendarError: null };
|
||||
let catalogue, pending, calendar, calendarPending, sources, sourcesPending, tree;
|
||||
const status = { catalogError: null, calendarError: null, catalogErrorSince: null, lastCrawl: null };
|
||||
// When the university last failed to answer: background refreshes back off for MIN_REFRESH,
|
||||
// so a down source is not crawled again on every request.
|
||||
const failedAt = { catalog: 0, calendar: 0 };
|
||||
@@ -88,6 +92,8 @@ export function decodeListing(xml, root = SHARE) {
|
||||
folder: props.resourcetype?.collection !== undefined,
|
||||
modified: props.getlastmodified || null,
|
||||
size: Number(props.getcontentlength || 0),
|
||||
// Nextcloud changes a folder's ETag whenever anything below it changes.
|
||||
etag: props.getetag ? String(props.getetag) : null,
|
||||
};
|
||||
})
|
||||
.filter((e) => typeof e.path === "string" && e.path.startsWith(root));
|
||||
@@ -175,10 +181,12 @@ async function discover(refresh = false) {
|
||||
const yearEntries = decodeListing(
|
||||
(await request(yearRoot, "PROPFIND")).toString(),
|
||||
).filter((e) => e.path.toLowerCase() !== yearRoot.toLowerCase());
|
||||
const folder = root.find((e) => nameOf(e.path) === year.name);
|
||||
return (sources = {
|
||||
checkedAt: Date.now(),
|
||||
year,
|
||||
yearRoot: root.find((e) => nameOf(e.path) === year.name)?.path || yearRoot,
|
||||
yearRoot: folder?.path || yearRoot,
|
||||
yearEntries,
|
||||
references: pickReferences(root, yearEntries),
|
||||
});
|
||||
})().finally(() => {
|
||||
@@ -187,43 +195,104 @@ async function discover(refresh = false) {
|
||||
return sourcesPending;
|
||||
}
|
||||
|
||||
async function crawl() {
|
||||
const source = await discover(true);
|
||||
let queue = [source.yearRoot],
|
||||
files = [],
|
||||
count = 0;
|
||||
while (queue.length) {
|
||||
if (count + queue.length > 180)
|
||||
throw apiError(502, "source_invalid", "Каталог университета слишком большой.");
|
||||
const batch = queue.splice(0, 5);
|
||||
count += batch.length;
|
||||
const results = await Promise.all(
|
||||
batch.map(async (path) =>
|
||||
decodeListing((await request(path, "PROPFIND")).toString()).filter(
|
||||
(e) => e.path.toLowerCase() !== path.toLowerCase(),
|
||||
),
|
||||
),
|
||||
);
|
||||
for (const entries of results)
|
||||
for (const entry of entries) {
|
||||
if (entry.folder) queue.push(entry.path);
|
||||
else if (/\.(docx?|xlsx?|pdf)$/i.test(entry.path))
|
||||
files.push(fileMetadata(entry));
|
||||
}
|
||||
const TREE = "catalog-tree.json";
|
||||
async function readTree() {
|
||||
if (tree !== undefined) return;
|
||||
try {
|
||||
tree = JSON.parse(await readFile(DATA + TREE, "utf8"));
|
||||
} catch {
|
||||
tree = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** A short fingerprint of the catalog's documents: it changes only when a file does. */
|
||||
export const catalogVersion = (files, year) =>
|
||||
createHash("sha1")
|
||||
.update([year, ...files.map((f) => `${f.id}:${f.modified}`).sort()].join("|"))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
|
||||
/**
|
||||
* Lists the year folder and everything below it. Folder listings of the last crawl are kept with
|
||||
* each folder's ETag: a folder whose ETag is unchanged is taken from there, subfolders included,
|
||||
* without a request. A check of an unchanged catalog therefore costs only the two listings
|
||||
* discover makes; a changed document costs the folders on the way to it.
|
||||
*/
|
||||
async function crawl() {
|
||||
const started = Date.now();
|
||||
const source = await discover(true);
|
||||
await readTree();
|
||||
const full = !tree || Date.now() - (tree.fullAt || 0) >= FULL_CRAWL;
|
||||
const known = full ? {} : tree.folders || {};
|
||||
const folders = {};
|
||||
const files = [];
|
||||
let queue = [],
|
||||
requested = 0,
|
||||
reused = 0;
|
||||
const visit = (entries) => {
|
||||
for (const entry of entries) {
|
||||
if (entry.folder) queue.push(entry);
|
||||
else if (/\.(docx?|xlsx?|pdf)$/i.test(entry.path)) files.push(fileMetadata(entry));
|
||||
}
|
||||
};
|
||||
// discover has just listed the year folder itself.
|
||||
folders[source.yearRoot] = { entries: source.yearEntries };
|
||||
visit(source.yearEntries);
|
||||
while (queue.length) {
|
||||
const fetch = [];
|
||||
for (const entry of queue.splice(0)) {
|
||||
const cached = known[entry.path];
|
||||
if (cached && entry.etag && cached.etag === entry.etag) {
|
||||
folders[entry.path] = cached;
|
||||
reused++;
|
||||
visit(cached.entries);
|
||||
} else fetch.push(entry);
|
||||
}
|
||||
for (let i = 0; i < fetch.length; i += 5) {
|
||||
if (requested + fetch.length - i > 180)
|
||||
throw apiError(502, "source_invalid", "Каталог университета слишком большой.");
|
||||
const batch = fetch.slice(i, i + 5);
|
||||
requested += batch.length;
|
||||
const listed = await Promise.all(
|
||||
batch.map(async ({ path }) =>
|
||||
decodeListing((await request(path, "PROPFIND")).toString()).filter(
|
||||
(e) => e.path.toLowerCase() !== path.toLowerCase(),
|
||||
),
|
||||
),
|
||||
);
|
||||
batch.forEach((entry, k) => {
|
||||
folders[entry.path] = { etag: entry.etag, entries: listed[k] };
|
||||
visit(listed[k]);
|
||||
});
|
||||
}
|
||||
}
|
||||
const docs = files.filter((f) => /\.docx?$/i.test(f.name) && !isReference(f.name));
|
||||
const result = {
|
||||
// Week tables and bell schedules are reference documents, not timetables.
|
||||
files: files.filter((f) => /\.docx?$/i.test(f.name) && !isReference(f.name)),
|
||||
files: docs,
|
||||
otherFiles: files.filter((f) => !/\.docx?$/i.test(f.name)).length,
|
||||
version: catalogVersion(docs, source.year.label),
|
||||
updatedAt: new Date().toISOString(),
|
||||
stale: false,
|
||||
year: source.year.label,
|
||||
currentYear: source.year.current,
|
||||
expectedYear: source.year.expected.replace("-", "–"),
|
||||
};
|
||||
tree = { fullAt: full ? Date.now() : tree.fullAt, folders };
|
||||
await mkdir(DATA, { recursive: true });
|
||||
await writeFile(DATA + TREE, JSON.stringify(tree));
|
||||
await writeFile(DATA + "catalog.json", JSON.stringify(result));
|
||||
status.catalogError = null;
|
||||
status.catalogErrorSince = null;
|
||||
status.lastCrawl = {
|
||||
at: result.updatedAt,
|
||||
full,
|
||||
// Two listings come from discover: the share's root and the year folder.
|
||||
requests: requested + 2,
|
||||
folders: Object.keys(folders).length,
|
||||
reused,
|
||||
ms: Date.now() - started,
|
||||
};
|
||||
return (catalogue = result);
|
||||
}
|
||||
async function readCatalogCache() {
|
||||
@@ -265,6 +334,7 @@ export async function getCatalog(refresh = false) {
|
||||
.catch((error) => {
|
||||
failedAt.catalog = Date.now();
|
||||
status.catalogError = error.message;
|
||||
status.catalogErrorSince ||= new Date().toISOString();
|
||||
if (catalogue)
|
||||
return {
|
||||
...catalogue,
|
||||
@@ -368,6 +438,10 @@ export async function getCalendar(refresh = false) {
|
||||
return calendarPending;
|
||||
}
|
||||
|
||||
/** Since when the university's catalog has been failing, with the error; null while it answers. */
|
||||
export const catalogFailure = () =>
|
||||
status.catalogError ? { error: status.catalogError, since: status.catalogErrorSince } : null;
|
||||
|
||||
/** What the server currently uses, for /api/health. Never touches the network. */
|
||||
export async function sourceStatus(today = todayIso()) {
|
||||
await Promise.all([readCatalogCache(), readCalendarCache()]);
|
||||
@@ -399,6 +473,9 @@ export async function sourceStatus(today = todayIso()) {
|
||||
updatedAt: catalogue.updatedAt,
|
||||
ageMinutes: age(catalogue.updatedAt),
|
||||
stale: Boolean(catalogue.stale || status.catalogError),
|
||||
// The last check: how many listings it cost and how many folders came from the cache.
|
||||
lastCrawl: status.lastCrawl,
|
||||
failingSince: status.catalogErrorSince,
|
||||
}
|
||||
: null,
|
||||
calendar: calendar
|
||||
|
||||
Reference in New Issue
Block a user