503 lines
20 KiB
JavaScript
503 lines
20 KiB
JavaScript
import { XMLParser } from "fast-xml-parser";
|
||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||
import { createHash } from "node:crypto";
|
||
import { parseWeeksDocx, parseBellsDocx } from "./calendar.js";
|
||
import { apiError, sourceError } from "./errors.js";
|
||
import { DATA } from "./paths.js";
|
||
import {
|
||
academicStartYear,
|
||
assignSemesters,
|
||
todayIso,
|
||
weekIndexOf,
|
||
} from "../shared/weeks.js";
|
||
// The university's Nextcloud; tests point PSU_SOURCE_ORIGIN at a local stand-in.
|
||
const ORIGIN = process.env.PSU_SOURCE_ORIGIN || "https://nextcloud.psu.by";
|
||
const SHARE = "/public.php/dav/files/no5zKbMrns6j5jQ/";
|
||
const YEAR_FOLDER = /^(\d{4})-(\d{4})$/;
|
||
const parser = new XMLParser({ removeNSPrefix: true, ignoreAttributes: false });
|
||
const arr = (value) => (value ? (Array.isArray(value) ? value : [value]) : []);
|
||
// A check of an unchanged catalog costs two listings (see crawl), so it can run every 5 minutes.
|
||
const DISCOVERY_TTL = 15 * 60 * 1000,
|
||
CATALOG_TTL = 5 * 60 * 1000,
|
||
CALENDAR_TTL = 6 * 60 * 60 * 1000;
|
||
// Even with unchanged ETags the whole catalog is listed again this often, in case the university's
|
||
// server ever fails to pass a change up to the parent folders.
|
||
const FULL_CRAWL = 6 * 60 * 60 * 1000;
|
||
// A forced refresh (?refresh=1, the PWA's "Обновить") reaches the university at most this often;
|
||
// sooner, the copy just read is the answer. Anyone can send the request, and a full catalog crawl
|
||
// is up to 180 listings of the university's Nextcloud.
|
||
export const MIN_REFRESH = process.env.PSU_MIN_REFRESH_MS ? Number(process.env.PSU_MIN_REFRESH_MS) : 2 * 60 * 1000;
|
||
const CALENDAR_MIN_REFRESH = 10 * 60 * 1000;
|
||
/** Milliseconds since an ISO time; Infinity for a missing or broken one. */
|
||
const ageOf = (iso) => {
|
||
const age = Date.now() - new Date(iso).getTime();
|
||
return Number.isFinite(age) ? age : Infinity;
|
||
};
|
||
let catalogue, pending, calendar, calendarPending, sources, sourcesPending, tree;
|
||
const status = { catalogError: null, calendarError: null, catalogErrorSince: null, lastCrawl: null };
|
||
// When the university last failed to answer: background refreshes back off for MIN_REFRESH,
|
||
// so a down source is not crawled again on every request.
|
||
const failedAt = { catalog: 0, calendar: 0 };
|
||
const backingOff = (what) => Date.now() - failedAt[what] < MIN_REFRESH;
|
||
|
||
const TIMEOUT = Number(process.env.PSU_SOURCE_TIMEOUT_MS) || 25000;
|
||
const MAX_SIZE = 10 * 1024 * 1024;
|
||
export async function request(path, method = "GET") {
|
||
// Only the university share is reachable; listings and catalog entries supply the paths.
|
||
if (
|
||
!path.startsWith(SHARE) ||
|
||
/[\r\n]/.test(path) ||
|
||
/(^|\/)\.\.(\/|$)/.test(decodeURIComponent(path)) ||
|
||
!["GET", "PROPFIND"].includes(method)
|
||
)
|
||
throw new Error("Недопустимый путь документа.");
|
||
// Node's fetch keeps connections to the university open between requests, so a catalog
|
||
// crawl of a hundred listings pays for one TLS handshake, not a hundred. Certificates are
|
||
// checked, redirects are refused, and the answer is cut off past 10 MB or 25 seconds.
|
||
try {
|
||
const response = await fetch(ORIGIN + path, {
|
||
method,
|
||
headers: method === "PROPFIND" ? { Depth: "1" } : {},
|
||
redirect: "manual",
|
||
signal: AbortSignal.timeout(TIMEOUT),
|
||
});
|
||
if (!response.ok) {
|
||
await response.body?.cancel();
|
||
throw new Error(`HTTP ${response.status}`);
|
||
}
|
||
if (Number(response.headers.get("content-length")) > MAX_SIZE) {
|
||
await response.body?.cancel();
|
||
throw new Error("Документ больше 10 МБ.");
|
||
}
|
||
const chunks = [];
|
||
let size = 0;
|
||
for await (const chunk of response.body || []) {
|
||
size += chunk.length;
|
||
if (size > MAX_SIZE) throw new Error("Документ больше 10 МБ.");
|
||
chunks.push(chunk);
|
||
}
|
||
return Buffer.concat(chunks);
|
||
} catch (error) {
|
||
throw sourceError(error);
|
||
}
|
||
}
|
||
export function decodeListing(xml, root = SHARE) {
|
||
const entries = arr(parser.parse(xml).multistatus?.response);
|
||
return entries
|
||
.map((entry) => {
|
||
const props =
|
||
arr(entry.propstat).find((p) => p.status?.includes("200"))?.prop || {};
|
||
return {
|
||
path: entry.href,
|
||
folder: props.resourcetype?.collection !== undefined,
|
||
modified: props.getlastmodified || null,
|
||
size: Number(props.getcontentlength || 0),
|
||
// Nextcloud changes a folder's ETag whenever anything below it changes.
|
||
etag: props.getetag ? String(props.getetag) : null,
|
||
};
|
||
})
|
||
.filter((e) => typeof e.path === "string" && e.path.startsWith(root));
|
||
}
|
||
const nameOf = (path) =>
|
||
decodeURIComponent(path).replace(/\/$/, "").split("/").at(-1);
|
||
export const isWeeksTable = (name) =>
|
||
/\.docx$/i.test(name) &&
|
||
/бел/i.test(name) &&
|
||
/зел[её]н/i.test(name) &&
|
||
/недел/i.test(name);
|
||
export const isBellSchedule = (name) =>
|
||
/\.docx$/i.test(name) && /звонк/i.test(name);
|
||
const isReference = (name) => isWeeksTable(name) || isBellSchedule(name);
|
||
|
||
export function fileMetadata(entry) {
|
||
const pieces = decodeURIComponent(entry.path).slice(SHARE.length).split("/");
|
||
const year = YEAR_FOLDER.test(pieces[0]) ? pieces.shift() : "";
|
||
const name = pieces.at(-1);
|
||
return {
|
||
...entry,
|
||
id: createHash("sha256").update(entry.path).digest("hex").slice(0, 24),
|
||
name,
|
||
year: year.replace("-", "–"),
|
||
faculty: pieces[0],
|
||
form: pieces[1] || "",
|
||
course: pieces.find((s) => /^\d курс$/.test(s)) || "",
|
||
section: pieces.slice(2, -1).join(" / "),
|
||
url: ORIGIN + entry.path,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* The folder of the current academic year (from 1 September), else the newest
|
||
* earlier one, else the oldest later one. Folders look like "2026-2027".
|
||
*/
|
||
export function pickYearFolder(names, today = todayIso()) {
|
||
const target = academicStartYear(today);
|
||
const years = names
|
||
.map((name) => ({ name, m: name.match(YEAR_FOLDER) }))
|
||
.filter(({ m }) => m && Number(m[2]) === Number(m[1]) + 1)
|
||
.map(({ name, m }) => ({ name, start: Number(m[1]) }))
|
||
.sort((a, b) => b.start - a.start);
|
||
const expected = `${target}-${target + 1}`;
|
||
const pick =
|
||
years.find((y) => y.start === target) ||
|
||
years.find((y) => y.start < target) ||
|
||
years.at(-1);
|
||
return pick
|
||
? { name: pick.name, label: pick.name.replace("-", "–"), current: pick.start === target, expected }
|
||
: null;
|
||
}
|
||
|
||
/** Reference documents: newest first, the year folder before the share root. */
|
||
export function pickReferences(rootEntries, yearEntries = []) {
|
||
const files = (entries, scope) =>
|
||
entries
|
||
.filter((e) => !e.folder)
|
||
.map((e) => ({ ...e, name: nameOf(e.path), scope }));
|
||
const all = [...files(yearEntries, "year"), ...files(rootEntries, "root")];
|
||
const order = (a, b) =>
|
||
(a.scope === b.scope ? 0 : a.scope === "year" ? -1 : 1) ||
|
||
new Date(b.modified || 0) - new Date(a.modified || 0);
|
||
return {
|
||
weeks: all.filter((f) => isWeeksTable(f.name)).sort(order),
|
||
bells: all.filter((f) => isBellSchedule(f.name)).sort(order)[0] || null,
|
||
};
|
||
}
|
||
|
||
/** Lists the share root and the chosen year folder; cached for 15 minutes. */
|
||
async function discover(refresh = false) {
|
||
if (!refresh && sources && Date.now() - sources.checkedAt < DISCOVERY_TTL)
|
||
return sources;
|
||
if (!sourcesPending)
|
||
sourcesPending = (async () => {
|
||
const root = decodeListing(
|
||
(await request(SHARE, "PROPFIND")).toString(),
|
||
).filter((e) => e.path.toLowerCase() !== SHARE.toLowerCase());
|
||
const year = pickYearFolder(
|
||
root.filter((e) => e.folder).map((e) => nameOf(e.path)),
|
||
);
|
||
if (!year)
|
||
throw apiError(502, "source_invalid", "В каталоге университета нет папки учебного года.");
|
||
const yearRoot = SHARE + encodeURIComponent(year.name) + "/";
|
||
const yearEntries = decodeListing(
|
||
(await request(yearRoot, "PROPFIND")).toString(),
|
||
).filter((e) => e.path.toLowerCase() !== yearRoot.toLowerCase());
|
||
const folder = root.find((e) => nameOf(e.path) === year.name);
|
||
return (sources = {
|
||
checkedAt: Date.now(),
|
||
year,
|
||
yearRoot: folder?.path || yearRoot,
|
||
yearEntries,
|
||
references: pickReferences(root, yearEntries),
|
||
});
|
||
})().finally(() => {
|
||
sourcesPending = null;
|
||
});
|
||
return sourcesPending;
|
||
}
|
||
|
||
const TREE = "catalog-tree.json";
|
||
async function readTree() {
|
||
if (tree !== undefined) return;
|
||
try {
|
||
tree = JSON.parse(await readFile(DATA + TREE, "utf8"));
|
||
} catch {
|
||
tree = null;
|
||
}
|
||
}
|
||
|
||
/** A short fingerprint of the catalog's documents: it changes only when a file does. */
|
||
export const catalogVersion = (files, year) =>
|
||
createHash("sha1")
|
||
.update([year, ...files.map((f) => `${f.id}:${f.modified}`).sort()].join("|"))
|
||
.digest("hex")
|
||
.slice(0, 16);
|
||
|
||
/**
|
||
* Lists the year folder and everything below it. Folder listings of the last crawl are kept with
|
||
* each folder's ETag: a folder whose ETag is unchanged is taken from there, subfolders included,
|
||
* without a request. A check of an unchanged catalog therefore costs only the two listings
|
||
* discover makes; a changed document costs the folders on the way to it.
|
||
*/
|
||
async function crawl() {
|
||
const started = Date.now();
|
||
const source = await discover(true);
|
||
await readTree();
|
||
const full = !tree || Date.now() - (tree.fullAt || 0) >= FULL_CRAWL;
|
||
const known = full ? {} : tree.folders || {};
|
||
const folders = {};
|
||
const files = [];
|
||
let queue = [],
|
||
requested = 0,
|
||
reused = 0;
|
||
const visit = (entries) => {
|
||
for (const entry of entries) {
|
||
if (entry.folder) queue.push(entry);
|
||
else if (/\.(docx?|xlsx?|pdf)$/i.test(entry.path)) files.push(fileMetadata(entry));
|
||
}
|
||
};
|
||
// discover has just listed the year folder itself.
|
||
folders[source.yearRoot] = { entries: source.yearEntries };
|
||
visit(source.yearEntries);
|
||
while (queue.length) {
|
||
const fetch = [];
|
||
for (const entry of queue.splice(0)) {
|
||
const cached = known[entry.path];
|
||
if (cached && entry.etag && cached.etag === entry.etag) {
|
||
folders[entry.path] = cached;
|
||
reused++;
|
||
visit(cached.entries);
|
||
} else fetch.push(entry);
|
||
}
|
||
for (let i = 0; i < fetch.length; i += 5) {
|
||
if (requested + fetch.length - i > 180)
|
||
throw apiError(502, "source_invalid", "Каталог университета слишком большой.");
|
||
const batch = fetch.slice(i, i + 5);
|
||
requested += batch.length;
|
||
const listed = await Promise.all(
|
||
batch.map(async ({ path }) =>
|
||
decodeListing((await request(path, "PROPFIND")).toString()).filter(
|
||
(e) => e.path.toLowerCase() !== path.toLowerCase(),
|
||
),
|
||
),
|
||
);
|
||
batch.forEach((entry, k) => {
|
||
folders[entry.path] = { etag: entry.etag, entries: listed[k] };
|
||
visit(listed[k]);
|
||
});
|
||
}
|
||
}
|
||
const docs = files.filter((f) => /\.docx?$/i.test(f.name) && !isReference(f.name));
|
||
const result = {
|
||
// Week tables and bell schedules are reference documents, not timetables.
|
||
files: docs,
|
||
otherFiles: files.filter((f) => !/\.docx?$/i.test(f.name)).length,
|
||
version: catalogVersion(docs, source.year.label),
|
||
updatedAt: new Date().toISOString(),
|
||
stale: false,
|
||
year: source.year.label,
|
||
currentYear: source.year.current,
|
||
expectedYear: source.year.expected.replace("-", "–"),
|
||
};
|
||
tree = { fullAt: full ? Date.now() : tree.fullAt, folders };
|
||
await mkdir(DATA, { recursive: true });
|
||
await writeFile(DATA + TREE, JSON.stringify(tree));
|
||
await writeFile(DATA + "catalog.json", JSON.stringify(result));
|
||
status.catalogError = null;
|
||
status.catalogErrorSince = null;
|
||
status.lastCrawl = {
|
||
at: result.updatedAt,
|
||
full,
|
||
// Two listings come from discover: the share's root and the year folder.
|
||
requests: requested + 2,
|
||
folders: Object.keys(folders).length,
|
||
reused,
|
||
ms: Date.now() - started,
|
||
};
|
||
return (catalogue = result);
|
||
}
|
||
async function readCatalogCache() {
|
||
if (catalogue) return;
|
||
try {
|
||
catalogue = JSON.parse(await readFile(DATA + "catalog.json", "utf8"));
|
||
} catch {
|
||
// An initial source snapshot can be used during development after an interrupted sync.
|
||
try {
|
||
const files = JSON.parse(
|
||
await readFile(DATA + "source-catalog.json", "utf8"),
|
||
);
|
||
catalogue = {
|
||
files: files
|
||
.filter((f) => /\.docx?$/i.test(f.name) && !isReference(f.name))
|
||
.map(fileMetadata),
|
||
updatedAt: null,
|
||
stale: true,
|
||
year: files[0] ? fileMetadata(files[0]).year : "",
|
||
};
|
||
} catch {}
|
||
}
|
||
}
|
||
export async function getCatalog(refresh = false) {
|
||
await readCatalogCache();
|
||
const expected = (() => {
|
||
const y = academicStartYear(todayIso());
|
||
return `${y}–${y + 1}`;
|
||
})();
|
||
const age = catalogue ? ageOf(catalogue.updatedAt) : Infinity;
|
||
// A cached catalog of last year is refreshed as soon as the new year begins.
|
||
const thisYear =
|
||
catalogue && (catalogue.year === expected || catalogue.currentYear === false);
|
||
const forced = refresh && age >= MIN_REFRESH;
|
||
if (thisYear && age < CATALOG_TTL && !forced) return catalogue;
|
||
if (thisYear && !forced && backingOff("catalog")) return catalogue;
|
||
if (!pending)
|
||
pending = crawl()
|
||
.catch((error) => {
|
||
failedAt.catalog = Date.now();
|
||
status.catalogError = error.message;
|
||
status.catalogErrorSince ||= new Date().toISOString();
|
||
if (catalogue)
|
||
return {
|
||
...catalogue,
|
||
stale: true,
|
||
error: "Источник недоступен. Показана сохранённая копия каталога.",
|
||
};
|
||
throw error;
|
||
})
|
||
.finally(() => {
|
||
pending = null;
|
||
});
|
||
// Stale-while-revalidate: an outdated copy of this year answers at once and the crawl runs
|
||
// behind it. Only a first start, a new academic year or a forced refresh wait for the crawl.
|
||
if (thisYear && !forced) return catalogue;
|
||
return pending;
|
||
}
|
||
export async function documentFor(id) {
|
||
const catalog = await getCatalog();
|
||
const file = catalog.files.find((f) => f.id === id);
|
||
if (!file) {
|
||
throw apiError(404, "document_not_found", "Документ не найден в каталоге. Обновите список.");
|
||
}
|
||
return file;
|
||
}
|
||
|
||
/** Merges every week table found (newest wins on overlap) and splits it into semesters. */
|
||
export function mergeWeekTables(tables) {
|
||
const byStart = new Map();
|
||
for (const weeks of tables)
|
||
for (const week of weeks)
|
||
if (!byStart.has(week.start)) byStart.set(week.start, week);
|
||
return assignSemesters([...byStart.values()]);
|
||
}
|
||
async function loadCalendar() {
|
||
const { references } = await discover(true);
|
||
if (!references.weeks.length)
|
||
throw apiError(502, "source_invalid", "Таблица белых и зелёных недель не найдена в каталоге.");
|
||
const tables = [];
|
||
for (const file of references.weeks)
|
||
try {
|
||
tables.push(parseWeeksDocx(await request(file.path)));
|
||
} catch (error) {
|
||
// One damaged copy must not hide a readable newer or older one.
|
||
console.error(`${file.name}: ${error.message}`);
|
||
}
|
||
if (!tables.length) throw apiError(502, "source_invalid", "Таблица недель не распознана.");
|
||
let bells = [];
|
||
if (references.bells)
|
||
try {
|
||
bells = parseBellsDocx(await request(references.bells.path));
|
||
} catch (error) {
|
||
console.error(`${references.bells.name}: ${error.message}`);
|
||
}
|
||
const { weeks, semesters } = mergeWeekTables(tables);
|
||
const result = {
|
||
weeks,
|
||
semesters,
|
||
// Keep the last known bells if the bell schedule became unreadable.
|
||
bells: bells.length ? bells : calendar?.bells || [],
|
||
source: {
|
||
weeks: references.weeks[0].modified,
|
||
bells: references.bells?.modified || null,
|
||
weeksFiles: references.weeks.map((f) => f.name),
|
||
bellsFile: references.bells?.name || null,
|
||
},
|
||
updatedAt: new Date().toISOString(),
|
||
stale: false,
|
||
};
|
||
await mkdir(DATA, { recursive: true });
|
||
await writeFile(DATA + "calendar.json", JSON.stringify(result));
|
||
status.calendarError = null;
|
||
return (calendar = result);
|
||
}
|
||
async function readCalendarCache() {
|
||
if (calendar) return;
|
||
try {
|
||
calendar = JSON.parse(await readFile(DATA + "calendar.json", "utf8"));
|
||
// Caches written before semesters existed are upgraded in place.
|
||
if (!calendar.semesters) calendar = { ...calendar, ...assignSemesters(calendar.weeks) };
|
||
} catch {}
|
||
}
|
||
export async function getCalendar(refresh = false) {
|
||
await readCalendarCache();
|
||
const age = calendar ? ageOf(calendar.updatedAt) : Infinity;
|
||
const forced = refresh && age >= CALENDAR_MIN_REFRESH;
|
||
if (calendar && age < CALENDAR_TTL && !forced) return calendar;
|
||
if (calendar && !forced && backingOff("calendar")) return calendar;
|
||
if (!calendarPending)
|
||
calendarPending = loadCalendar()
|
||
.catch((error) => {
|
||
failedAt.calendar = Date.now();
|
||
status.calendarError = error.message;
|
||
if (calendar) return { ...calendar, stale: true, error: error.message };
|
||
throw error;
|
||
})
|
||
.finally(() => {
|
||
calendarPending = null;
|
||
});
|
||
// As with the catalog, a saved week table answers while the new one loads.
|
||
if (calendar && !forced) return calendar;
|
||
return calendarPending;
|
||
}
|
||
|
||
/** Since when the university's catalog has been failing, with the error; null while it answers. */
|
||
export const catalogFailure = () =>
|
||
status.catalogError ? { error: status.catalogError, since: status.catalogErrorSince } : null;
|
||
|
||
/** What the server currently uses, for /api/health. Never touches the network. */
|
||
export async function sourceStatus(today = todayIso()) {
|
||
await Promise.all([readCatalogCache(), readCalendarCache()]);
|
||
const warnings = [];
|
||
const expected = academicStartYear(today);
|
||
const expectedYear = `${expected}–${expected + 1}`;
|
||
if (catalogue && catalogue.year !== expectedYear)
|
||
warnings.push(
|
||
`Папка учебного года ${expectedYear} не найдена или каталог ещё не обновлён; используется ${catalogue.year || "неизвестный год"}.`,
|
||
);
|
||
if (status.catalogError)
|
||
warnings.push(`Каталог не обновился: ${status.catalogError}`);
|
||
if (!calendar) warnings.push("Таблица недель ещё не загружена.");
|
||
else if (weekIndexOf(calendar, today) < 0)
|
||
warnings.push(
|
||
"Сегодняшняя дата вне таблицы недель: приложение показывает все строки Word.",
|
||
);
|
||
if (status.calendarError)
|
||
warnings.push(`Таблица недель не обновилась: ${status.calendarError}`);
|
||
const age = (iso) =>
|
||
iso ? Math.round((Date.now() - new Date(iso).getTime()) / 60000) : null;
|
||
return {
|
||
today,
|
||
year: sources?.year || (catalogue ? { label: catalogue.year } : null),
|
||
catalog: catalogue
|
||
? {
|
||
year: catalogue.year,
|
||
files: catalogue.files.length,
|
||
updatedAt: catalogue.updatedAt,
|
||
ageMinutes: age(catalogue.updatedAt),
|
||
stale: Boolean(catalogue.stale || status.catalogError),
|
||
// The last check: how many listings it cost and how many folders came from the cache.
|
||
lastCrawl: status.lastCrawl,
|
||
failingSince: status.catalogErrorSince,
|
||
}
|
||
: null,
|
||
calendar: calendar
|
||
? {
|
||
weeksFiles: calendar.source.weeksFiles || [],
|
||
weeksModified: calendar.source.weeks,
|
||
bellsFile: calendar.source.bellsFile || null,
|
||
semesters: (calendar.semesters || []).map((s) => ({
|
||
id: s.id,
|
||
label: s.label,
|
||
start: s.start,
|
||
end: s.end,
|
||
weeks: calendar.weeks.filter((w) => w.semester === s.id).length,
|
||
})),
|
||
bells: calendar.bells.length,
|
||
updatedAt: calendar.updatedAt,
|
||
ageMinutes: age(calendar.updatedAt),
|
||
coversToday: weekIndexOf(calendar, today) >= 0,
|
||
stale: Boolean(calendar.stale || status.calendarError),
|
||
}
|
||
: null,
|
||
warnings,
|
||
};
|
||
}
|