initial commit
This commit is contained in:
400
server/source.js
Normal file
400
server/source.js
Normal file
@@ -0,0 +1,400 @@
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { parseWeeksDocx, parseBellsDocx } from "./calendar.js";
|
||||
import {
|
||||
academicStartYear,
|
||||
assignSemesters,
|
||||
todayIso,
|
||||
weekIndexOf,
|
||||
} from "../shared/weeks.js";
|
||||
const run = promisify(execFile);
|
||||
const ORIGIN = "https://nextcloud.psu.by";
|
||||
const SHARE = "/public.php/dav/files/no5zKbMrns6j5jQ/";
|
||||
const YEAR_FOLDER = /^(\d{4})-(\d{4})$/;
|
||||
const DATA = fileURLToPath(new URL("../.data/", import.meta.url));
|
||||
const parser = new XMLParser({ removeNSPrefix: true, ignoreAttributes: false });
|
||||
const arr = (value) => (value ? (Array.isArray(value) ? value : [value]) : []);
|
||||
const DISCOVERY_TTL = 15 * 60 * 1000,
|
||||
CATALOG_TTL = 15 * 60 * 1000,
|
||||
CALENDAR_TTL = 6 * 60 * 60 * 1000;
|
||||
let catalogue, pending, calendar, calendarPending, sources, sourcesPending;
|
||||
const status = { catalogError: null, calendarError: null };
|
||||
|
||||
export async function request(path, method = "GET") {
|
||||
// Only the university share is reachable; listings and catalog entries supply the paths.
|
||||
if (
|
||||
!path.startsWith(SHARE) ||
|
||||
/[\r\n]/.test(path) ||
|
||||
/(^|\/)\.\.(\/|$)/.test(decodeURIComponent(path)) ||
|
||||
!["GET", "PROPFIND"].includes(method)
|
||||
)
|
||||
throw new Error("Недопустимый путь документа.");
|
||||
const url = ORIGIN + path;
|
||||
// curl also works with the system trust store on macOS. Certificate checks stay enabled.
|
||||
const args = [
|
||||
"--fail",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--max-time",
|
||||
"25",
|
||||
"--max-filesize",
|
||||
"10485760",
|
||||
"--max-redirs",
|
||||
"0",
|
||||
"--request",
|
||||
method,
|
||||
];
|
||||
if (method === "PROPFIND") args.push("--header", "Depth: 1");
|
||||
args.push(url);
|
||||
const { stdout } = await run("curl", args, {
|
||||
encoding: "buffer",
|
||||
maxBuffer: 11 * 1024 * 1024,
|
||||
timeout: 28000,
|
||||
});
|
||||
return stdout;
|
||||
}
|
||||
export function decodeListing(xml, root = SHARE) {
|
||||
const entries = arr(parser.parse(xml).multistatus?.response);
|
||||
return entries
|
||||
.map((entry) => {
|
||||
const props =
|
||||
arr(entry.propstat).find((p) => p.status?.includes("200"))?.prop || {};
|
||||
return {
|
||||
path: entry.href,
|
||||
folder: props.resourcetype?.collection !== undefined,
|
||||
modified: props.getlastmodified || null,
|
||||
size: Number(props.getcontentlength || 0),
|
||||
};
|
||||
})
|
||||
.filter((e) => typeof e.path === "string" && e.path.startsWith(root));
|
||||
}
|
||||
const nameOf = (path) =>
|
||||
decodeURIComponent(path).replace(/\/$/, "").split("/").at(-1);
|
||||
export const isWeeksTable = (name) =>
|
||||
/\.docx$/i.test(name) &&
|
||||
/бел/i.test(name) &&
|
||||
/зел[её]н/i.test(name) &&
|
||||
/недел/i.test(name);
|
||||
export const isBellSchedule = (name) =>
|
||||
/\.docx$/i.test(name) && /звонк/i.test(name);
|
||||
const isReference = (name) => isWeeksTable(name) || isBellSchedule(name);
|
||||
|
||||
export function fileMetadata(entry) {
|
||||
const pieces = decodeURIComponent(entry.path).slice(SHARE.length).split("/");
|
||||
const year = YEAR_FOLDER.test(pieces[0]) ? pieces.shift() : "";
|
||||
const name = pieces.at(-1);
|
||||
return {
|
||||
...entry,
|
||||
id: createHash("sha256").update(entry.path).digest("hex").slice(0, 24),
|
||||
name,
|
||||
year: year.replace("-", "–"),
|
||||
faculty: pieces[0],
|
||||
form: pieces[1] || "",
|
||||
course: pieces.find((s) => /^\d курс$/.test(s)) || "",
|
||||
section: pieces.slice(2, -1).join(" / "),
|
||||
url: ORIGIN + entry.path,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The folder of the current academic year (from 1 September), else the newest
|
||||
* earlier one, else the oldest later one. Folders look like "2026-2027".
|
||||
*/
|
||||
export function pickYearFolder(names, today = todayIso()) {
|
||||
const target = academicStartYear(today);
|
||||
const years = names
|
||||
.map((name) => ({ name, m: name.match(YEAR_FOLDER) }))
|
||||
.filter(({ m }) => m && Number(m[2]) === Number(m[1]) + 1)
|
||||
.map(({ name, m }) => ({ name, start: Number(m[1]) }))
|
||||
.sort((a, b) => b.start - a.start);
|
||||
const expected = `${target}-${target + 1}`;
|
||||
const pick =
|
||||
years.find((y) => y.start === target) ||
|
||||
years.find((y) => y.start < target) ||
|
||||
years.at(-1);
|
||||
return pick
|
||||
? { name: pick.name, label: pick.name.replace("-", "–"), current: pick.start === target, expected }
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Reference documents: newest first, the year folder before the share root. */
|
||||
export function pickReferences(rootEntries, yearEntries = []) {
|
||||
const files = (entries, scope) =>
|
||||
entries
|
||||
.filter((e) => !e.folder)
|
||||
.map((e) => ({ ...e, name: nameOf(e.path), scope }));
|
||||
const all = [...files(yearEntries, "year"), ...files(rootEntries, "root")];
|
||||
const order = (a, b) =>
|
||||
(a.scope === b.scope ? 0 : a.scope === "year" ? -1 : 1) ||
|
||||
new Date(b.modified || 0) - new Date(a.modified || 0);
|
||||
return {
|
||||
weeks: all.filter((f) => isWeeksTable(f.name)).sort(order),
|
||||
bells: all.filter((f) => isBellSchedule(f.name)).sort(order)[0] || null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Lists the share root and the chosen year folder; cached for 15 minutes. */
|
||||
async function discover(refresh = false) {
|
||||
if (!refresh && sources && Date.now() - sources.checkedAt < DISCOVERY_TTL)
|
||||
return sources;
|
||||
if (!sourcesPending)
|
||||
sourcesPending = (async () => {
|
||||
const root = decodeListing(
|
||||
(await request(SHARE, "PROPFIND")).toString(),
|
||||
).filter((e) => e.path.toLowerCase() !== SHARE.toLowerCase());
|
||||
const year = pickYearFolder(
|
||||
root.filter((e) => e.folder).map((e) => nameOf(e.path)),
|
||||
);
|
||||
if (!year)
|
||||
throw new Error("В каталоге университета нет папки учебного года.");
|
||||
const yearRoot = SHARE + encodeURIComponent(year.name) + "/";
|
||||
const yearEntries = decodeListing(
|
||||
(await request(yearRoot, "PROPFIND")).toString(),
|
||||
).filter((e) => e.path.toLowerCase() !== yearRoot.toLowerCase());
|
||||
return (sources = {
|
||||
checkedAt: Date.now(),
|
||||
year,
|
||||
yearRoot: root.find((e) => nameOf(e.path) === year.name)?.path || yearRoot,
|
||||
references: pickReferences(root, yearEntries),
|
||||
});
|
||||
})().finally(() => {
|
||||
sourcesPending = null;
|
||||
});
|
||||
return sourcesPending;
|
||||
}
|
||||
|
||||
async function crawl() {
|
||||
const source = await discover(true);
|
||||
let queue = [source.yearRoot],
|
||||
files = [],
|
||||
count = 0;
|
||||
while (queue.length) {
|
||||
if (count + queue.length > 180) throw new Error("Каталог слишком большой.");
|
||||
const batch = queue.splice(0, 5);
|
||||
count += batch.length;
|
||||
const results = await Promise.all(
|
||||
batch.map(async (path) =>
|
||||
decodeListing((await request(path, "PROPFIND")).toString()).filter(
|
||||
(e) => e.path.toLowerCase() !== path.toLowerCase(),
|
||||
),
|
||||
),
|
||||
);
|
||||
for (const entries of results)
|
||||
for (const entry of entries) {
|
||||
if (entry.folder) queue.push(entry.path);
|
||||
else if (/\.(docx?|xlsx?|pdf)$/i.test(entry.path))
|
||||
files.push(fileMetadata(entry));
|
||||
}
|
||||
}
|
||||
const result = {
|
||||
// Week tables and bell schedules are reference documents, not timetables.
|
||||
files: files.filter((f) => /\.docx?$/i.test(f.name) && !isReference(f.name)),
|
||||
otherFiles: files.filter((f) => !/\.docx?$/i.test(f.name)).length,
|
||||
updatedAt: new Date().toISOString(),
|
||||
stale: false,
|
||||
year: source.year.label,
|
||||
currentYear: source.year.current,
|
||||
expectedYear: source.year.expected.replace("-", "–"),
|
||||
};
|
||||
await mkdir(DATA, { recursive: true });
|
||||
await writeFile(DATA + "catalog.json", JSON.stringify(result));
|
||||
status.catalogError = null;
|
||||
return (catalogue = result);
|
||||
}
|
||||
async function readCatalogCache() {
|
||||
if (catalogue) return;
|
||||
try {
|
||||
catalogue = JSON.parse(await readFile(DATA + "catalog.json", "utf8"));
|
||||
} catch {
|
||||
// An initial source snapshot can be used during development after an interrupted sync.
|
||||
try {
|
||||
const files = JSON.parse(
|
||||
await readFile(DATA + "source-catalog.json", "utf8"),
|
||||
);
|
||||
catalogue = {
|
||||
files: files
|
||||
.filter((f) => /\.docx?$/i.test(f.name) && !isReference(f.name))
|
||||
.map(fileMetadata),
|
||||
updatedAt: null,
|
||||
stale: true,
|
||||
year: files[0] ? fileMetadata(files[0]).year : "",
|
||||
};
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
export async function getCatalog(refresh = false) {
|
||||
await readCatalogCache();
|
||||
const expected = (() => {
|
||||
const y = academicStartYear(todayIso());
|
||||
return `${y}–${y + 1}`;
|
||||
})();
|
||||
// A cached catalog of last year is refreshed as soon as the new year begins.
|
||||
const fresh =
|
||||
catalogue &&
|
||||
Date.now() - new Date(catalogue.updatedAt).getTime() < CATALOG_TTL &&
|
||||
(catalogue.year === expected || catalogue.currentYear === false);
|
||||
if (!refresh && fresh) return catalogue;
|
||||
if (!pending)
|
||||
pending = crawl()
|
||||
.catch((error) => {
|
||||
status.catalogError = error.message;
|
||||
if (catalogue)
|
||||
return {
|
||||
...catalogue,
|
||||
stale: true,
|
||||
error: "Источник недоступен. Показана сохранённая копия каталога.",
|
||||
};
|
||||
throw error;
|
||||
})
|
||||
.finally(() => {
|
||||
pending = null;
|
||||
});
|
||||
return pending;
|
||||
}
|
||||
export async function documentFor(id) {
|
||||
const catalog = await getCatalog();
|
||||
const file = catalog.files.find((f) => f.id === id);
|
||||
if (!file) {
|
||||
const error = new Error("Документ не найден в каталоге. Обновите список.");
|
||||
error.status = 404;
|
||||
throw error;
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
/** Merges every week table found (newest wins on overlap) and splits it into semesters. */
|
||||
export function mergeWeekTables(tables) {
|
||||
const byStart = new Map();
|
||||
for (const weeks of tables)
|
||||
for (const week of weeks)
|
||||
if (!byStart.has(week.start)) byStart.set(week.start, week);
|
||||
return assignSemesters([...byStart.values()]);
|
||||
}
|
||||
async function loadCalendar() {
|
||||
const { references } = await discover(true);
|
||||
if (!references.weeks.length)
|
||||
throw new Error("Таблица белых и зелёных недель не найдена в каталоге.");
|
||||
const tables = [];
|
||||
for (const file of references.weeks)
|
||||
try {
|
||||
tables.push(parseWeeksDocx(await request(file.path)));
|
||||
} catch (error) {
|
||||
// One damaged copy must not hide a readable newer or older one.
|
||||
console.error(`${file.name}: ${error.message}`);
|
||||
}
|
||||
if (!tables.length) throw new Error("Таблица недель не распознана.");
|
||||
let bells = [];
|
||||
if (references.bells)
|
||||
try {
|
||||
bells = parseBellsDocx(await request(references.bells.path));
|
||||
} catch (error) {
|
||||
console.error(`${references.bells.name}: ${error.message}`);
|
||||
}
|
||||
const { weeks, semesters } = mergeWeekTables(tables);
|
||||
const result = {
|
||||
weeks,
|
||||
semesters,
|
||||
// Keep the last known bells if the bell schedule became unreadable.
|
||||
bells: bells.length ? bells : calendar?.bells || [],
|
||||
source: {
|
||||
weeks: references.weeks[0].modified,
|
||||
bells: references.bells?.modified || null,
|
||||
weeksFiles: references.weeks.map((f) => f.name),
|
||||
bellsFile: references.bells?.name || null,
|
||||
},
|
||||
updatedAt: new Date().toISOString(),
|
||||
stale: false,
|
||||
};
|
||||
await mkdir(DATA, { recursive: true });
|
||||
await writeFile(DATA + "calendar.json", JSON.stringify(result));
|
||||
status.calendarError = null;
|
||||
return (calendar = result);
|
||||
}
|
||||
async function readCalendarCache() {
|
||||
if (calendar) return;
|
||||
try {
|
||||
calendar = JSON.parse(await readFile(DATA + "calendar.json", "utf8"));
|
||||
// Caches written before semesters existed are upgraded in place.
|
||||
if (!calendar.semesters) calendar = { ...calendar, ...assignSemesters(calendar.weeks) };
|
||||
} catch {}
|
||||
}
|
||||
export async function getCalendar(refresh = false) {
|
||||
await readCalendarCache();
|
||||
if (
|
||||
!refresh &&
|
||||
calendar &&
|
||||
Date.now() - new Date(calendar.updatedAt).getTime() < CALENDAR_TTL
|
||||
)
|
||||
return calendar;
|
||||
if (!calendarPending)
|
||||
calendarPending = loadCalendar()
|
||||
.catch((error) => {
|
||||
status.calendarError = error.message;
|
||||
if (calendar) return { ...calendar, stale: true, error: error.message };
|
||||
throw error;
|
||||
})
|
||||
.finally(() => {
|
||||
calendarPending = null;
|
||||
});
|
||||
return calendarPending;
|
||||
}
|
||||
|
||||
/** What the server currently uses, for /api/health. Never touches the network. */
|
||||
export async function sourceStatus(today = todayIso()) {
|
||||
await Promise.all([readCatalogCache(), readCalendarCache()]);
|
||||
const warnings = [];
|
||||
const expected = academicStartYear(today);
|
||||
const expectedYear = `${expected}–${expected + 1}`;
|
||||
if (catalogue && catalogue.year !== expectedYear)
|
||||
warnings.push(
|
||||
`Папка учебного года ${expectedYear} не найдена или каталог ещё не обновлён; используется ${catalogue.year || "неизвестный год"}.`,
|
||||
);
|
||||
if (status.catalogError)
|
||||
warnings.push(`Каталог не обновился: ${status.catalogError}`);
|
||||
if (!calendar) warnings.push("Таблица недель ещё не загружена.");
|
||||
else if (weekIndexOf(calendar, today) < 0)
|
||||
warnings.push(
|
||||
"Сегодняшняя дата вне таблицы недель: приложение показывает все строки Word.",
|
||||
);
|
||||
if (status.calendarError)
|
||||
warnings.push(`Таблица недель не обновилась: ${status.calendarError}`);
|
||||
const age = (iso) =>
|
||||
iso ? Math.round((Date.now() - new Date(iso).getTime()) / 60000) : null;
|
||||
return {
|
||||
today,
|
||||
year: sources?.year || (catalogue ? { label: catalogue.year } : null),
|
||||
catalog: catalogue
|
||||
? {
|
||||
year: catalogue.year,
|
||||
files: catalogue.files.length,
|
||||
updatedAt: catalogue.updatedAt,
|
||||
ageMinutes: age(catalogue.updatedAt),
|
||||
stale: Boolean(catalogue.stale || status.catalogError),
|
||||
}
|
||||
: null,
|
||||
calendar: calendar
|
||||
? {
|
||||
weeksFiles: calendar.source.weeksFiles || [],
|
||||
weeksModified: calendar.source.weeks,
|
||||
bellsFile: calendar.source.bellsFile || null,
|
||||
semesters: (calendar.semesters || []).map((s) => ({
|
||||
id: s.id,
|
||||
label: s.label,
|
||||
start: s.start,
|
||||
end: s.end,
|
||||
weeks: calendar.weeks.filter((w) => w.semester === s.id).length,
|
||||
})),
|
||||
bells: calendar.bells.length,
|
||||
updatedAt: calendar.updatedAt,
|
||||
ageMinutes: age(calendar.updatedAt),
|
||||
coversToday: weekIndexOf(calendar, today) >= 0,
|
||||
stale: Boolean(calendar.stale || status.calendarError),
|
||||
}
|
||||
: null,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user