import { readFile, writeFile, mkdir } from "node:fs/promises"; import { parseWord, PARSER_VERSION } from "./parser.js"; import { request, MIN_REFRESH } from "./source.js"; import { DATA } from "./paths.js"; const data = DATA + "parsed/"; // The downloaded Word files: a new parser reads them again without asking the university. const originals = DATA + "documents/"; // When each document was last downloaded, and the downloads under way: a forced refresh // downloads a document at most once per MIN_REFRESH, and simultaneous ones share a download. const fetchedAt = new Map(), loading = new Map(); /** A parsed catalog document, from the disk cache while the file and parser are unchanged. */ export async function scheduleFor(file, { refresh = false } = {}) { const cache = data + file.id + ".json"; if (refresh && Date.now() - (fetchedAt.get(file.id) || 0) < MIN_REFRESH) refresh = false; if (!refresh) try { const cached = JSON.parse(await readFile(cache, "utf8")); if (cached.source.modified === file.modified) { if (cached.schemaVersion === PARSER_VERSION) return cached; // The same file read by an older parser: parse the saved copy again. const saved = await readFile(originals + file.id).catch(() => null); if (saved) return parse(file, saved, cache); } } catch {} if (!loading.has(file.id)) loading.set( file.id, download(file, cache).finally(() => loading.delete(file.id)), ); return loading.get(file.id); } async function download(file, cache) { const buffer = await request(file.path); fetchedAt.set(file.id, Date.now()); await mkdir(originals, { recursive: true }); await writeFile(originals + file.id, buffer); return parse(file, buffer, cache); } async function parse(file, buffer, cache) { const parsed = await parseWord(buffer, { name: file.name, url: file.url, faculty: file.faculty, modified: file.modified, fileId: file.id, form: file.form, course: file.course, year: file.year, }); await mkdir(data, { recursive: true }); await writeFile(cache, JSON.stringify(parsed)); return parsed; }