59 lines
2.1 KiB
JavaScript
59 lines
2.1 KiB
JavaScript
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
|
import { parseWord, PARSER_VERSION } from "./parser.js";
|
|
import { request, MIN_REFRESH } from "./source.js";
|
|
import { DATA } from "./paths.js";
|
|
|
|
const data = DATA + "parsed/";
|
|
// The downloaded Word files: a new parser reads them again without asking the university.
|
|
const originals = DATA + "documents/";
|
|
// When each document was last downloaded, and the downloads under way: a forced refresh
|
|
// downloads a document at most once per MIN_REFRESH, and simultaneous ones share a download.
|
|
const fetchedAt = new Map(),
|
|
loading = new Map();
|
|
|
|
/** A parsed catalog document, from the disk cache while the file and parser are unchanged. */
|
|
export async function scheduleFor(file, { refresh = false } = {}) {
|
|
const cache = data + file.id + ".json";
|
|
if (refresh && Date.now() - (fetchedAt.get(file.id) || 0) < MIN_REFRESH) refresh = false;
|
|
if (!refresh)
|
|
try {
|
|
const cached = JSON.parse(await readFile(cache, "utf8"));
|
|
if (cached.source.modified === file.modified) {
|
|
if (cached.schemaVersion === PARSER_VERSION) return cached;
|
|
// The same file read by an older parser: parse the saved copy again.
|
|
const saved = await readFile(originals + file.id).catch(() => null);
|
|
if (saved) return parse(file, saved, cache);
|
|
}
|
|
} catch {}
|
|
if (!loading.has(file.id))
|
|
loading.set(
|
|
file.id,
|
|
download(file, cache).finally(() => loading.delete(file.id)),
|
|
);
|
|
return loading.get(file.id);
|
|
}
|
|
|
|
async function download(file, cache) {
|
|
const buffer = await request(file.path);
|
|
fetchedAt.set(file.id, Date.now());
|
|
await mkdir(originals, { recursive: true });
|
|
await writeFile(originals + file.id, buffer);
|
|
return parse(file, buffer, cache);
|
|
}
|
|
|
|
async function parse(file, buffer, cache) {
|
|
const parsed = await parseWord(buffer, {
|
|
name: file.name,
|
|
url: file.url,
|
|
faculty: file.faculty,
|
|
modified: file.modified,
|
|
fileId: file.id,
|
|
form: file.form,
|
|
course: file.course,
|
|
year: file.year,
|
|
});
|
|
await mkdir(data, { recursive: true });
|
|
await writeFile(cache, JSON.stringify(parsed));
|
|
return parsed;
|
|
}
|