psu-schedule/scripts/snapshot.js

// Catalog snapshot: every catalog document re-parsed by the current parser and compared with
// a saved result, so a parser change shows what it changes across all faculties before the
// container is rebuilt. The originals and the snapshot live in .data/ (not in git).
//
//   npm run snapshot -- fetch   download every catalog document into .data/originals/
//   npm run snapshot -- save    parse them and save the result as the reference
//   npm run snapshot            parse them again and print what changed (exit code 1 if anything)
//   npm run snapshot -- --all   the same, with every changed line instead of the first ten
import { readFile, writeFile, mkdir, readdir } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { parseWord, PARSER_VERSION } from "../server/parser.js";
import { diagnose } from "../server/diagnostics.js";
import { snapshotOf, diffLines } from "./snapshot-lines.js";

const data = fileURLToPath(new URL("../.data/", import.meta.url));
const originals = data + "originals/";
const reference = data + "snapshot.json";
const args = process.argv.slice(2);
const command = args.find((a) => !a.startsWith("--")) || "check";
const limit = args.includes("--all") ? Infinity : 10;

const catalog = async () => {
  const { getCatalog } = await import("../server/source.js");
  return getCatalog();
};

async function fetchAll() {
  const { request } = await import("../server/source.js");
  const files = (await catalog()).files;
  await mkdir(originals, { recursive: true });
  const index = {};
  let failed = 0;
  for (const file of files) {
    try {
      await writeFile(originals + file.id, await request(file.path));
      index[file.id] = { name: file.name, year: file.year, modified: file.modified, faculty: file.faculty };
      process.stdout.write(".");
    } catch (e) {
      failed++;
      console.error(`\n${file.name}: ${e.message}`);
    }
  }
  await writeFile(originals + "index.json", JSON.stringify(index, null, 1));
  console.log(`\nСкачано ${Object.keys(index).length} из ${files.length} документов${failed ? `, ошибок: ${failed}` : ""}.`);
}

/** Every downloaded document parsed now: {id: {name, lines, problems} | {name, error}}. */
async function parseAll() {
  let index;
  try {
    index = JSON.parse(await readFile(originals + "index.json", "utf8"));
  } catch {
    throw new Error("Нет скачанных документов: сначала npm run snapshot -- fetch");
  }
  const present = new Set(await readdir(originals));
  const result = {};
  for (const [id, meta] of Object.entries(index)) {
    if (!present.has(id)) continue;
    try {
      const schedule = await parseWord(await readFile(originals + id), { ...meta, fileId: id });
      const problems = {};
      for (const p of diagnose(schedule)) problems[p.code] = (problems[p.code] || 0) + 1;
      result[id] = { name: meta.name, lines: snapshotOf(schedule), problems };
    } catch (e) {
      result[id] = { name: meta.name, error: e.message };
    }
  }
  return result;
}

const total = (docs) =>
  Object.values(docs).reduce((sum, d) => {
    for (const [code, n] of Object.entries(d.problems || {})) sum[code] = (sum[code] || 0) + n;
    return sum;
  }, {});

async function save() {
  const docs = await parseAll();
  await writeFile(reference, JSON.stringify({ parserVersion: PARSER_VERSION, savedAt: new Date().toISOString(), docs }));
  const lessons = Object.values(docs).reduce((n, d) => n + (d.lines?.length - 1 || 0), 0);
  console.log(`Снимок сохранён: ${Object.keys(docs).length} документов, ${lessons} занятий, парсер ${PARSER_VERSION}.`);
  console.log("Подозрительные ячейки:", total(docs));
}

async function check() {
  let saved;
  try {
    saved = JSON.parse(await readFile(reference, "utf8"));
  } catch {
    throw new Error("Нет сохранённого снимка: сначала npm run snapshot -- save");
  }
  const now = await parseAll();
  let changed = 0, same = 0;
  for (const id of new Set([...Object.keys(saved.docs), ...Object.keys(now)])) {
    const before = saved.docs[id], after = now[id];
    const name = (after || before).name;
    if (!before || !after) {
      changed++;
      console.log(`\n${before ? "−" : "+"} ${name}: документ ${before ? "пропал" : "появился"}`);
      continue;
    }
    if (before.error || after.error) {
      if (before.error === after.error) { same++; continue; }
      changed++;
      console.log(`\n~ ${name}: ${after.error ? `теперь не разбирается — ${after.error}` : "теперь разбирается"}`);
      continue;
    }
    const { removed, added } = diffLines(before.lines, after.lines);
    if (!removed.length && !added.length) { same++; continue; }
    changed++;
    console.log(`\n~ ${name}: −${removed.length} +${added.length}`);
    for (const line of removed.slice(0, limit)) console.log(`  − ${line}`);
    if (removed.length > limit) console.log(`  … ещё ${removed.length - limit}`);
    for (const line of added.slice(0, limit)) console.log(`  + ${line}`);
    if (added.length > limit) console.log(`  … ещё ${added.length - limit}`);
  }
  const was = total(saved.docs), is = total(now);
  const codes = [...new Set([...Object.keys(was), ...Object.keys(is)])].sort();
  console.log(`\nПарсер ${saved.parserVersion} → ${PARSER_VERSION}. Изменённых документов: ${changed}, без изменений: ${same}.`);
  if (codes.length)
    console.log(
      "Подозрительные ячейки:",
      codes.map((c) => `${c} ${was[c] || 0}→${is[c] || 0}`).join(", "),
    );
  if (changed) process.exitCode = 1;
}

const commands = { fetch: fetchAll, save, check };
if (!commands[command]) {
  console.error(`Неизвестная команда «${command}». Есть: fetch, save, check.`);
  process.exitCode = 2;
} else
  await commands[command]().catch((e) => {
    console.error(e.message);
    process.exitCode = 2;
  });