psu-schedule/server/index.js

import express from "express";
import compression from "compression";
import { fileURLToPath } from "node:url";
import { resolve } from "node:path";
import multer from "multer";
import { parseWord, PARSER_VERSION } from "./parser.js";
import { scheduleFor } from "./schedules.js";
import { rateLimit, semaphore } from "./limits.js";
import { apiError, describe } from "./errors.js";
import { createHash } from "node:crypto";
import { timetable } from "./timetable.js";
import { catalogDiagnostics } from "./diagnostics.js";
import { securityHeaders, requestTimeout } from "./protect.js";
import { startWatch, watchOnce, watchStatus } from "./watch.js";
import { telegram } from "./alerts.js";
import { requestLog, trafficStatus } from "./metrics.js";
import { dashboardAuth } from "./auth.js";
import { listEvents } from "./events.js";
import {
  getIndex,
  indexStatus,
  search,
  dayOf,
  freeRooms,
  examsOf,
  findGroups,
  readyIndex,
  sessionEntriesOf,
  directoryCalendar,
  todayIso,
} from "./directory.js";
import {
  getCatalog,
  getCalendar,
  documentFor,
  request,
  sourceStatus,
} from "./source.js";
const app = express();
app.disable("x-powered-by");
// Behind Caddy or a tunnel on this host, the client's address comes from X-Forwarded-For;
// only proxies on private networks are believed, so a client cannot forge its own.
app.set("trust proxy", "loopback, linklocal, uniquelocal");
const dist = fileURLToPath(new URL("../dist/", import.meta.url));
app.use(securityHeaders(dist + "index.html"));
// One JSON line per API request and the counters of /api/health; first, so limited and timed
// out requests are in them too.
app.use("/api", requestLog());
// gzip or brotli: a semester of lessons is about 200 KB of JSON and shrinks tenfold.
app.use(compression());
// A generous limit per address: students behind one dormitory or university NAT share it, so
// it only stops scrapers and runaway clients. Health checks are not counted.
app.use(
  "/api",
  rateLimit({
    max: Number(process.env.PSU_API_RATE_LIMIT) || 300,
    windowMs: 60 * 1000,
    message: "Слишком много запросов с вашего адреса. Подождите минуту.",
    skip: (req) => req.path === "/health" || req.path === "/ready",
  }),
);
// Parsing an import may wait for a LibreOffice conversion and other imports.
app.use("/api", requestTimeout(Number(process.env.PSU_REQUEST_TIMEOUT_MS) || 30_000, { "/import": 90_000 }));
app.use("/api", (_req, res, next) => {
  // Clients may keep a response but must ask again each time; Express's ETag turns an
  // unchanged answer into an empty 304, so a phone re-downloads a timetable only when it changed.
  res.set("Cache-Control", "no-cache");
  next();
});
// Raised whenever an endpoint is added or changed, so a deployment is easy to verify:
// 2 — /api/timetable for the Android client; 3 — its range=semester;
// 4 — compression, ETag revalidation, throttled refresh and import limits;
// 5 — error codes (`code` beside `error`, see server/errors.js), timetable 304 before any work;
// 6 — the timetable carries the group's exams and credits from other catalog documents;
// 7 — /api/diagnostics, cells the parser could not read with confidence;
// 8 — /api/ready, and `ready` in /api/health;
// 9 — a request limit on /api (429 too_many_requests), a 30 s time limit (504 timeout), security headers;
// 10 — parserVersion in /api/catalog;
// 11 — the catalog is checked by ETag every 5 minutes; `watch`, `telegram` and `catalog.lastCrawl` in /api/health;
// 12 — a JSON log line per request;
// 13 — the dashboard at /status (password), /api/health only says alive, ready and versions;
// 14 — a lesson's `place` and a day's `places` ("Занятия в Полоцке") in /api/timetable.
const API_VERSION = 14;

/**
 * What the server still waits for before it answers from memory: the catalog, the week table
 * and the directory index. A first start without a cache crawls and parses every document.
 */
const waitingFor = (sources, directory) =>
  [!sources.catalog && "catalog", !sources.calendar && "calendar", !directory.builtAt && "index"].filter(Boolean);
// Liveness plus what the server currently reads from the university, without network calls.
// Public: alive, ready, versions. Everything else about the server is behind the dashboard's
// password (/status), as it names documents, clients and the owner's set-up.
app.get("/api/health", async (req, res) => {
  const sources = await sourceStatus().catch(() => ({}));
  const waiting = waitingFor(sources, indexStatus());
  res.json({
    ok: true,
    // The caller's address as the server sees it: behind a proxy it must be the client's own,
    // or every visitor shares one request limit.
    client: req.ip,
    ready: waiting.length === 0,
    waiting,
    apiVersion: API_VERSION,
    parserVersion: PARSER_VERSION,
    year: sources.catalog?.year || null,
  });
});
const startedAt = new Date().toISOString();
// The dashboard (server/status/) and its data, behind DASHBOARD_PASSWORD. The data sits under
// /status too, so the browser sends the password it was given for the page.
const statusPage = fileURLToPath(new URL("./status/", import.meta.url));
app.use("/status", dashboardAuth());
app.get("/status/data.json", async (_req, res, next) => {
  try {
    const sources = await sourceStatus();
    const directory = indexStatus();
    const idx = readyIndex();
    res.json({
      server: {
        apiVersion: API_VERSION,
        parserVersion: PARSER_VERSION,
        startedAt,
        node: process.version,
        ready: waitingFor(sources, directory).length === 0,
        waiting: waitingFor(sources, directory),
      },
      ...sources,
      directory,
      watch: watchStatus,
      telegram: { enabled: telegram.enabled, sent: telegram.sent, lastSentAt: telegram.lastSentAt, lastError: telegram.lastError },
      traffic: trafficStatus(),
      events: await listEvents(50),
      diagnostics: idx && directory.builtAt ? catalogDiagnostics(idx) : null,
    });
  } catch (e) {
    next(e);
  }
});
app.use("/status", express.static(statusPage, { maxAge: 0 }));
// Readiness for Docker and a proxy: 503 until the catalog, the week table and the index are
// loaded, so traffic does not reach a server whose first requests would wait half a minute.
app.get("/api/ready", async (_req, res) => {
  const sources = await sourceStatus().catch(() => ({}));
  const directory = indexStatus();
  const waiting = waitingFor(sources, directory);
  if (waiting.length)
    return res.status(503).json({
      ready: false,
      waiting,
      error: "Сервер ещё загружает каталог расписаний. Попробуйте через минуту.",
      code: "not_ready",
    });
  res.json({ ready: true, documents: directory.documents, builtAt: directory.builtAt });
});
const isDate = (s) => /^\d{4}-\d{2}-\d{2}$/.test(s || "");
const badRequest = (res, error) => res.status(400).json({ error, code: "bad_request" });
// University-wide lookups over every catalog document.
app.get("/api/directory/search", async (req, res, next) => {
  try {
    res.json(search(await getIndex(), String(req.query.q || "").slice(0, 60)));
  } catch (e) {
    next(e);
  }
});
app.get("/api/directory/day", async (req, res, next) => {
  try {
    const date = req.query.date || todayIso();
    const teacher = String(req.query.teacher || "").slice(0, 80),
      room = String(req.query.room || "").slice(0, 20);
    if (!isDate(date)) return badRequest(res, "Дата должна быть в формате ГГГГ-ММ-ДД.");
    if (!teacher && !room) return badRequest(res, "Укажите преподавателя или аудиторию.");
    res.json(dayOf(await getIndex(), await directoryCalendar(), { teacher, room }, date));
  } catch (e) {
    next(e);
  }
});
app.get("/api/directory/free", async (req, res, next) => {
  try {
    const date = req.query.date || todayIso(),
      pair = Number(req.query.pair);
    if (!isDate(date)) return badRequest(res, "Дата должна быть в формате ГГГГ-ММ-ДД.");
    const calendar = await directoryCalendar();
    const result = freeRooms(await getIndex(), calendar, date, pair);
    if (!result) return badRequest(res, "Нет такой пары в расписании звонков.");
    res.json(result);
  } catch (e) {
    next(e);
  }
});
app.get("/api/groups", async (req, res, next) => {
  try {
    const text = (name) => String(req.query[name] || "").slice(0, 120);
    res.json(
      findGroups(await getIndex(), {
        q: text("q"),
        faculty: text("faculty"),
        course: text("course"),
        form: text("form"),
      }),
    );
  } catch (e) {
    next(e);
  }
});
// A group's lessons by date with weeks applied: for native and other clients.
app.get("/api/timetable", async (req, res, next) => {
  try {
    const doc = String(req.query.doc || ""),
      group = String(req.query.group || "").slice(0, 40),
      from = req.query.from || todayIso();
    if (!doc || !group) return badRequest(res, "Укажите документ (doc) и группу (group).");
    if (!isDate(from)) return badRequest(res, "Дата должна быть в формате ГГГГ-ММ-ДД.");
    const file = await documentFor(doc);
    const calendar = await directoryCalendar();
    const range = req.query.range === "semester" ? "semester" : undefined;
    const today = todayIso();
    // The group's exams and credits from other documents ("Расписание зачетов"); a server
    // still building its first index answers without them rather than wait.
    const idx = readyIndex();
    // The answer follows only from these: the document's version, the parser and the server
    // that read it, the week table, the request and the date. An unchanged one comes back
    // as an empty 304 before the document is opened or a single day is computed.
    const etag = 'W/"tt-' + createHash("sha1")
      .update([API_VERSION, PARSER_VERSION, file.id, file.modified, calendar?.updatedAt, idx?.builtAt, group, from, req.query.days, range, today].join("|"))
      .digest("base64url")
      .slice(0, 22) + '"';
    res.set("ETag", etag);
    if (req.fresh) return res.status(304).end();
    const schedule = await scheduleFor(file);
    res.json(
      timetable({
        file,
        schedule,
        calendar,
        group,
        from,
        days: req.query.days,
        range,
        today,
        session: sessionEntriesOf(idx, group, file.faculty || schedule.source.faculty, file.id),
      }),
    );
  } catch (e) {
    next(e);
  }
});
app.get("/api/exams", async (req, res, next) => {
  try {
    const group = String(req.query.group || "").slice(0, 40);
    if (!group) return badRequest(res, "Укажите группу.");
    res.json({
      group,
      exams: examsOf(await getIndex(), group, String(req.query.faculty || "") || null),
    });
  } catch (e) {
    next(e);
  }
});
// Cells of catalog documents that look misread: a new layout or a typo in Word, with the
// original text, so they can be checked against the document and the parser taught.
app.get("/api/diagnostics", async (req, res, next) => {
  try {
    const text = (name) => String(req.query[name] || "").slice(0, 60);
    res.json(catalogDiagnostics(await getIndex(), { group: text("group"), doc: text("doc"), code: text("code") }));
  } catch (e) {
    next(e);
  }
});
app.get("/api/catalog", async (req, res, next) => {
  try {
    // The parser version lets the PWA re-read schedules it saved from an older parser.
    res.json({ ...(await getCatalog(req.query.refresh === "1")), parserVersion: PARSER_VERSION });
  } catch (e) {
    next(e);
  }
});
app.get("/api/calendar", async (req, res, next) => {
  try {
    res.json(await getCalendar(req.query.refresh === "1"));
  } catch (e) {
    next(e);
  }
});
app.get("/api/schedule/:id", async (req, res, next) => {
  try {
    const file = await documentFor(req.params.id);
    res.json(await scheduleFor(file, { refresh: req.query.refresh === "1" }));
  } catch (e) {
    next(e);
  }
});
app.get("/api/original/:id", async (req, res, next) => {
  try {
    const file = await documentFor(req.params.id);
    res.set("Content-Type", "application/octet-stream");
    res.set(
      "Content-Disposition",
      `attachment; filename*=UTF-8''${encodeURIComponent(file.name)}`,
    );
    res.send(await request(file.path));
  } catch (e) {
    next(e);
  }
});
// Parsing runs a LibreOffice conversion for old .doc files: two at a time, a few more wait,
// and each address may import ten files in ten minutes.
const importLimit = rateLimit({
  max: 10,
  windowMs: 10 * 60 * 1000,
  message: "Слишком много файлов подряд. Попробуйте через несколько минут.",
});
const parsing = semaphore({
  limit: 2,
  queue: 4,
  message: "Сервер занят разбором других файлов. Попробуйте через минуту.",
});
const upload = multer({
  storage: multer.memoryStorage(),
  limits: { fileSize: 10 * 1024 * 1024, files: 1, fields: 0 },
});
app.post("/api/import", importLimit, upload.single("file"), async (req, res, next) => {
  try {
    if (!req.file)
      return badRequest(res, "Выберите файл Word.");
    // Multer decodes multipart names as latin1 by default.
    const name = Buffer.from(req.file.originalname, "latin1").toString("utf8");
    if (!/\.docx?$/i.test(name))
      return badRequest(res, "Поддерживаются файлы .docx и .doc.");
    res.json(await parsing(() => parseWord(req.file.buffer, { name, uploaded: true })));
  } catch (e) {
    // A file that cannot be read is the client's problem; a busy server keeps its 429.
    next(e.status || e.code === "LIMIT_FILE_SIZE" ? e : apiError(422, "unreadable_file", e.message));
  }
});
app.use(express.static(dist, { maxAge: 0 }));
app.get("/{*path}", (req, res, next) =>
  req.path.startsWith("/api/")
    ? res.status(404).json({ error: "Адрес не найден.", code: "unknown_endpoint" })
    : res.sendFile(dist + "index.html", (err) => err && next(err)),
);
app.use((error, req, res, _next) => {
  const { status, code, message } = describe(error);
  // Answered already, by the time limit: the late error is only logged.
  if (res.headersSent) return console.error(`${req.method} ${req.originalUrl} после ответа: ${error.message}`);
  // Faults of the server keep their stack; expected errors need a line.
  if (status >= 500 && code === "internal") console.error(req.method, req.originalUrl, error);
  else console.error(`${req.method} ${req.originalUrl} → ${status} ${code}: ${error.message}`);
  res.status(status).json({ error: message, code });
});
export { app };

/** Starts listening and loads the week table and the whole catalog in the background. */
export function start(port = Number(process.env.PORT || 3001), host = process.env.HOST || "127.0.0.1") {
  const server = app.listen(port, host, () => {
    console.log(`API и PWA: http://127.0.0.1:${port}`);
    // So the first visit and the first search do not wait for the university.
    getCalendar().catch((e) => console.error(`Таблица недель: ${e.message}`));
    getIndex()
      .catch((e) => console.error(`Индекс: ${e.message}`))
      // Then the catalog is watched every few minutes, with messages in Telegram if set up.
      .finally(() => {
        watchOnce();
        startWatch();
      });
  });
  // A client must send its headers within 15 s and the whole request (a Word upload of up to
  // 10 MB on a slow phone) within two minutes, so slow connections cannot pile up.
  server.headersTimeout = 15_000;
  server.requestTimeout = 120_000;
  return server;
}
// `node server/index.js` serves; tests import the app without listening.
if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) start();