341 lines
14 KiB
JavaScript
341 lines
14 KiB
JavaScript
import express from "express";
|
||
import compression from "compression";
|
||
import { fileURLToPath } from "node:url";
|
||
import { resolve } from "node:path";
|
||
import multer from "multer";
|
||
import { parseWord, PARSER_VERSION } from "./parser.js";
|
||
import { scheduleFor } from "./schedules.js";
|
||
import { rateLimit, semaphore } from "./limits.js";
|
||
import { apiError, describe } from "./errors.js";
|
||
import { createHash } from "node:crypto";
|
||
import { timetable } from "./timetable.js";
|
||
import { catalogDiagnostics } from "./diagnostics.js";
|
||
import { securityHeaders, requestTimeout } from "./protect.js";
|
||
import { startWatch, watchOnce, watchStatus } from "./watch.js";
|
||
import { telegram } from "./alerts.js";
|
||
import { requestLog, trafficStatus } from "./metrics.js";
|
||
import {
|
||
getIndex,
|
||
indexStatus,
|
||
search,
|
||
dayOf,
|
||
freeRooms,
|
||
examsOf,
|
||
findGroups,
|
||
readyIndex,
|
||
sessionEntriesOf,
|
||
directoryCalendar,
|
||
todayIso,
|
||
} from "./directory.js";
|
||
import {
|
||
getCatalog,
|
||
getCalendar,
|
||
documentFor,
|
||
request,
|
||
sourceStatus,
|
||
} from "./source.js";
|
||
const app = express();
|
||
app.disable("x-powered-by");
|
||
// Behind Caddy or a tunnel on this host, the client's address comes from X-Forwarded-For;
|
||
// only proxies on private networks are believed, so a client cannot forge its own.
|
||
app.set("trust proxy", "loopback, linklocal, uniquelocal");
|
||
const dist = fileURLToPath(new URL("../dist/", import.meta.url));
|
||
app.use(securityHeaders(dist + "index.html"));
|
||
// One JSON line per API request and the counters of /api/health; first, so limited and timed
|
||
// out requests are in them too.
|
||
app.use("/api", requestLog());
|
||
// gzip or brotli: a semester of lessons is about 200 KB of JSON and shrinks tenfold.
|
||
app.use(compression());
|
||
// A generous limit per address: students behind one dormitory or university NAT share it, so
|
||
// it only stops scrapers and runaway clients. Health checks are not counted.
|
||
app.use(
|
||
"/api",
|
||
rateLimit({
|
||
max: Number(process.env.PSU_API_RATE_LIMIT) || 300,
|
||
windowMs: 60 * 1000,
|
||
message: "Слишком много запросов с вашего адреса. Подождите минуту.",
|
||
skip: (req) => req.path === "/health" || req.path === "/ready",
|
||
}),
|
||
);
|
||
// Parsing an import may wait for a LibreOffice conversion and other imports.
|
||
app.use("/api", requestTimeout(Number(process.env.PSU_REQUEST_TIMEOUT_MS) || 30_000, { "/import": 90_000 }));
|
||
app.use("/api", (_req, res, next) => {
|
||
// Clients may keep a response but must ask again each time; Express's ETag turns an
|
||
// unchanged answer into an empty 304, so a phone re-downloads a timetable only when it changed.
|
||
res.set("Cache-Control", "no-cache");
|
||
next();
|
||
});
|
||
// Raised whenever an endpoint is added or changed, so a deployment is easy to verify:
|
||
// 2 — /api/timetable for the Android client; 3 — its range=semester;
|
||
// 4 — compression, ETag revalidation, throttled refresh and import limits;
|
||
// 5 — error codes (`code` beside `error`, see server/errors.js), timetable 304 before any work;
|
||
// 6 — the timetable carries the group's exams and credits from other catalog documents;
|
||
// 7 — /api/diagnostics, cells the parser could not read with confidence;
|
||
// 8 — /api/ready, and `ready` in /api/health;
|
||
// 9 — a request limit on /api (429 too_many_requests), a 30 s time limit (504 timeout), security headers;
|
||
// 10 — parserVersion in /api/catalog;
|
||
// 11 — the catalog is checked by ETag every 5 minutes; `watch`, `telegram` and `catalog.lastCrawl` in /api/health;
|
||
// 12 — a JSON log line per request and `traffic` in /api/health.
|
||
const API_VERSION = 12;
|
||
|
||
/**
|
||
* What the server still waits for before it answers from memory: the catalog, the week table
|
||
* and the directory index. A first start without a cache crawls and parses every document.
|
||
*/
|
||
const waitingFor = (sources, directory) =>
|
||
[!sources.catalog && "catalog", !sources.calendar && "calendar", !directory.builtAt && "index"].filter(Boolean);
|
||
// Liveness plus what the server currently reads from the university, without network calls.
|
||
app.get("/api/health", async (req, res) => {
|
||
const sources = await sourceStatus().catch((e) => ({ error: e.message }));
|
||
res.json({
|
||
ok: true,
|
||
// The caller's address as the server sees it: behind a proxy it must be the client's own,
|
||
// or every visitor shares one request limit.
|
||
client: req.ip,
|
||
ready: waitingFor(sources, indexStatus()).length === 0,
|
||
apiVersion: API_VERSION,
|
||
parserVersion: PARSER_VERSION,
|
||
...sources,
|
||
directory: indexStatus(),
|
||
// The server's own checks of the catalog and its messages to the owner.
|
||
watch: watchStatus,
|
||
// What the API was asked in the last hour and since the start (server/metrics.js).
|
||
traffic: trafficStatus(),
|
||
telegram: { enabled: telegram.enabled, sent: telegram.sent, lastSentAt: telegram.lastSentAt, lastError: telegram.lastError },
|
||
});
|
||
});
|
||
// Readiness for Docker and a proxy: 503 until the catalog, the week table and the index are
|
||
// loaded, so traffic does not reach a server whose first requests would wait half a minute.
|
||
app.get("/api/ready", async (_req, res) => {
|
||
const sources = await sourceStatus().catch(() => ({}));
|
||
const directory = indexStatus();
|
||
const waiting = waitingFor(sources, directory);
|
||
if (waiting.length)
|
||
return res.status(503).json({
|
||
ready: false,
|
||
waiting,
|
||
error: "Сервер ещё загружает каталог расписаний. Попробуйте через минуту.",
|
||
code: "not_ready",
|
||
});
|
||
res.json({ ready: true, documents: directory.documents, builtAt: directory.builtAt });
|
||
});
|
||
const isDate = (s) => /^\d{4}-\d{2}-\d{2}$/.test(s || "");
|
||
const badRequest = (res, error) => res.status(400).json({ error, code: "bad_request" });
|
||
// University-wide lookups over every catalog document.
|
||
app.get("/api/directory/search", async (req, res, next) => {
|
||
try {
|
||
res.json(search(await getIndex(), String(req.query.q || "").slice(0, 60)));
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/directory/day", async (req, res, next) => {
|
||
try {
|
||
const date = req.query.date || todayIso();
|
||
const teacher = String(req.query.teacher || "").slice(0, 80),
|
||
room = String(req.query.room || "").slice(0, 20);
|
||
if (!isDate(date)) return badRequest(res, "Дата должна быть в формате ГГГГ-ММ-ДД.");
|
||
if (!teacher && !room) return badRequest(res, "Укажите преподавателя или аудиторию.");
|
||
res.json(dayOf(await getIndex(), await directoryCalendar(), { teacher, room }, date));
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/directory/free", async (req, res, next) => {
|
||
try {
|
||
const date = req.query.date || todayIso(),
|
||
pair = Number(req.query.pair);
|
||
if (!isDate(date)) return badRequest(res, "Дата должна быть в формате ГГГГ-ММ-ДД.");
|
||
const calendar = await directoryCalendar();
|
||
const result = freeRooms(await getIndex(), calendar, date, pair);
|
||
if (!result) return badRequest(res, "Нет такой пары в расписании звонков.");
|
||
res.json(result);
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/groups", async (req, res, next) => {
|
||
try {
|
||
const text = (name) => String(req.query[name] || "").slice(0, 120);
|
||
res.json(
|
||
findGroups(await getIndex(), {
|
||
q: text("q"),
|
||
faculty: text("faculty"),
|
||
course: text("course"),
|
||
form: text("form"),
|
||
}),
|
||
);
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
// A group's lessons by date with weeks applied: for native and other clients.
|
||
app.get("/api/timetable", async (req, res, next) => {
|
||
try {
|
||
const doc = String(req.query.doc || ""),
|
||
group = String(req.query.group || "").slice(0, 40),
|
||
from = req.query.from || todayIso();
|
||
if (!doc || !group) return badRequest(res, "Укажите документ (doc) и группу (group).");
|
||
if (!isDate(from)) return badRequest(res, "Дата должна быть в формате ГГГГ-ММ-ДД.");
|
||
const file = await documentFor(doc);
|
||
const calendar = await directoryCalendar();
|
||
const range = req.query.range === "semester" ? "semester" : undefined;
|
||
const today = todayIso();
|
||
// The group's exams and credits from other documents ("Расписание зачетов"); a server
|
||
// still building its first index answers without them rather than wait.
|
||
const idx = readyIndex();
|
||
// The answer follows only from these: the document's version, the parser and the server
|
||
// that read it, the week table, the request and the date. An unchanged one comes back
|
||
// as an empty 304 before the document is opened or a single day is computed.
|
||
const etag = 'W/"tt-' + createHash("sha1")
|
||
.update([API_VERSION, PARSER_VERSION, file.id, file.modified, calendar?.updatedAt, idx?.builtAt, group, from, req.query.days, range, today].join("|"))
|
||
.digest("base64url")
|
||
.slice(0, 22) + '"';
|
||
res.set("ETag", etag);
|
||
if (req.fresh) return res.status(304).end();
|
||
const schedule = await scheduleFor(file);
|
||
res.json(
|
||
timetable({
|
||
file,
|
||
schedule,
|
||
calendar,
|
||
group,
|
||
from,
|
||
days: req.query.days,
|
||
range,
|
||
today,
|
||
session: sessionEntriesOf(idx, group, file.faculty || schedule.source.faculty, file.id),
|
||
}),
|
||
);
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/exams", async (req, res, next) => {
|
||
try {
|
||
const group = String(req.query.group || "").slice(0, 40);
|
||
if (!group) return badRequest(res, "Укажите группу.");
|
||
res.json({
|
||
group,
|
||
exams: examsOf(await getIndex(), group, String(req.query.faculty || "") || null),
|
||
});
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
// Cells of catalog documents that look misread: a new layout or a typo in Word, with the
|
||
// original text, so they can be checked against the document and the parser taught.
|
||
app.get("/api/diagnostics", async (req, res, next) => {
|
||
try {
|
||
const text = (name) => String(req.query[name] || "").slice(0, 60);
|
||
res.json(catalogDiagnostics(await getIndex(), { group: text("group"), doc: text("doc"), code: text("code") }));
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/catalog", async (req, res, next) => {
|
||
try {
|
||
// The parser version lets the PWA re-read schedules it saved from an older parser.
|
||
res.json({ ...(await getCatalog(req.query.refresh === "1")), parserVersion: PARSER_VERSION });
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/calendar", async (req, res, next) => {
|
||
try {
|
||
res.json(await getCalendar(req.query.refresh === "1"));
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/schedule/:id", async (req, res, next) => {
|
||
try {
|
||
const file = await documentFor(req.params.id);
|
||
res.json(await scheduleFor(file, { refresh: req.query.refresh === "1" }));
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
app.get("/api/original/:id", async (req, res, next) => {
|
||
try {
|
||
const file = await documentFor(req.params.id);
|
||
res.set("Content-Type", "application/octet-stream");
|
||
res.set(
|
||
"Content-Disposition",
|
||
`attachment; filename*=UTF-8''${encodeURIComponent(file.name)}`,
|
||
);
|
||
res.send(await request(file.path));
|
||
} catch (e) {
|
||
next(e);
|
||
}
|
||
});
|
||
// Parsing runs a LibreOffice conversion for old .doc files: two at a time, a few more wait,
|
||
// and each address may import ten files in ten minutes.
|
||
const importLimit = rateLimit({
|
||
max: 10,
|
||
windowMs: 10 * 60 * 1000,
|
||
message: "Слишком много файлов подряд. Попробуйте через несколько минут.",
|
||
});
|
||
const parsing = semaphore({
|
||
limit: 2,
|
||
queue: 4,
|
||
message: "Сервер занят разбором других файлов. Попробуйте через минуту.",
|
||
});
|
||
const upload = multer({
|
||
storage: multer.memoryStorage(),
|
||
limits: { fileSize: 10 * 1024 * 1024, files: 1, fields: 0 },
|
||
});
|
||
app.post("/api/import", importLimit, upload.single("file"), async (req, res, next) => {
|
||
try {
|
||
if (!req.file)
|
||
return badRequest(res, "Выберите файл Word.");
|
||
// Multer decodes multipart names as latin1 by default.
|
||
const name = Buffer.from(req.file.originalname, "latin1").toString("utf8");
|
||
if (!/\.docx?$/i.test(name))
|
||
return badRequest(res, "Поддерживаются файлы .docx и .doc.");
|
||
res.json(await parsing(() => parseWord(req.file.buffer, { name, uploaded: true })));
|
||
} catch (e) {
|
||
// A file that cannot be read is the client's problem; a busy server keeps its 429.
|
||
next(e.status || e.code === "LIMIT_FILE_SIZE" ? e : apiError(422, "unreadable_file", e.message));
|
||
}
|
||
});
|
||
app.use(express.static(dist, { maxAge: 0 }));
|
||
app.get("/{*path}", (req, res, next) =>
|
||
req.path.startsWith("/api/")
|
||
? res.status(404).json({ error: "Адрес не найден.", code: "unknown_endpoint" })
|
||
: res.sendFile(dist + "index.html", (err) => err && next(err)),
|
||
);
|
||
app.use((error, req, res, _next) => {
|
||
const { status, code, message } = describe(error);
|
||
// Answered already, by the time limit: the late error is only logged.
|
||
if (res.headersSent) return console.error(`${req.method} ${req.originalUrl} после ответа: ${error.message}`);
|
||
// Faults of the server keep their stack; expected errors need a line.
|
||
if (status >= 500 && code === "internal") console.error(req.method, req.originalUrl, error);
|
||
else console.error(`${req.method} ${req.originalUrl} → ${status} ${code}: ${error.message}`);
|
||
res.status(status).json({ error: message, code });
|
||
});
|
||
export { app };
|
||
|
||
/** Starts listening and loads the week table and the whole catalog in the background. */
|
||
export function start(port = Number(process.env.PORT || 3001), host = process.env.HOST || "127.0.0.1") {
|
||
const server = app.listen(port, host, () => {
|
||
console.log(`API и PWA: http://127.0.0.1:${port}`);
|
||
// So the first visit and the first search do not wait for the university.
|
||
getCalendar().catch((e) => console.error(`Таблица недель: ${e.message}`));
|
||
getIndex()
|
||
.catch((e) => console.error(`Индекс: ${e.message}`))
|
||
// Then the catalog is watched every few minutes, with messages in Telegram if set up.
|
||
.finally(() => {
|
||
watchOnce();
|
||
startWatch();
|
||
});
|
||
});
|
||
// A client must send its headers within 15 s and the whole request (a Word upload of up to
|
||
// 10 MB on a slow phone) within two minutes, so slow connections cannot pile up.
|
||
server.headersTimeout = 15_000;
|
||
server.requestTimeout = 120_000;
|
||
return server;
|
||
}
|
||
// `node server/index.js` serves; tests import the app without listening.
|
||
if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) start();
|