feat: Implement caching and rate limiting for API requests

server/index.js

@@ -1,8 +1,10 @@
 import express from "express";
+import compression from "compression";
 import { fileURLToPath } from "node:url";
 import multer from "multer";
 import { parseWord, PARSER_VERSION } from "./parser.js";
 import { scheduleFor } from "./schedules.js";
+import { rateLimit, semaphore } from "./limits.js";
 import { timetable } from "./timetable.js";
 import {
   getIndex,
@@ -24,13 +26,21 @@ import {
 } from "./source.js";
 const app = express();
 app.disable("x-powered-by");
+// Behind Caddy or a tunnel on this host, the client's address comes from X-Forwarded-For;
+// only proxies on private networks are believed, so a client cannot forge its own.
+app.set("trust proxy", "loopback, linklocal, uniquelocal");
+// gzip or brotli: a semester of lessons is about 200 KB of JSON and shrinks tenfold.
+app.use(compression());
 app.use("/api", (_req, res, next) => {
-  res.set("Cache-Control", "no-store");
+  // Clients may keep a response but must ask again each time; Express's ETag turns an
+  // unchanged answer into an empty 304, so a phone re-downloads a timetable only when it changed.
+  res.set("Cache-Control", "no-cache");
   next();
 });
 // Raised whenever an endpoint is added or changed, so a deployment is easy to verify:
-// 2 — /api/timetable for the Android client; 3 — its range=semester.
-const API_VERSION = 3;
+// 2 — /api/timetable for the Android client; 3 — its range=semester;
+// 4 — compression, ETag revalidation, throttled refresh and import limits.
+const API_VERSION = 4;
 
 // Liveness plus what the server currently reads from the university, without network calls.
 app.get("/api/health", async (_req, res) => {
@@ -165,11 +175,23 @@ app.get("/api/original/:id", async (req, res, next) => {
     next(e);
   }
 });
+// Parsing runs a LibreOffice conversion for old .doc files: two at a time, a few more wait,
+// and each address may import ten files in ten minutes.
+const importLimit = rateLimit({
+  max: 10,
+  windowMs: 10 * 60 * 1000,
+  message: "Слишком много файлов подряд. Попробуйте через несколько минут.",
+});
+const parsing = semaphore({
+  limit: 2,
+  queue: 4,
+  message: "Сервер занят разбором других файлов. Попробуйте через минуту.",
+});
 const upload = multer({
   storage: multer.memoryStorage(),
   limits: { fileSize: 10 * 1024 * 1024, files: 1, fields: 0 },
 });
-app.post("/api/import", upload.single("file"), async (req, res, next) => {
+app.post("/api/import", importLimit, upload.single("file"), async (req, res, next) => {
   try {
     if (!req.file)
       return res.status(400).json({ error: "Выберите файл Word." });
@@ -179,9 +201,10 @@ app.post("/api/import", upload.single("file"), async (req, res, next) => {
       return res
         .status(400)
         .json({ error: "Поддерживаются файлы .docx и .doc." });
-    res.json(await parseWord(req.file.buffer, { name, uploaded: true }));
+    res.json(await parsing(() => parseWord(req.file.buffer, { name, uploaded: true })));
   } catch (e) {
-    e.status = 422;
+    // A file that cannot be read is the client's problem (422); a busy server keeps its 429.
+    e.status ??= 422;
     next(e);
   }
 });
@@ -200,11 +223,9 @@ app.use((error, _req, res, _next) => {
       error:
         error.code === "LIMIT_FILE_SIZE"
           ? "Файл больше 10 МБ. Выберите документ меньшего размера."
-          : error.status === 422
+          : [404, 422, 429].includes(error.status)
             ? error.message
-            : error.status === 404
-              ? error.message
-              : "Не удалось загрузить расписание. Проверьте подключение и попробуйте ещё раз.",
+            : "Не удалось загрузить расписание. Проверьте подключение и попробуйте ещё раз.",
     });
 });
 const port = Number(process.env.PORT || 3001);

server/limits.js

@@ -0,0 +1,42 @@
+/**
+ * Limits for the expensive endpoints: parsing an uploaded Word file can take a LibreOffice
+ * conversion and seconds of CPU, so one client must not be able to occupy the server.
+ */
+
+/** At most `max` requests per client address in each `windowMs`; the rest get 429. */
+export function rateLimit({ max, windowMs, message }) {
+  const windows = new Map();
+  return (req, res, next) => {
+    const now = Date.now();
+    // Finished windows are dropped on the way, so the map holds only active clients.
+    for (const [key, w] of windows) if (now - w.start >= windowMs) windows.delete(key);
+    const w = windows.get(req.ip) ?? { start: now, count: 0 };
+    w.count++;
+    windows.set(req.ip, w);
+    if (w.count <= max) return next();
+    res.set("Retry-After", String(Math.ceil((w.start + windowMs - now) / 1000)));
+    res.status(429).json({ error: message });
+  };
+}
+
+/**
+ * Runs at most `limit` tasks at once; up to `queue` more wait for a free slot,
+ * and beyond that a task is refused with a 429 error carrying `message`.
+ */
+export function semaphore({ limit, queue, message }) {
+  let active = 0;
+  const waiting = [];
+  return async function run(task) {
+    if (active < limit) active++;
+    else if (waiting.length < queue) await new Promise((resolve) => waiting.push(resolve));
+    else throw Object.assign(new Error(message), { status: 429 });
+    try {
+      return await task();
+    } finally {
+      // The slot passes straight to the next waiting task, or frees up.
+      const next = waiting.shift();
+      if (next) next();
+      else active--;
+    }
+  };
+}

server/schedules.js

@@ -1,13 +1,18 @@
 import { readFile, writeFile, mkdir } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import { parseWord, PARSER_VERSION } from "./parser.js";
-import { request } from "./source.js";
+import { request, MIN_REFRESH } from "./source.js";
 
 const data = fileURLToPath(new URL("../.data/parsed/", import.meta.url));
+// When each document was last downloaded, and the downloads under way: a forced refresh
+// downloads a document at most once per MIN_REFRESH, and simultaneous ones share a download.
+const fetchedAt = new Map(),
+  loading = new Map();
 
 /** A parsed catalog document, from the disk cache while the file and parser are unchanged. */
 export async function scheduleFor(file, { refresh = false } = {}) {
   const cache = data + file.id + ".json";
+  if (refresh && Date.now() - (fetchedAt.get(file.id) || 0) < MIN_REFRESH) refresh = false;
   if (!refresh)
     try {
       const cached = JSON.parse(await readFile(cache, "utf8"));
@@ -17,7 +22,17 @@ export async function scheduleFor(file, { refresh = false } = {}) {
       )
         return cached;
     } catch {}
+  if (!loading.has(file.id))
+    loading.set(
+      file.id,
+      download(file, cache).finally(() => loading.delete(file.id)),
+    );
+  return loading.get(file.id);
+}
+
+async function download(file, cache) {
   const buffer = await request(file.path);
+  fetchedAt.set(file.id, Date.now());
   const parsed = await parseWord(buffer, {
     name: file.name,
     url: file.url,

server/source.js

@@ -21,8 +21,22 @@ const arr = (value) => (value ? (Array.isArray(value) ? value : [value]) : []);
 const DISCOVERY_TTL = 15 * 60 * 1000,
   CATALOG_TTL = 15 * 60 * 1000,
   CALENDAR_TTL = 6 * 60 * 60 * 1000;
+// A forced refresh (?refresh=1, the PWA's "Обновить") reaches the university at most this often;
+// sooner, the copy just read is the answer. Anyone can send the request, and one catalog crawl
+// is up to 180 listings of the university's Nextcloud.
+export const MIN_REFRESH = 2 * 60 * 1000;
+const CALENDAR_MIN_REFRESH = 10 * 60 * 1000;
+/** Milliseconds since an ISO time; Infinity for a missing or broken one. */
+const ageOf = (iso) => {
+  const age = Date.now() - new Date(iso).getTime();
+  return Number.isFinite(age) ? age : Infinity;
+};
 let catalogue, pending, calendar, calendarPending, sources, sourcesPending;
 const status = { catalogError: null, calendarError: null };
+// When the university last failed to answer: background refreshes back off for MIN_REFRESH,
+// so a down source is not crawled again on every request.
+const failedAt = { catalog: 0, calendar: 0 };
+const backingOff = (what) => Date.now() - failedAt[what] < MIN_REFRESH;
 
 export async function request(path, method = "GET") {
   // Only the university share is reachable; listings and catalog entries supply the paths.
@@ -232,15 +246,17 @@ export async function getCatalog(refresh = false) {
     const y = academicStartYear(todayIso());
     return `${y}–${y + 1}`;
   })();
+  const age = catalogue ? ageOf(catalogue.updatedAt) : Infinity;
   // A cached catalog of last year is refreshed as soon as the new year begins.
-  const fresh =
-    catalogue &&
-    Date.now() - new Date(catalogue.updatedAt).getTime() < CATALOG_TTL &&
-    (catalogue.year === expected || catalogue.currentYear === false);
-  if (!refresh && fresh) return catalogue;
+  const thisYear =
+    catalogue && (catalogue.year === expected || catalogue.currentYear === false);
+  const forced = refresh && age >= MIN_REFRESH;
+  if (thisYear && age < CATALOG_TTL && !forced) return catalogue;
+  if (thisYear && !forced && backingOff("catalog")) return catalogue;
   if (!pending)
     pending = crawl()
       .catch((error) => {
+        failedAt.catalog = Date.now();
         status.catalogError = error.message;
         if (catalogue)
           return {
@@ -253,6 +269,9 @@ export async function getCatalog(refresh = false) {
       .finally(() => {
         pending = null;
       });
+  // Stale-while-revalidate: an outdated copy of this year answers at once and the crawl runs
+  // behind it. Only a first start, a new academic year or a forced refresh wait for the crawl.
+  if (thisYear && !forced) return catalogue;
   return pending;
 }
 export async function documentFor(id) {
@@ -324,15 +343,14 @@ async function readCalendarCache() {
 }
 export async function getCalendar(refresh = false) {
   await readCalendarCache();
-  if (
-    !refresh &&
-    calendar &&
-    Date.now() - new Date(calendar.updatedAt).getTime() < CALENDAR_TTL
-  )
-    return calendar;
+  const age = calendar ? ageOf(calendar.updatedAt) : Infinity;
+  const forced = refresh && age >= CALENDAR_MIN_REFRESH;
+  if (calendar && age < CALENDAR_TTL && !forced) return calendar;
+  if (calendar && !forced && backingOff("calendar")) return calendar;
   if (!calendarPending)
     calendarPending = loadCalendar()
       .catch((error) => {
+        failedAt.calendar = Date.now();
         status.calendarError = error.message;
         if (calendar) return { ...calendar, stale: true, error: error.message };
         throw error;
@@ -340,6 +358,8 @@ export async function getCalendar(refresh = false) {
       .finally(() => {
         calendarPending = null;
       });
+  // As with the catalog, a saved week table answers while the new one loads.
+  if (calendar && !forced) return calendar;
   return calendarPending;
 }
 

server/timetable.js

@@ -109,6 +109,5 @@ export function timetable({ file, schedule, calendar, group, from, days, range,
     days: out,
     weekly,
     notes: schedule.notes,
-    generatedAt: new Date().toISOString(),
   };
 }