feat: Implement caching and rate limiting for API requests

README.md

@@ -49,7 +49,9 @@ Vite работает на http://127.0.0.1:5173 и проксирует `/api`
 
 **Обновления у студента.** При запуске и при возвращении в приложение, если прошло больше часа, клиент сверяет сохранённое расписание с каталогом. Тот же файл с новой датой тихо перечитывается. Если для группы появился более новый документ (следующий семестр или год), показывается предложение его открыть; отказ запоминается. Если документ исчез из каталога, приложение предлагает найти группу заново.
 
-Каталог и папка года перечитываются раз в 15 минут, таблица недель и звонки — раз в 6 часов, распознанный документ — при изменении даты файла или версии парсера. «Обновить» принудительно перечитывает всё. Если источник недоступен, используются сохранённые копии с явной пометкой. Фоновая синхронизация при закрытом приложении не реализована.
+Каталог и папка года перечитываются раз в 15 минут, таблица недель и звонки — раз в 6 часов, распознанный документ — при изменении даты файла или версии парсера. Устаревшая копия отвечает сразу, а перечитывание идёт в фоне. Ждать приходится только при первом запуске, в начале нового учебного года и при «Обновить». «Обновить» (`?refresh=1`) принудительно перечитывает всё, но обращается к университету не чаще раза в 2 минуты (таблицу недель — раз в 10 минут), иначе отвечает только что прочитанной копией. После сбоя источника фоновые попытки тоже ждут 2 минуты. Если источник недоступен, используются сохранённые копии с явной пометкой. Фоновая синхронизация при закрытом приложении не реализована.
+
+Ответы API сжимаются (gzip или brotli; семестр группы — около 200 КБ JSON, в gzip — около 5 КБ) и отдаются с `Cache-Control: no-cache` и `ETag`. Клиент хранит ответ, но каждый раз спрашивает заново. Если ничего не изменилось, приходит пустой `304`. Android-клиент делает это сам через дисковый кэш OkHttp. Импорт Word ограничен: два разбора одновременно, ещё четыре ждут, остальным — `429`; с одного адреса — до 10 файлов за 10 минут. За прокси (Caddy, туннель) адрес клиента берётся из `X-Forwarded-For`, если прокси находится в локальной или частной сети.
 
 `GET /api/health` без обращения к сети показывает выбранный учебный год, возраст каталога, найденные файлы недель и звонков, семестры и предупреждения: папка нового года не найдена, сегодняшняя дата вне таблицы недель, источник не обновляется.
 

android/app/src/main/kotlin/org/xtimms/psu/schedule/ScheduleApp.kt

@@ -22,7 +22,7 @@ class ScheduleApp : Application() {
     override fun onCreate() {
         super.onCreate()
         settings = Settings(this)
-        repository = Repository(this, Api(BuildConfig.API_URL), settings)
+        repository = Repository(this, Api(BuildConfig.API_URL, cacheDir), settings)
         notes = NotesStore(this)
         attendance = AttendanceStore(this)
         Reminders.createChannel(this)

android/app/src/main/kotlin/org/xtimms/psu/schedule/data/Api.kt

@@ -6,10 +6,12 @@ import kotlinx.coroutines.withContext
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.jsonObject
 import kotlinx.serialization.json.jsonPrimitive
+import okhttp3.Cache
 import okhttp3.HttpUrl
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import okhttp3.Request
+import java.io.File
 import java.io.IOException
 import java.util.concurrent.TimeUnit
 
@@ -22,9 +24,12 @@ val json = Json {
 }
 
 /** The schedule server's HTTP API. Every error message is ready to show. */
-class Api(baseUrl: String) {
+class Api(baseUrl: String, cacheDir: File? = null) {
     private val base: HttpUrl = baseUrl.toHttpUrl()
     private val client = OkHttpClient.Builder()
+        // The server marks answers "no-cache" with an ETag: OkHttp keeps them here and asks
+        // again with If-None-Match, so an unchanged semester comes back as an empty 304.
+        .apply { if (cacheDir != null) cache(Cache(File(cacheDir, "http"), 10L * 1024 * 1024)) }
         .connectTimeout(10, TimeUnit.SECONDS)
         // The first request after a server start can wait for the catalog index.
         .readTimeout(60, TimeUnit.SECONDS)

android/app/src/main/kotlin/org/xtimms/psu/schedule/data/Models.kt

@@ -17,7 +17,6 @@ data class Timetable(
     val days: List<Day> = emptyList(),
     val weekly: List<WeeklyDay> = emptyList(),
     val notes: List<String> = emptyList(),
-    val generatedAt: String? = null,
 )
 
 @Serializable

package-lock.json

@@ -11,6 +11,7 @@
         "@material-symbols/svg-400": "^0.48.0",
         "adm-zip": "^0.5.16",
         "cheerio": "^1.1.2",
+        "compression": "^1.8.2",
         "express": "^5.1.0",
         "fast-xml-parser": "^5.3.0",
         "multer": "^2.0.2",
@@ -2530,6 +2531,65 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/compressible": {
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
+      "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": ">= 1.43.0 < 2"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/compression": {
+      "version": "1.8.2",
+      "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.2.tgz",
+      "integrity": "sha512-o8vI5RE5A6EVVOd9o41jKp41aJom+QTEO/Bx8MYNjexMo/Bv2WOjUfZr+aL0WnYSgymUy6zeguqLTsIhV0gMvQ==",
+      "license": "MIT",
+      "dependencies": {
+        "bytes": "3.1.2",
+        "compressible": "~2.0.18",
+        "debug": "2.6.9",
+        "destroy": "1.2.0",
+        "negotiator": "~0.6.4",
+        "on-headers": "~1.1.0",
+        "safe-buffer": "5.2.1",
+        "vary": "~1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/compression/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "2.0.0"
+      }
+    },
+    "node_modules/compression/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "license": "MIT"
+    },
+    "node_modules/compression/node_modules/negotiator": {
+      "version": "0.6.4",
+      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz",
+      "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/concurrently": {
       "version": "9.2.5",
       "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.5.tgz",
@@ -2694,6 +2754,16 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/destroy": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
+      "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8",
+        "npm": "1.2.8000 || >= 1.4.16"
+      }
+    },
     "node_modules/detect-libc": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
@@ -3656,6 +3726,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/on-headers": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz",
+      "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -4045,6 +4124,26 @@
         "tslib": "^2.1.0"
       }
     },
+    "node_modules/safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT"
+    },
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",

package.json

@@ -15,6 +15,7 @@
     "@material-symbols/svg-400": "^0.48.0",
     "adm-zip": "^0.5.16",
     "cheerio": "^1.1.2",
+    "compression": "^1.8.2",
     "express": "^5.1.0",
     "fast-xml-parser": "^5.3.0",
     "multer": "^2.0.2",

server/index.js

@@ -1,8 +1,10 @@
 import express from "express";
+import compression from "compression";
 import { fileURLToPath } from "node:url";
 import multer from "multer";
 import { parseWord, PARSER_VERSION } from "./parser.js";
 import { scheduleFor } from "./schedules.js";
+import { rateLimit, semaphore } from "./limits.js";
 import { timetable } from "./timetable.js";
 import {
   getIndex,
@@ -24,13 +26,21 @@ import {
 } from "./source.js";
 const app = express();
 app.disable("x-powered-by");
+// Behind Caddy or a tunnel on this host, the client's address comes from X-Forwarded-For;
+// only proxies on private networks are believed, so a client cannot forge its own.
+app.set("trust proxy", "loopback, linklocal, uniquelocal");
+// gzip or brotli: a semester of lessons is about 200 KB of JSON and shrinks tenfold.
+app.use(compression());
 app.use("/api", (_req, res, next) => {
-  res.set("Cache-Control", "no-store");
+  // Clients may keep a response but must ask again each time; Express's ETag turns an
+  // unchanged answer into an empty 304, so a phone re-downloads a timetable only when it changed.
+  res.set("Cache-Control", "no-cache");
   next();
 });
 // Raised whenever an endpoint is added or changed, so a deployment is easy to verify:
-// 2 — /api/timetable for the Android client; 3 — its range=semester.
+// 2 — /api/timetable for the Android client; 3 — its range=semester;
-const API_VERSION = 3;
+// 4 — compression, ETag revalidation, throttled refresh and import limits.
+const API_VERSION = 4;
 
 // Liveness plus what the server currently reads from the university, without network calls.
 app.get("/api/health", async (_req, res) => {
@@ -165,11 +175,23 @@ app.get("/api/original/:id", async (req, res, next) => {
     next(e);
   }
 });
+// Parsing runs a LibreOffice conversion for old .doc files: two at a time, a few more wait,
+// and each address may import ten files in ten minutes.
+const importLimit = rateLimit({
+  max: 10,
+  windowMs: 10 * 60 * 1000,
+  message: "Слишком много файлов подряд. Попробуйте через несколько минут.",
+});
+const parsing = semaphore({
+  limit: 2,
+  queue: 4,
+  message: "Сервер занят разбором других файлов. Попробуйте через минуту.",
+});
 const upload = multer({
   storage: multer.memoryStorage(),
   limits: { fileSize: 10 * 1024 * 1024, files: 1, fields: 0 },
 });
-app.post("/api/import", upload.single("file"), async (req, res, next) => {
+app.post("/api/import", importLimit, upload.single("file"), async (req, res, next) => {
   try {
     if (!req.file)
       return res.status(400).json({ error: "Выберите файл Word." });
@@ -179,9 +201,10 @@ app.post("/api/import", upload.single("file"), async (req, res, next) => {
       return res
         .status(400)
         .json({ error: "Поддерживаются файлы .docx и .doc." });
-    res.json(await parseWord(req.file.buffer, { name, uploaded: true }));
+    res.json(await parsing(() => parseWord(req.file.buffer, { name, uploaded: true })));
   } catch (e) {
-    e.status = 422;
+    // A file that cannot be read is the client's problem (422); a busy server keeps its 429.
+    e.status ??= 422;
     next(e);
   }
 });
@@ -200,11 +223,9 @@ app.use((error, _req, res, _next) => {
       error:
         error.code === "LIMIT_FILE_SIZE"
           ? "Файл больше 10 МБ. Выберите документ меньшего размера."
-          : error.status === 422
+          : [404, 422, 429].includes(error.status)
             ? error.message
-            : error.status === 404
+            : "Не удалось загрузить расписание. Проверьте подключение и попробуйте ещё раз.",
-              ? error.message
-              : "Не удалось загрузить расписание. Проверьте подключение и попробуйте ещё раз.",
     });
 });
 const port = Number(process.env.PORT || 3001);

server/limits.js

@@ -0,0 +1,42 @@
+/**
+ * Limits for the expensive endpoints: parsing an uploaded Word file can take a LibreOffice
+ * conversion and seconds of CPU, so one client must not be able to occupy the server.
+ */
+
+/** At most `max` requests per client address in each `windowMs`; the rest get 429. */
+export function rateLimit({ max, windowMs, message }) {
+  const windows = new Map();
+  return (req, res, next) => {
+    const now = Date.now();
+    // Finished windows are dropped on the way, so the map holds only active clients.
+    for (const [key, w] of windows) if (now - w.start >= windowMs) windows.delete(key);
+    const w = windows.get(req.ip) ?? { start: now, count: 0 };
+    w.count++;
+    windows.set(req.ip, w);
+    if (w.count <= max) return next();
+    res.set("Retry-After", String(Math.ceil((w.start + windowMs - now) / 1000)));
+    res.status(429).json({ error: message });
+  };
+}
+
+/**
+ * Runs at most `limit` tasks at once; up to `queue` more wait for a free slot,
+ * and beyond that a task is refused with a 429 error carrying `message`.
+ */
+export function semaphore({ limit, queue, message }) {
+  let active = 0;
+  const waiting = [];
+  return async function run(task) {
+    if (active < limit) active++;
+    else if (waiting.length < queue) await new Promise((resolve) => waiting.push(resolve));
+    else throw Object.assign(new Error(message), { status: 429 });
+    try {
+      return await task();
+    } finally {
+      // The slot passes straight to the next waiting task, or frees up.
+      const next = waiting.shift();
+      if (next) next();
+      else active--;
+    }
+  };
+}

server/schedules.js

@@ -1,13 +1,18 @@
 import { readFile, writeFile, mkdir } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import { parseWord, PARSER_VERSION } from "./parser.js";
-import { request } from "./source.js";
+import { request, MIN_REFRESH } from "./source.js";
 
 const data = fileURLToPath(new URL("../.data/parsed/", import.meta.url));
+// When each document was last downloaded, and the downloads under way: a forced refresh
+// downloads a document at most once per MIN_REFRESH, and simultaneous ones share a download.
+const fetchedAt = new Map(),
+  loading = new Map();
 
 /** A parsed catalog document, from the disk cache while the file and parser are unchanged. */
 export async function scheduleFor(file, { refresh = false } = {}) {
   const cache = data + file.id + ".json";
+  if (refresh && Date.now() - (fetchedAt.get(file.id) || 0) < MIN_REFRESH) refresh = false;
   if (!refresh)
     try {
       const cached = JSON.parse(await readFile(cache, "utf8"));
@@ -17,7 +22,17 @@ export async function scheduleFor(file, { refresh = false } = {}) {
       )
         return cached;
     } catch {}
+  if (!loading.has(file.id))
+    loading.set(
+      file.id,
+      download(file, cache).finally(() => loading.delete(file.id)),
+    );
+  return loading.get(file.id);
+}
+
+async function download(file, cache) {
   const buffer = await request(file.path);
+  fetchedAt.set(file.id, Date.now());
   const parsed = await parseWord(buffer, {
     name: file.name,
     url: file.url,

server/source.js

@@ -21,8 +21,22 @@ const arr = (value) => (value ? (Array.isArray(value) ? value : [value]) : []);
 const DISCOVERY_TTL = 15 * 60 * 1000,
   CATALOG_TTL = 15 * 60 * 1000,
   CALENDAR_TTL = 6 * 60 * 60 * 1000;
+// A forced refresh (?refresh=1, the PWA's "Обновить") reaches the university at most this often;
+// sooner, the copy just read is the answer. Anyone can send the request, and one catalog crawl
+// is up to 180 listings of the university's Nextcloud.
+export const MIN_REFRESH = 2 * 60 * 1000;
+const CALENDAR_MIN_REFRESH = 10 * 60 * 1000;
+/** Milliseconds since an ISO time; Infinity for a missing or broken one. */
+const ageOf = (iso) => {
+  const age = Date.now() - new Date(iso).getTime();
+  return Number.isFinite(age) ? age : Infinity;
+};
 let catalogue, pending, calendar, calendarPending, sources, sourcesPending;
 const status = { catalogError: null, calendarError: null };
+// When the university last failed to answer: background refreshes back off for MIN_REFRESH,
+// so a down source is not crawled again on every request.
+const failedAt = { catalog: 0, calendar: 0 };
+const backingOff = (what) => Date.now() - failedAt[what] < MIN_REFRESH;
 
 export async function request(path, method = "GET") {
   // Only the university share is reachable; listings and catalog entries supply the paths.
@@ -232,15 +246,17 @@ export async function getCatalog(refresh = false) {
     const y = academicStartYear(todayIso());
     return `${y}–${y + 1}`;
   })();
+  const age = catalogue ? ageOf(catalogue.updatedAt) : Infinity;
   // A cached catalog of last year is refreshed as soon as the new year begins.
-  const fresh =
+  const thisYear =
-    catalogue &&
+    catalogue && (catalogue.year === expected || catalogue.currentYear === false);
-    Date.now() - new Date(catalogue.updatedAt).getTime() < CATALOG_TTL &&
+  const forced = refresh && age >= MIN_REFRESH;
-    (catalogue.year === expected || catalogue.currentYear === false);
+  if (thisYear && age < CATALOG_TTL && !forced) return catalogue;
-  if (!refresh && fresh) return catalogue;
+  if (thisYear && !forced && backingOff("catalog")) return catalogue;
   if (!pending)
     pending = crawl()
       .catch((error) => {
+        failedAt.catalog = Date.now();
         status.catalogError = error.message;
         if (catalogue)
           return {
@@ -253,6 +269,9 @@ export async function getCatalog(refresh = false) {
       .finally(() => {
         pending = null;
       });
+  // Stale-while-revalidate: an outdated copy of this year answers at once and the crawl runs
+  // behind it. Only a first start, a new academic year or a forced refresh wait for the crawl.
+  if (thisYear && !forced) return catalogue;
   return pending;
 }
 export async function documentFor(id) {
@@ -324,15 +343,14 @@ async function readCalendarCache() {
 }
 export async function getCalendar(refresh = false) {
   await readCalendarCache();
-  if (
+  const age = calendar ? ageOf(calendar.updatedAt) : Infinity;
-    !refresh &&
+  const forced = refresh && age >= CALENDAR_MIN_REFRESH;
-    calendar &&
+  if (calendar && age < CALENDAR_TTL && !forced) return calendar;
-    Date.now() - new Date(calendar.updatedAt).getTime() < CALENDAR_TTL
+  if (calendar && !forced && backingOff("calendar")) return calendar;
-  )
-    return calendar;
   if (!calendarPending)
     calendarPending = loadCalendar()
       .catch((error) => {
+        failedAt.calendar = Date.now();
         status.calendarError = error.message;
         if (calendar) return { ...calendar, stale: true, error: error.message };
         throw error;
@@ -340,6 +358,8 @@ export async function getCalendar(refresh = false) {
       .finally(() => {
         calendarPending = null;
       });
+  // As with the catalog, a saved week table answers while the new one loads.
+  if (calendar && !forced) return calendar;
   return calendarPending;
 }
 

server/timetable.js

@@ -109,6 +109,5 @@ export function timetable({ file, schedule, calendar, group, from, days, range,
     days: out,
     weekly,
     notes: schedule.notes,
-    generatedAt: new Date().toISOString(),
   };
 }

test/limits.test.js

@@ -0,0 +1,52 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { rateLimit, semaphore } from "../server/limits.js";
+
+test("Импорт: не больше двух разборов сразу, лишние ждут или получают 429", async () => {
+  const run = semaphore({ limit: 2, queue: 1, message: "Занято" });
+  let active = 0,
+    peak = 0;
+  const release = [];
+  const task = () =>
+    run(async () => {
+      peak = Math.max(peak, ++active);
+      await new Promise((resolve) => release.push(resolve));
+      active--;
+    });
+  const running = [task(), task(), task()];
+  await assert.rejects(task(), (e) => e.status === 429 && e.message === "Занято");
+  // Each finished task hands its slot to the one waiting.
+  while (release.length) {
+    release.shift()();
+    await new Promise((resolve) => setImmediate(resolve));
+  }
+  await Promise.all(running);
+  assert.equal(peak, 2);
+  // All slots are free again.
+  assert.equal(await run(async () => "ok"), "ok");
+});
+
+test("Импорт: лимит запросов считается по адресу клиента", () => {
+  const limit = rateLimit({ max: 2, windowMs: 60_000, message: "Подождите" });
+  const call = (ip) => {
+    let status = 200,
+      body = null,
+      passed = false;
+    const res = {
+      set() {},
+      status(code) {
+        status = code;
+        return this;
+      },
+      json(value) {
+        body = value;
+      },
+    };
+    limit({ ip }, res, () => (passed = true));
+    return { status, body, passed };
+  };
+  assert.ok(call("10.0.0.1").passed);
+  assert.ok(call("10.0.0.1").passed);
+  assert.deepEqual(call("10.0.0.1"), { status: 429, body: { error: "Подождите" }, passed: false });
+  assert.ok(call("10.0.0.2").passed);
+});