feat: add schedule import functionality and quality control page
- Implemented the schedule import feature with a dedicated page for importing schedules from the transport website. - Added a new QualityPage to check the quality of routes, stops, schedules, and notifications without making automatic changes. - Created CSS styles for both the ScheduleImportPage and QualityPage to enhance the user interface. - Introduced API calls for fetching schedule data and quality reports. - Added error handling and loading states for better user experience during data fetching. - Included functionality for filtering quality issues and displaying detailed reports.
This commit is contained in:
@@ -96,6 +96,24 @@ function parseAlertSnapshot(value) {
|
||||
return value;
|
||||
}
|
||||
|
||||
function isAlertDue(alert, now = new Date()) {
|
||||
if (alert.is_active === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const startTime = new Date(alert.start_time);
|
||||
if (Number.isNaN(startTime.getTime()) || startTime > now) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!alert.end_time) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const endTime = new Date(alert.end_time);
|
||||
return !Number.isNaN(endTime.getTime()) && endTime >= now;
|
||||
}
|
||||
|
||||
class AlertCreationService {
|
||||
constructor({ pool, notificationService, logger }) {
|
||||
this.pool = pool;
|
||||
@@ -110,7 +128,7 @@ class AlertCreationService {
|
||||
? await this.createIdempotentAlert(alertInput, idempotencyKey)
|
||||
: await this.insertAlert(this.pool, alertInput);
|
||||
|
||||
if (result.created && result.alert.is_active !== false) {
|
||||
if (result.created && isAlertDue(result.alert)) {
|
||||
await this.sendNotification(result.notificationAlert);
|
||||
}
|
||||
|
||||
@@ -263,5 +281,6 @@ module.exports = {
|
||||
normalizeAlertInput,
|
||||
normalizeIdempotencyKey,
|
||||
parseAlertSnapshot,
|
||||
isAlertDue,
|
||||
setAlertCreationServiceForTesting
|
||||
};
|
||||
|
||||
109
backend/src/services/alertSchedulerService.js
Normal file
109
backend/src/services/alertSchedulerService.js
Normal file
@@ -0,0 +1,109 @@
|
||||
const CLAIM_DUE_ALERTS_SQL = `
|
||||
WITH due AS (
|
||||
SELECT a.id
|
||||
FROM alerts a
|
||||
WHERE a.is_active = true
|
||||
AND a.start_time > a.created_at
|
||||
AND a.start_time <= CURRENT_TIMESTAMP
|
||||
AND (a.end_time IS NULL OR a.end_time >= CURRENT_TIMESTAMP)
|
||||
AND a.notification_sent_at IS NULL
|
||||
AND a.notification_attempts < 5
|
||||
AND (
|
||||
a.notification_claimed_at IS NULL
|
||||
OR a.notification_claimed_at < CURRENT_TIMESTAMP - INTERVAL '5 minutes'
|
||||
)
|
||||
ORDER BY a.start_time ASC, a.id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE alerts a
|
||||
SET notification_claimed_at = CURRENT_TIMESTAMP,
|
||||
notification_attempts = a.notification_attempts + 1
|
||||
FROM due
|
||||
WHERE a.id = due.id
|
||||
RETURNING a.*, (
|
||||
SELECT r.route_number
|
||||
FROM routes r
|
||||
WHERE r.id = a.route_id
|
||||
) AS notification_route_number
|
||||
`;
|
||||
|
||||
class AlertSchedulerService {
|
||||
constructor({ pool, notificationService, logger, batchSize = 50 } = {}) {
|
||||
this.pool = pool || require('../config/database');
|
||||
this.notificationService = notificationService || require('./alertNotificationService').getAlertNotificationService();
|
||||
this.logger = logger || require('../config/logger');
|
||||
this.batchSize = batchSize;
|
||||
this.running = false;
|
||||
}
|
||||
|
||||
async dispatchDueAlerts() {
|
||||
if (this.running || !this.notificationService.enabled) {
|
||||
return { dispatched: 0, skipped: this.running ? 'running' : 'disabled' };
|
||||
}
|
||||
|
||||
this.running = true;
|
||||
try {
|
||||
const claimed = await this.pool.query(CLAIM_DUE_ALERTS_SQL, [this.batchSize]);
|
||||
let dispatched = 0;
|
||||
|
||||
for (const row of claimed.rows) {
|
||||
const { notification_route_number: routeNumber, ...alert } = row;
|
||||
try {
|
||||
const result = await this.notificationService.sendAlertCreated({
|
||||
...alert,
|
||||
route_number: routeNumber || ''
|
||||
});
|
||||
|
||||
if (result.sent) {
|
||||
await this.pool.query(
|
||||
`UPDATE alerts
|
||||
SET notification_sent_at = CURRENT_TIMESTAMP,
|
||||
notification_claimed_at = NULL
|
||||
WHERE id = $1`,
|
||||
[alert.id]
|
||||
);
|
||||
dispatched += 1;
|
||||
} else {
|
||||
await this.releaseClaim(alert.id);
|
||||
}
|
||||
} catch (error) {
|
||||
await this.releaseClaim(alert.id);
|
||||
this.logger.error('Failed to dispatch scheduled alert notification', {
|
||||
alertId: String(alert.id),
|
||||
reason: error.message || 'Unknown notification error',
|
||||
firebaseCode: error.code || 'unknown'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { dispatched, claimed: claimed.rows.length };
|
||||
} finally {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
|
||||
async releaseClaim(alertId) {
|
||||
await this.pool.query(
|
||||
`UPDATE alerts
|
||||
SET notification_claimed_at = NULL
|
||||
WHERE id = $1 AND notification_sent_at IS NULL`,
|
||||
[alertId]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function startAlertScheduler({ intervalMs = 30_000, service } = {}) {
|
||||
const scheduler = service || new AlertSchedulerService();
|
||||
const run = () => scheduler.dispatchDueAlerts().catch((error) => {
|
||||
scheduler.logger.error('Scheduled alert delivery cycle failed', {
|
||||
reason: error.message || 'Unknown scheduler error'
|
||||
});
|
||||
});
|
||||
run();
|
||||
const timer = setInterval(run, intervalMs);
|
||||
timer.unref?.();
|
||||
return () => clearInterval(timer);
|
||||
}
|
||||
|
||||
module.exports = { AlertSchedulerService, CLAIM_DUE_ALERTS_SQL, startAlertScheduler };
|
||||
40
backend/src/services/alertTemplates.js
Normal file
40
backend/src/services/alertTemplates.js
Normal file
@@ -0,0 +1,40 @@
|
||||
const ALERT_TEMPLATES = Object.freeze([
|
||||
Object.freeze({
|
||||
id: 'delay',
|
||||
label: 'Задержка',
|
||||
description: 'Сообщить о задержке движения',
|
||||
alert_type: 'delay',
|
||||
title: 'Задержка движения',
|
||||
message: 'На маршруте возможна задержка до 15 минут. Приносим извинения за неудобства.'
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'cancellation',
|
||||
label: 'Отмена рейса',
|
||||
description: 'Сообщить об отмене или переносе',
|
||||
alert_type: 'cancellation',
|
||||
title: 'Отмена рейса',
|
||||
message: 'Один из рейсов временно отменён. Пожалуйста, выберите ближайшее подходящее отправление.'
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'detour',
|
||||
label: 'Изменение маршрута',
|
||||
description: 'Предупредить об объезде',
|
||||
alert_type: 'detour',
|
||||
title: 'Временное изменение маршрута',
|
||||
message: 'Маршрут временно следует по изменённой схеме. Уточните детали ниже перед публикацией.'
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'info',
|
||||
label: 'Информация',
|
||||
description: 'Нейтральное сообщение пассажирам',
|
||||
alert_type: 'info',
|
||||
title: 'Информация для пассажиров',
|
||||
message: 'Пожалуйста, ознакомьтесь с важной информацией о работе транспорта.'
|
||||
})
|
||||
]);
|
||||
|
||||
function listAlertTemplates() {
|
||||
return ALERT_TEMPLATES.map((template) => ({ ...template }));
|
||||
}
|
||||
|
||||
module.exports = { ALERT_TEMPLATES, listAlertTemplates };
|
||||
285
backend/src/services/qualityReport.js
Normal file
285
backend/src/services/qualityReport.js
Normal file
@@ -0,0 +1,285 @@
|
||||
const DAY_TYPES = [
|
||||
{ value: 'weekday', label: 'будних дней' },
|
||||
{ value: 'saturday', label: 'субботы' },
|
||||
{ value: 'sunday', label: 'воскресенья' },
|
||||
];
|
||||
|
||||
const SEVERITY_ORDER = { critical: 0, warning: 1, info: 2 };
|
||||
|
||||
const qualityChecks = [
|
||||
{
|
||||
name: 'routesWithoutStops',
|
||||
query: `
|
||||
SELECT r.id AS route_id, r.route_number, r.name AS route_name,
|
||||
COUNT(s.id)::integer AS active_stops_count
|
||||
FROM routes r
|
||||
LEFT JOIN route_stops rs ON rs.route_id = r.id
|
||||
LEFT JOIN stops s ON s.id = rs.stop_id AND s.is_active = true
|
||||
WHERE r.is_active = true
|
||||
GROUP BY r.id
|
||||
HAVING COUNT(s.id) < 2
|
||||
ORDER BY r.route_number, r.name
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'missingSchedules',
|
||||
query: `
|
||||
SELECT r.id AS route_id, r.route_number, r.name AS route_name, day_types.day_type
|
||||
FROM routes r
|
||||
CROSS JOIN (VALUES ('weekday'), ('saturday'), ('sunday')) AS day_types(day_type)
|
||||
LEFT JOIN schedules s ON s.route_id = r.id
|
||||
AND s.day_type = day_types.day_type
|
||||
AND (s.valid_from IS NULL OR s.valid_from <= CURRENT_DATE)
|
||||
AND (s.valid_until IS NULL OR s.valid_until >= CURRENT_DATE)
|
||||
WHERE r.is_active = true
|
||||
GROUP BY r.id, r.route_number, r.name, day_types.day_type
|
||||
HAVING COUNT(s.id) = 0
|
||||
ORDER BY r.route_number, r.name, day_types.day_type
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'inactiveRouteStops',
|
||||
query: `
|
||||
SELECT r.id AS route_id, r.route_number, r.name AS route_name,
|
||||
s.id AS stop_id, s.name AS stop_name
|
||||
FROM route_stops rs
|
||||
JOIN routes r ON r.id = rs.route_id AND r.is_active = true
|
||||
JOIN stops s ON s.id = rs.stop_id AND s.is_active = false
|
||||
ORDER BY r.route_number, r.name, rs.sequence
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'invalidStopOrder',
|
||||
query: `
|
||||
SELECT r.id AS route_id, r.route_number, r.name AS route_name,
|
||||
COUNT(*)::integer AS stops_count, MIN(rs.sequence)::integer AS first_sequence,
|
||||
MAX(rs.sequence)::integer AS last_sequence
|
||||
FROM routes r
|
||||
JOIN route_stops rs ON rs.route_id = r.id
|
||||
WHERE r.is_active = true
|
||||
GROUP BY r.id
|
||||
HAVING MIN(rs.sequence) <> 1 OR MAX(rs.sequence) <> COUNT(*) OR COUNT(DISTINCT rs.sequence) <> COUNT(*)
|
||||
ORDER BY r.route_number, r.name
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'invalidTimeOffsets',
|
||||
query: `
|
||||
SELECT r.id AS route_id, r.route_number, r.name AS route_name,
|
||||
s.id AS stop_id, s.name AS stop_name, rs.sequence, rs.time_offset_minutes
|
||||
FROM route_stops rs
|
||||
JOIN routes r ON r.id = rs.route_id AND r.is_active = true
|
||||
JOIN stops s ON s.id = rs.stop_id
|
||||
WHERE rs.time_offset_minutes < 0
|
||||
OR (rs.sequence = 1 AND rs.time_offset_minutes <> 0)
|
||||
ORDER BY r.route_number, r.name, rs.sequence
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'overlappingSchedules',
|
||||
query: `
|
||||
SELECT newer.route_id, r.route_number, r.name AS route_name, newer.day_type,
|
||||
newer.id AS schedule_id, older.id AS conflicting_schedule_id
|
||||
FROM schedules newer
|
||||
JOIN schedules older ON older.route_id = newer.route_id
|
||||
AND older.day_type = newer.day_type
|
||||
AND older.id < newer.id
|
||||
AND daterange(
|
||||
COALESCE(older.valid_from, '-infinity'::date),
|
||||
COALESCE(older.valid_until, 'infinity'::date),
|
||||
'[]'
|
||||
) && daterange(
|
||||
COALESCE(newer.valid_from, '-infinity'::date),
|
||||
COALESCE(newer.valid_until, 'infinity'::date),
|
||||
'[]'
|
||||
)
|
||||
JOIN routes r ON r.id = newer.route_id AND r.is_active = true
|
||||
ORDER BY r.route_number, r.name, newer.day_type, newer.id
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'invalidSchedulePeriods',
|
||||
query: `
|
||||
SELECT s.id AS schedule_id, s.route_id, r.route_number, r.name AS route_name, s.day_type
|
||||
FROM schedules s
|
||||
JOIN routes r ON r.id = s.route_id AND r.is_active = true
|
||||
WHERE s.valid_from IS NOT NULL
|
||||
AND s.valid_until IS NOT NULL
|
||||
AND s.valid_until < s.valid_from
|
||||
ORDER BY r.route_number, r.name, s.day_type, s.id
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'emptySchedules',
|
||||
query: `
|
||||
SELECT s.id AS schedule_id, s.route_id, r.route_number, r.name AS route_name, s.day_type
|
||||
FROM schedules s
|
||||
JOIN routes r ON r.id = s.route_id AND r.is_active = true
|
||||
WHERE COALESCE(cardinality(s.departure_times), 0) = 0
|
||||
AND COALESCE(jsonb_array_length(s.trips), 0) = 0
|
||||
ORDER BY r.route_number, r.name, s.day_type, s.id
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'unusedStops',
|
||||
query: `
|
||||
SELECT s.id AS stop_id, s.name AS stop_name
|
||||
FROM stops s
|
||||
WHERE s.is_active = true
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM route_stops rs
|
||||
JOIN routes r ON r.id = rs.route_id AND r.is_active = true
|
||||
WHERE rs.stop_id = s.id
|
||||
)
|
||||
ORDER BY s.name
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: 'expiredAlerts',
|
||||
query: `
|
||||
SELECT a.id AS alert_id, a.title
|
||||
FROM alerts a
|
||||
WHERE a.is_active = true
|
||||
AND a.end_time IS NOT NULL
|
||||
AND a.end_time < CURRENT_TIMESTAMP
|
||||
ORDER BY a.end_time DESC
|
||||
`,
|
||||
},
|
||||
];
|
||||
|
||||
const dayTypeLabel = (value) => DAY_TYPES.find((dayType) => dayType.value === value)?.label || value;
|
||||
|
||||
function routeLabel(row) {
|
||||
return `№${row.route_number} · ${row.route_name}`;
|
||||
}
|
||||
|
||||
function routeAction(routeId, label) {
|
||||
return { to: `/routes/${routeId}/edit`, label };
|
||||
}
|
||||
|
||||
function scheduleAction(routeId) {
|
||||
return { to: `/schedules?route=${routeId}`, label: 'Открыть расписания' };
|
||||
}
|
||||
|
||||
function createIssue({ code, severity, title, description, entity, action }) {
|
||||
return { id: `${code}:${entity.type}:${entity.id}`, code, severity, title, description, entity, action };
|
||||
}
|
||||
|
||||
function toIssues(rowsByCheck) {
|
||||
const issues = [];
|
||||
|
||||
for (const row of rowsByCheck.routesWithoutStops || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'route_without_stops', severity: 'critical', title: 'Маршрут не готов к работе',
|
||||
description: `${routeLabel(row)}: активных остановок — ${row.active_stops_count}. Для направления нужны как минимум две.`,
|
||||
entity: { type: 'route', id: row.route_id, label: routeLabel(row) },
|
||||
action: routeAction(row.route_id, 'Настроить остановки'),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.missingSchedules || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'missing_schedule', severity: 'critical', title: 'Нет действующего расписания',
|
||||
description: `${routeLabel(row)}: не задана версия для ${dayTypeLabel(row.day_type)}.`,
|
||||
entity: { type: 'route-day-type', id: `${row.route_id}:${row.day_type}`, label: routeLabel(row) },
|
||||
action: scheduleAction(row.route_id),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.inactiveRouteStops || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'inactive_stop_in_route', severity: 'warning', title: 'В маршруте неактивная остановка',
|
||||
description: `${routeLabel(row)}: «${row.stop_name}» исключена из данных, которые получают пассажиры.`,
|
||||
entity: { type: 'route-stop', id: `${row.route_id}:${row.stop_id}`, label: row.stop_name },
|
||||
action: routeAction(row.route_id, 'Проверить маршрут'),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.invalidStopOrder || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'invalid_stop_order', severity: 'warning', title: 'Нарушен порядок остановок',
|
||||
description: `${routeLabel(row)}: порядковые номера должны идти без пропусков от 1 до ${row.stops_count}.`,
|
||||
entity: { type: 'route', id: row.route_id, label: routeLabel(row) },
|
||||
action: routeAction(row.route_id, 'Исправить порядок'),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.invalidTimeOffsets || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'invalid_time_offset', severity: 'warning', title: 'Проверьте время проезда',
|
||||
description: `${routeLabel(row)}: у остановки «${row.stop_name}» задано смещение ${row.time_offset_minutes} мин. Первая остановка должна иметь 0, остальные — неотрицательное значение.`,
|
||||
entity: { type: 'route-stop', id: `${row.route_id}:${row.stop_id}:${row.sequence}`, label: row.stop_name },
|
||||
action: routeAction(row.route_id, 'Исправить время'),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.overlappingSchedules || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'overlapping_schedules', severity: 'critical', title: 'Пересекаются версии расписания',
|
||||
description: `${routeLabel(row)}: версии №${row.schedule_id} и №${row.conflicting_schedule_id} одновременно действуют для ${dayTypeLabel(row.day_type)}.`,
|
||||
entity: { type: 'schedule-pair', id: `${row.schedule_id}:${row.conflicting_schedule_id}`, label: routeLabel(row) },
|
||||
action: scheduleAction(row.route_id),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.invalidSchedulePeriods || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'invalid_schedule_period', severity: 'critical', title: 'Некорректный период расписания',
|
||||
description: `${routeLabel(row)}: у версии №${row.schedule_id} дата окончания раньше даты начала.`,
|
||||
entity: { type: 'schedule', id: row.schedule_id, label: routeLabel(row) },
|
||||
action: scheduleAction(row.route_id),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.emptySchedules || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'empty_schedule', severity: 'critical', title: 'Пустое расписание',
|
||||
description: `${routeLabel(row)}: в версии №${row.schedule_id} для ${dayTypeLabel(row.day_type)} нет ни одного рейса.`,
|
||||
entity: { type: 'schedule', id: row.schedule_id, label: routeLabel(row) },
|
||||
action: scheduleAction(row.route_id),
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.unusedStops || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'unused_stop', severity: 'info', title: 'Остановка не используется',
|
||||
description: `«${row.stop_name}» не привязана ни к одному активному маршруту.`,
|
||||
entity: { type: 'stop', id: row.stop_id, label: row.stop_name },
|
||||
action: { to: '/stops', label: 'Открыть остановки' },
|
||||
}));
|
||||
}
|
||||
|
||||
for (const row of rowsByCheck.expiredAlerts || []) {
|
||||
issues.push(createIssue({
|
||||
code: 'expired_alert', severity: 'warning', title: 'Истёкшее уведомление активно',
|
||||
description: `«${row.title}» уже завершилось, но продолжает быть помечено активным.`,
|
||||
entity: { type: 'alert', id: row.alert_id, label: row.title },
|
||||
action: { to: '/alerts', label: 'Открыть уведомления' },
|
||||
}));
|
||||
}
|
||||
|
||||
return issues.sort((left, right) =>
|
||||
SEVERITY_ORDER[left.severity] - SEVERITY_ORDER[right.severity]
|
||||
|| left.title.localeCompare(right.title, 'ru')
|
||||
|| left.id.localeCompare(right.id)
|
||||
);
|
||||
}
|
||||
|
||||
function buildQualityReport(rowsByCheck, checkedAt = new Date().toISOString()) {
|
||||
const issues = toIssues(rowsByCheck);
|
||||
const summary = issues.reduce((result, issue) => {
|
||||
result[issue.severity] += 1;
|
||||
result.total += 1;
|
||||
return result;
|
||||
}, { total: 0, critical: 0, warning: 0, info: 0 });
|
||||
|
||||
return { checked_at: checkedAt, summary, issues };
|
||||
}
|
||||
|
||||
async function runQualityChecks(pool) {
|
||||
const results = await Promise.all(qualityChecks.map(async ({ name, query }) => [name, (await pool.query(query)).rows]));
|
||||
return buildQualityReport(Object.fromEntries(results));
|
||||
}
|
||||
|
||||
module.exports = { buildQualityReport, qualityChecks, runQualityChecks };
|
||||
300
backend/src/services/scheduleImportService.js
Normal file
300
backend/src/services/scheduleImportService.js
Normal file
@@ -0,0 +1,300 @@
|
||||
const { randomUUID } = require('node:crypto');
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { getLocalDateTime } = require('../utils/etaTime');
|
||||
const { HOME_URL, CATALOG_URL, hash, normalizeName, normalizeNumber, sourceError,
|
||||
safeUrl, fetchHtml, parseCatalog, parseSchedule, parseNews } = require('./scheduleSource');
|
||||
|
||||
const DAY_LABELS = { weekday: 'Будние дни', saturday: 'Суббота', sunday: 'Воскресенье' };
|
||||
const tripKey = trip => JSON.stringify({
|
||||
departureTime: trip.departureTime?.length === 5 ? `${trip.departureTime}:00` : trip.departureTime,
|
||||
startStopId: trip.startStopId || null, endStopId: trip.endStopId || null,
|
||||
viaStopIds: trip.viaStopIds || [], daysOfWeek: trip.daysOfWeek || [],
|
||||
exceptDaysOfWeek: trip.exceptDaysOfWeek || [], note: trip.note || ''
|
||||
});
|
||||
const blankTrip = (time, startStopId = null, endStopId = null) => ({
|
||||
departureTime: time, startStopId, endStopId, viaStopIds: [], daysOfWeek: [], exceptDaysOfWeek: [], note: ''
|
||||
});
|
||||
|
||||
function compareTrips(before, after) {
|
||||
// Multisets preserve duplicate departures in legacy data in the removal preview.
|
||||
const remaining = before.map(trip => ({ trip, key: tripKey(trip) }));
|
||||
const added = [];
|
||||
let unchanged = 0;
|
||||
for (const trip of after) {
|
||||
const index = remaining.findIndex(item => item.key === tripKey(trip));
|
||||
if (index === -1) added.push(trip);
|
||||
else { remaining.splice(index, 1); unchanged += 1; }
|
||||
}
|
||||
return { added, removed: remaining.map(item => item.trip), unchanged };
|
||||
}
|
||||
|
||||
function buildPlan(source, mappings, snapshot, validFrom, today = getLocalDateTime().date) {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(validFrom || '')
|
||||
|| Number.isNaN(Date.parse(validFrom)) || new Date(validFrom).toISOString().slice(0, 10) !== validFrom
|
||||
|| validFrom < today) throw sourceError('Выберите дату начала не раньше сегодняшнего дня.');
|
||||
if (source.warnings.length) throw sourceError('В источнике есть нераспознанные условия. Автоматический импорт заблокирован.');
|
||||
if (normalizeNumber(source.number) !== normalizeNumber(snapshot.route.route_number)) {
|
||||
throw sourceError('Номер маршрута в базе не совпадает с номером на сайте.');
|
||||
}
|
||||
const stops = new Set(snapshot.stops.map(stop => Number(stop.stop_id)));
|
||||
const groups = {};
|
||||
const notices = new Set();
|
||||
for (const row of source.rows) {
|
||||
if (!mappings || !Object.hasOwn(mappings, row.stopName)) throw sourceError(`Укажите назначение строки «${row.stopName}».`);
|
||||
const mapping = mappings[row.stopName];
|
||||
for (const day of row.days) groups[day] ||= [];
|
||||
if (mapping?.mode === 'checkpoint') continue;
|
||||
if (mapping?.mode !== 'departure') throw sourceError(`Укажите назначение строки «${row.stopName}».`);
|
||||
const start = Number(mapping.startStopId);
|
||||
const end = Number(mapping.endStopId);
|
||||
if (!stops.has(start) || !stops.has(end) || start === end) {
|
||||
throw sourceError(`Для «${row.stopName}» выберите разные начальную и конечную остановки этого маршрута.`);
|
||||
}
|
||||
const departures = row.departures || row.times.map(departureTime => ({ departureTime }));
|
||||
for (const departure of departures) {
|
||||
const trip = { ...blankTrip(departure.departureTime, start, end),
|
||||
daysOfWeek: departure.daysOfWeek || [], exceptDaysOfWeek: departure.exceptDaysOfWeek || [] };
|
||||
const unresolved = [];
|
||||
for (const name of departure.viaStopNames || []) {
|
||||
const matches = snapshot.stops.filter(stop => normalizeName(stop.name) === normalizeName(name));
|
||||
if (matches.length === 1) trip.viaStopIds.push(Number(matches[0].stop_id));
|
||||
else {
|
||||
unresolved.push(name);
|
||||
notices.add(`«${name}»: ${matches.length ? 'найдено несколько остановок с этим именем' : 'нет в остановках направления'}. Условие «через» сохранится текстовым примечанием без привязки к остановке.`);
|
||||
}
|
||||
}
|
||||
trip.note = unresolved.map(name => `через «${name}»`).join('; ');
|
||||
let included = false;
|
||||
for (const day of row.days) {
|
||||
const activeDays = { weekday: [1, 2, 3, 4, 5], saturday: [6], sunday: [7] }[day];
|
||||
if (!activeDays.some(value => (!trip.daysOfWeek.length || trip.daysOfWeek.includes(value)) && !trip.exceptDaysOfWeek.includes(value))) continue;
|
||||
groups[day].push(trip);
|
||||
included = true;
|
||||
}
|
||||
if (!included) throw sourceError(`${departure.departureTime.slice(0, 5)}: условие по дням противоречит разделу «${row.days.map(day => DAY_LABELS[day]).join(', ')}». Проверьте источник.`);
|
||||
}
|
||||
}
|
||||
const changes = [];
|
||||
for (const [dayType, trips] of Object.entries(groups)) {
|
||||
if (!trips.length) throw sourceError(`${DAY_LABELS[dayType]}: выберите хотя бы одну строку отправлений.`);
|
||||
if (trips.length > 2000 || new Set(trips.map(tripKey)).size !== trips.length) {
|
||||
throw sourceError('После сопоставления появились повторяющиеся рейсы или слишком много отправлений.');
|
||||
}
|
||||
trips.sort((a, b) => a.departureTime.localeCompare(b.departureTime) || a.startStopId - b.startStopId);
|
||||
const candidates = snapshot.schedules.filter(s => s.day_type === dayType && (!s.valid_until || s.valid_until >= validFrom));
|
||||
if (candidates.length > 1 || candidates.some(s => s.valid_from && s.valid_from > validFrom)) {
|
||||
throw sourceError(`${DAY_LABELS[dayType]}: есть пересекающиеся или будущие расписания. Сначала проверьте их в разделе «Расписания».`);
|
||||
}
|
||||
const current = candidates[0];
|
||||
const before = current?.trips?.length ? current.trips : (current?.departure_times || []).map(time => blankTrip(time));
|
||||
changes.push({ dayType, label: DAY_LABELS[dayType], existingId: current?.id || null,
|
||||
previousValidFrom: current?.valid_from || null, validFrom, validUntil: current?.valid_until || null,
|
||||
before, after: trips, ...compareTrips(before, trips) });
|
||||
}
|
||||
return { changes, mappings, validFrom, notices: [...notices], before: snapshot, beforeHash: hash(snapshot),
|
||||
hasChanges: changes.some(change => change.added.length || change.removed.length) };
|
||||
}
|
||||
|
||||
function departureStops(source) {
|
||||
const names = [...new Set(source.rows.map(row => row.stopName))];
|
||||
const origins = names.filter(name => source.directions.some(direction => normalizeName(direction.stops[0]) === normalizeName(name)));
|
||||
return origins.length ? origins : names;
|
||||
}
|
||||
|
||||
function buildDirectionPlan(source, sourceStopName, snapshot, validFrom, today) {
|
||||
if (!departureStops(source).includes(sourceStopName)) throw sourceError('Выберите строку отправлений для каждого направления.');
|
||||
const first = snapshot.stops[0];
|
||||
const last = snapshot.stops.at(-1);
|
||||
if (!first || !last || first.stop_id === last.stop_id) {
|
||||
throw sourceError(`В маршруте «${snapshot.route.name}» должны быть заданы разные первая и последняя остановки.`);
|
||||
}
|
||||
const rows = source.rows.filter(row => row.stopName === sourceStopName);
|
||||
const mappings = { [sourceStopName]: { mode: 'departure', startStopId: first.stop_id, endStopId: last.stop_id } };
|
||||
return { ...buildPlan({ ...source, rows }, mappings, snapshot, validFrom, today),
|
||||
routeId: snapshot.route.id, route: snapshot.route, stops: snapshot.stops, sourceStopName };
|
||||
}
|
||||
|
||||
class ScheduleImportService {
|
||||
constructor({ pool, load = fetchHtml, invalidate = async () => {} }) {
|
||||
this.pool = pool;
|
||||
this.load = load;
|
||||
this.invalidate = invalidate;
|
||||
this.schemaPromise = null;
|
||||
this.catalogCache = null;
|
||||
}
|
||||
|
||||
ensureSchema() {
|
||||
this.schemaPromise ||= fs.readFile(path.join(__dirname, '../../migrations/008_schedule_imports.sql'), 'utf8')
|
||||
.then(sql => this.pool.query(sql)).catch(error => { this.schemaPromise = null; throw error; });
|
||||
return this.schemaPromise;
|
||||
}
|
||||
|
||||
async catalog() {
|
||||
if (this.catalogCache && this.catalogCache.expires > Date.now()) return this.catalogCache.value;
|
||||
const page = await this.load(CATALOG_URL);
|
||||
const value = parseCatalog(page.html);
|
||||
this.catalogCache = { value, expires: Date.now() + 5 * 60000 };
|
||||
return value;
|
||||
}
|
||||
|
||||
async snapshot(routeId, db = this.pool) {
|
||||
const route = (await db.query('SELECT id, route_number, name FROM routes WHERE id = $1', [routeId])).rows[0];
|
||||
if (!route) throw sourceError('Маршрут не найден.', 404);
|
||||
const stops = (await db.query(`SELECT rs.stop_id, rs.sequence, s.name FROM route_stops rs
|
||||
JOIN stops s ON s.id = rs.stop_id WHERE rs.route_id = $1 ORDER BY rs.sequence, rs.id`, [routeId])).rows;
|
||||
const schedules = (await db.query(`SELECT id, day_type, departure_times, trips,
|
||||
to_char(valid_from, 'YYYY-MM-DD') AS valid_from, to_char(valid_until, 'YYYY-MM-DD') AS valid_until,
|
||||
updated_at::text FROM schedules WHERE route_id = $1 ORDER BY id`, [routeId])).rows;
|
||||
return { route, stops, schedules };
|
||||
}
|
||||
|
||||
async store({ routeId, source, status, payload = null, userId }) {
|
||||
const id = randomUUID();
|
||||
await this.pool.query(`INSERT INTO schedule_imports
|
||||
(id, route_id, source_url, source_hash, source, status, payload, created_by)
|
||||
VALUES ($1,$2,$3,$4,$5::jsonb,$6,$7::jsonb,$8)`,
|
||||
[id, routeId, source.url, hash(source), JSON.stringify(source), status, JSON.stringify(payload), userId]);
|
||||
return id;
|
||||
}
|
||||
|
||||
async getDraft(id, db = this.pool, lock = false) {
|
||||
if (!/^[0-9a-f-]{36}$/i.test(id || '')) throw sourceError('Некорректный идентификатор импорта.', 400);
|
||||
const draft = (await db.query(`SELECT * FROM schedule_imports WHERE id = $1${lock ? ' FOR UPDATE' : ''}`, [id])).rows[0];
|
||||
if (!draft) throw sourceError('Предпросмотр не найден. Загрузите расписание заново.', 404);
|
||||
if (draft.status !== 'applied' && Date.now() - new Date(draft.created_at).getTime() > 30 * 60000) {
|
||||
throw sourceError('Предпросмотр устарел. Загрузите расписание заново.', 409);
|
||||
}
|
||||
return draft;
|
||||
}
|
||||
|
||||
async inspect({ routeIds, sourceUrl, userId }) {
|
||||
await this.ensureSchema();
|
||||
if (!Array.isArray(routeIds) || !routeIds.length || routeIds.length > 30
|
||||
|| routeIds.some(id => !Number.isInteger(Number(id)) || Number(id) <= 0)
|
||||
|| new Set(routeIds.map(Number)).size !== routeIds.length) throw sourceError('Выберите направления в админке.', 400);
|
||||
routeIds = routeIds.map(Number).sort((a, b) => a - b);
|
||||
const url = safeUrl(sourceUrl);
|
||||
if (!(await this.catalog()).some(route => route.url === url)) throw sourceError('Выберите маршрут из списка сайта.');
|
||||
const page = await this.load(url);
|
||||
const source = parseSchedule(page.html, page.url);
|
||||
const snapshots = await Promise.all(routeIds.map(id => this.snapshot(id)));
|
||||
if (snapshots.some(snapshot => normalizeNumber(source.number) !== normalizeNumber(snapshot.route.route_number))) {
|
||||
throw sourceError('Все выбранные направления должны иметь тот же номер, что и маршрут на сайте.');
|
||||
}
|
||||
const mappings = {};
|
||||
const origins = departureStops(source);
|
||||
for (const snapshot of snapshots) {
|
||||
const matches = origins.filter(name => normalizeName(name) === normalizeName(snapshot.stops[0]?.name));
|
||||
mappings[snapshot.route.id] = matches.length === 1 ? matches[0] : '';
|
||||
}
|
||||
await this.pool.query("DELETE FROM schedule_imports WHERE status <> 'applied' AND created_at < now() - interval '7 days'");
|
||||
const id = await this.store({ routeId: null, source, status: 'inspection', payload: { version: 2, routeIds }, userId });
|
||||
return { id, source, mappings, directions: snapshots.map(({ route, stops }) => ({ route, stops })),
|
||||
departureStops: origins, today: getLocalDateTime().date };
|
||||
}
|
||||
|
||||
async preview({ inspectionId, mappings, validFrom, userId }) {
|
||||
await this.ensureSchema();
|
||||
const inspection = await this.getDraft(inspectionId);
|
||||
if (inspection.status !== 'inspection') throw sourceError('Загрузите исходную таблицу заново.', 409);
|
||||
if (inspection.payload?.version !== 2) throw sourceError('Формат импорта обновлён. Загрузите таблицы заново.', 409);
|
||||
const routeIds = inspection.payload.routeIds;
|
||||
if (!mappings || Object.keys(mappings).length !== routeIds.length
|
||||
|| routeIds.some(id => !Object.hasOwn(mappings, id))) throw sourceError('Сопоставьте каждое выбранное направление.');
|
||||
const snapshots = await Promise.all(routeIds.map(id => this.snapshot(id)));
|
||||
const plans = snapshots.map(snapshot => buildDirectionPlan(inspection.source, mappings[snapshot.route.id], snapshot, validFrom));
|
||||
const payload = { version: 2, plans, validFrom, hasChanges: plans.some(plan => plan.hasChanges) };
|
||||
const id = await this.store({ routeId: null, source: inspection.source, status: 'ready', payload, userId });
|
||||
return { id, ...payload, sourceUrl: inspection.source_url };
|
||||
}
|
||||
|
||||
async apply(id) {
|
||||
await this.ensureSchema();
|
||||
const draft = await this.getDraft(id);
|
||||
if (draft.status === 'applied') return { id, applied: true, alreadyApplied: true };
|
||||
if (draft.payload?.version !== 2) throw sourceError('Формат импорта обновлён. Загрузите таблицы заново.', 409);
|
||||
if (draft.status !== 'ready' || !draft.payload?.hasChanges) throw sourceError('Нет подготовленных изменений.', 409);
|
||||
// A changed website invalidates the preview rather than silently importing a different table.
|
||||
const page = await this.load(draft.source_url);
|
||||
if (hash(parseSchedule(page.html, page.url)) !== draft.source_hash) throw sourceError('Расписание на сайте изменилось. Загрузите новый предпросмотр.', 409);
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const locked = await this.getDraft(id, client, true);
|
||||
if (locked.status === 'applied') {
|
||||
await client.query('COMMIT');
|
||||
return { id, applied: true, alreadyApplied: true };
|
||||
}
|
||||
// Existing CRUD endpoints do not share an advisory lock; table locks also cover inserts.
|
||||
await client.query('LOCK TABLE schedules, route_stops IN SHARE ROW EXCLUSIVE MODE');
|
||||
for (const plan of locked.payload.plans) {
|
||||
const snapshot = await this.snapshot(plan.routeId, client);
|
||||
if (hash(snapshot) !== plan.beforeHash) throw sourceError(`Направление «${plan.route.name}» изменилось после предпросмотра. Постройте его заново.`, 409);
|
||||
}
|
||||
if (locked.payload.validFrom < getLocalDateTime().date) throw sourceError('Дата начала уже прошла. Постройте новый предпросмотр.', 409);
|
||||
for (const plan of locked.payload.plans) {
|
||||
for (const change of plan.changes) {
|
||||
if (!change.added.length && !change.removed.length) continue;
|
||||
if (change.existingId && change.previousValidFrom === change.validFrom) {
|
||||
await client.query(`UPDATE schedules SET departure_times = $1::time[], trips = $2::jsonb,
|
||||
updated_at = CURRENT_TIMESTAMP WHERE id = $3`,
|
||||
[change.after.map(trip => trip.departureTime), JSON.stringify(change.after), change.existingId]);
|
||||
} else {
|
||||
if (change.existingId) await client.query(`UPDATE schedules SET valid_until = $1::date - 1,
|
||||
updated_at = CURRENT_TIMESTAMP WHERE id = $2`, [change.validFrom, change.existingId]);
|
||||
await client.query(`INSERT INTO schedules (route_id, day_type, departure_times, trips, valid_from, valid_until)
|
||||
VALUES ($1,$2,$3::time[],$4::jsonb,$5,$6)`,
|
||||
[plan.routeId, change.dayType, change.after.map(trip => trip.departureTime), JSON.stringify(change.after), change.validFrom, change.validUntil]);
|
||||
}
|
||||
}
|
||||
}
|
||||
await client.query("UPDATE schedule_imports SET status = 'applied', applied_at = now() WHERE id = $1", [id]);
|
||||
await client.query('COMMIT');
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
throw error;
|
||||
} finally { client.release(); }
|
||||
let cacheWarning = null;
|
||||
const cacheResults = await Promise.allSettled(draft.payload.plans.map(plan => this.invalidate(plan.routeId)));
|
||||
if (cacheResults.some(result => result.status === 'rejected')) {
|
||||
cacheWarning = 'Расписание сохранено, но кеш не удалось обновить. Данные в кеше обновятся после истечения его срока.';
|
||||
}
|
||||
return { id, applied: true, cacheWarning };
|
||||
}
|
||||
|
||||
async history() {
|
||||
await this.ensureSchema();
|
||||
return (await this.pool.query(`SELECT i.id, i.source_url, i.applied_at, i.payload->>'validFrom' AS valid_from,
|
||||
(SELECT jsonb_agg(p->'route') FROM jsonb_array_elements(COALESCE(i.payload->'plans', '[]'::jsonb)) p) AS directions,
|
||||
r.route_number, r.name AS route_name FROM schedule_imports i LEFT JOIN routes r ON r.id = i.route_id
|
||||
WHERE i.status = 'applied' ORDER BY i.applied_at DESC LIMIT 30`)).rows;
|
||||
}
|
||||
|
||||
async news(refresh = false) {
|
||||
await this.ensureSchema();
|
||||
if (refresh) {
|
||||
const page = await this.load(HOME_URL);
|
||||
const items = parseNews(page.html);
|
||||
for (const item of items) {
|
||||
await this.pool.query(`INSERT INTO schedule_import_news (url,title,published_date,content,content_hash)
|
||||
VALUES ($1,$2,$3,$4,$5) ON CONFLICT (url) DO UPDATE SET title = EXCLUDED.title,
|
||||
published_date = EXCLUDED.published_date, content = EXCLUDED.content,
|
||||
content_hash = EXCLUDED.content_hash, checked_at = now()`,
|
||||
[item.url, item.title, item.publishedDate, item.content, item.contentHash]);
|
||||
}
|
||||
}
|
||||
return (await this.pool.query(`SELECT *, (reviewed_hash = content_hash) IS TRUE AS reviewed
|
||||
FROM schedule_import_news ORDER BY published_date DESC NULLS LAST, url DESC LIMIT 50`)).rows;
|
||||
}
|
||||
|
||||
async reviewNews({ url, contentHash }) {
|
||||
await this.ensureSchema();
|
||||
const result = await this.pool.query(`UPDATE schedule_import_news SET reviewed_hash = content_hash
|
||||
WHERE url = $1 AND content_hash = $2 RETURNING url`, [safeUrl(url), contentHash]);
|
||||
if (!result.rows.length) throw sourceError('Новость изменилась. Обновите список перед проверкой.', 409);
|
||||
return { reviewed: true };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { ScheduleImportService, buildPlan, compareTrips, buildDirectionPlan, departureStops };
|
||||
220
backend/src/services/scheduleSource.js
Normal file
220
backend/src/services/scheduleSource.js
Normal file
@@ -0,0 +1,220 @@
|
||||
const cheerio = require('cheerio');
|
||||
const { createHash } = require('node:crypto');
|
||||
|
||||
const HOME_URL = 'https://ap2polotsk.of.by/ap2/';
|
||||
const CATALOG_URL = `${HOME_URL}rasp/rasp-polotsk/polotsk-gorod/`;
|
||||
const clean = value => String(value || '').replace(/\s+/g, ' ').trim();
|
||||
// JSONB can reorder object keys, so content hashes must be canonical across DB round trips.
|
||||
const canonical = value => Array.isArray(value) ? value.map(canonical)
|
||||
: value && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
const hash = value => createHash('sha256').update(JSON.stringify(canonical(value))).digest('hex');
|
||||
const normalizeName = value => clean(value).toLowerCase().replace(/ё/g, 'е').replace(/[«»"()]/g, '').replace(/\s*[–—-]\s*/g, '-').trim();
|
||||
const normalizeNumber = value => clean(value).toLowerCase().replace(/a/g, 'а').replace(/\s/g, '');
|
||||
|
||||
function sourceError(message, status = 422) {
|
||||
return Object.assign(new Error(message), { status });
|
||||
}
|
||||
|
||||
function safeUrl(value, base = HOME_URL) {
|
||||
let url;
|
||||
try { url = new URL(value, base); } catch { throw sourceError('Некорректная ссылка источника.'); }
|
||||
if (!['http:', 'https:'].includes(url.protocol) || url.hostname !== 'ap2polotsk.of.by'
|
||||
|| url.port || url.username || url.password || !url.pathname.startsWith('/ap2/')) {
|
||||
throw sourceError('Разрешены только страницы официального сайта автопарка.');
|
||||
}
|
||||
url.protocol = 'https:';
|
||||
url.hash = '';
|
||||
return url.href;
|
||||
}
|
||||
|
||||
// Bound redirects, total response time and decoded body size. Never forward credentials.
|
||||
async function fetchHtml(value, fetchImpl = fetch) {
|
||||
let url = safeUrl(value);
|
||||
const signal = AbortSignal.timeout(20000);
|
||||
try {
|
||||
for (let redirects = 0; redirects <= 5; redirects += 1) {
|
||||
const response = await fetchImpl(url, {
|
||||
redirect: 'manual', signal,
|
||||
headers: { 'User-Agent': 'PolotskTransitAdmin/1.0', Accept: 'text/html' }
|
||||
});
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
await response.body?.cancel();
|
||||
if (!response.headers.get('location')) throw sourceError('Пустое перенаправление источника.', 502);
|
||||
url = safeUrl(response.headers.get('location'), url);
|
||||
continue;
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw sourceError(`Сайт автопарка ответил HTTP ${response.status}. Попробуйте позже.`, 502);
|
||||
}
|
||||
if (!/text\/html|application\/xhtml\+xml/i.test(response.headers.get('content-type') || '')) {
|
||||
await response.body?.cancel();
|
||||
throw sourceError('Источник вернул не HTML-страницу.', 502);
|
||||
}
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
for await (const chunk of response.body) {
|
||||
size += chunk.length;
|
||||
if (size > 2 * 1024 * 1024) throw sourceError('Страница источника слишком большая.', 502);
|
||||
chunks.push(chunk);
|
||||
}
|
||||
return { url, html: Buffer.concat(chunks).toString('utf8') };
|
||||
}
|
||||
throw sourceError('Слишком много перенаправлений источника.', 502);
|
||||
} catch (error) {
|
||||
if (error.status) throw error;
|
||||
throw sourceError('Не удалось загрузить сайт автопарка. Проверьте доступ сервера к интернету и повторите попытку.', 502);
|
||||
}
|
||||
}
|
||||
|
||||
function parseCatalog(html) {
|
||||
const $ = cheerio.load(html);
|
||||
const routes = new Map();
|
||||
$('.entry-content a[href]').each((_, node) => {
|
||||
const a = $(node);
|
||||
let url;
|
||||
try { url = safeUrl(a.attr('href')); } catch { return; }
|
||||
const match = new URL(url).pathname.match(/\/m-(\d+)([aаэ]?)(?:-\d+)?\/$/i);
|
||||
if (!match) return;
|
||||
const label = clean(a.text() || a.find('img').attr('alt'));
|
||||
// The site's image for 8A is incorrectly labelled "8".
|
||||
const number = match[2] ? `${match[1]}${match[2]}` : (/^\d+[аaэ]?$/i.test(label) ? label : match[1]);
|
||||
routes.set(url, { url, number: normalizeNumber(number).toUpperCase() });
|
||||
});
|
||||
if (!routes.size) throw sourceError('На сайте не найден список городских маршрутов.');
|
||||
return [...routes.values()];
|
||||
}
|
||||
|
||||
function dayTypes(text) {
|
||||
const label = clean(text).toLowerCase();
|
||||
if (/^рабочи[йе] (день|дни)$|^будни[йе] (день|дни)$/.test(label)) return ['weekday'];
|
||||
if (/^выходно[йе] (день|дни)$|^выходные дни$/.test(label)) return ['saturday', 'sunday'];
|
||||
if (/^суббота$/.test(label)) return ['saturday'];
|
||||
if (/^воскресенье$/.test(label)) return ['sunday'];
|
||||
return null;
|
||||
}
|
||||
|
||||
const WEEKDAY_FORMS = [
|
||||
['понедельник', 'понедельникам', 'понедельника'], ['вторник', 'вторникам', 'вторника'],
|
||||
['среда', 'средам', 'среды', 'среду'], ['четверг', 'четвергам', 'четверга'],
|
||||
['пятница', 'пятницам', 'пятницы', 'пятницу'], ['суббота', 'субботам', 'субботы', 'субботу'],
|
||||
['воскресенье', 'воскресеньям', 'воскресенья'],
|
||||
];
|
||||
|
||||
function parseDeparture(value) {
|
||||
const match = clean(value).match(/^([01]?\d|2[0-3]):([0-5]\d)(?:\s+(.+))?$/);
|
||||
if (!match) return null;
|
||||
const trip = { departureTime: `${match[1].padStart(2, '0')}:${match[2]}:00`,
|
||||
viaStopNames: [], daysOfWeek: [], exceptDaysOfWeek: [] };
|
||||
let rest = match[3] || '';
|
||||
// Consume the entire annotation. Unknown suffixes must never silently lose conditions.
|
||||
while (rest) {
|
||||
const clause = rest.match(/^(через|(?:только\s+)?по|кроме)\s+(.+?)(?=\s+(?:через|(?:только\s+)?по|кроме)\s+|$)/i);
|
||||
if (!clause) return null;
|
||||
const kind = clause[1].toLowerCase();
|
||||
const content = clause[2];
|
||||
if (kind === 'через') {
|
||||
// Quoted names allow punctuation; unquoted names must contain no unexplained syntax.
|
||||
const quoted = content.match(/^(?:«([^«»]+)»|"([^"\n]+)"|“([^“”]+)”|„([^„“]+)“)$/);
|
||||
const name = clean(quoted ? quoted.slice(1).find(Boolean) : content);
|
||||
if (!name || (!quoted && !/^[а-яёa-z][а-яёa-z\s–—-]*$/i.test(name)) || trip.viaStopNames.length) return null;
|
||||
trip.viaStopNames.push(name);
|
||||
} else {
|
||||
const field = kind === 'кроме' ? 'exceptDaysOfWeek' : 'daysOfWeek';
|
||||
if (trip[field].length) return null;
|
||||
const words = content.toLowerCase().split(/\s*,\s*|\s+и\s+/);
|
||||
const days = words.map(word => WEEKDAY_FORMS.findIndex(forms => forms.includes(word)) + 1);
|
||||
if (!days.length || days.includes(0)) return null;
|
||||
trip[field] = [...new Set(days)].sort((a, b) => a - b);
|
||||
}
|
||||
rest = clean(rest.slice(clause[0].length));
|
||||
}
|
||||
if (trip.daysOfWeek.some(day => trip.exceptDaysOfWeek.includes(day))) return null;
|
||||
return trip;
|
||||
}
|
||||
|
||||
function parseSchedule(html, url) {
|
||||
const $ = cheerio.load(html);
|
||||
const content = $('.entry-content').first();
|
||||
const title = clean($('.entry-title').first().text());
|
||||
const number = title.match(/маршрут\s*№\s*(\d+[аaэ]?)/i)?.[1];
|
||||
if (!number || /\/\d{4}\/\d{2}\/\d{2}\//.test(new URL(safeUrl(url)).pathname)) {
|
||||
throw sourceError('Выбрана не страница основного расписания маршрута.');
|
||||
}
|
||||
const rows = [];
|
||||
const warnings = [];
|
||||
content.find('table').each((tableIndex, table) => {
|
||||
let days = null;
|
||||
$(table).find('tr').each((rowIndex, tr) => {
|
||||
const cells = $(tr).children('td, th').toArray();
|
||||
const texts = cells.map(cell => clean($(cell).text()));
|
||||
if (!texts.some(Boolean)) return;
|
||||
const parsedDays = dayTypes(texts.join(' '));
|
||||
if (parsedDays) { days = parsedDays; return; }
|
||||
if (texts.join(' ').includes('к списку маршрутов')) return;
|
||||
if (cells.length !== 2 || !days) {
|
||||
warnings.push(`Не распознана строка таблицы: ${texts.join(' — ').slice(0, 240)}`);
|
||||
return;
|
||||
}
|
||||
const [stopName, rawTimes] = texts;
|
||||
const tokens = rawTimes.split('|').map(clean).filter(Boolean);
|
||||
const departures = tokens.map(parseDeparture);
|
||||
const invalid = !tokens.length || departures.some(trip => !trip);
|
||||
// Marked/crossed-out times require human interpretation, even if their text looks normal.
|
||||
const marked = $(tr).find('sup, del, s, strike, [style*="line-through"]').length > 0;
|
||||
if (!stopName || invalid || marked) {
|
||||
warnings.push(`«${stopName}»: есть примечания или нераспознанные времена: ${rawTimes.slice(0, 240)}`);
|
||||
}
|
||||
const times = invalid ? [] : departures.map(trip => trip.departureTime);
|
||||
if (new Set(times).size !== times.length) warnings.push(`«${stopName}»: повторяющиеся времена требуют проверки.`);
|
||||
rows.push({ key: `${tableIndex}:${rowIndex}`, stopName, days, times, departures: invalid ? [] : departures, rawTimes });
|
||||
});
|
||||
});
|
||||
if (!rows.length) throw sourceError('Таблицы расписания не распознаны. Существующие данные сохранены.');
|
||||
const directions = [];
|
||||
content.find('p, div').each((_, node) => {
|
||||
const text = clean($(node).text());
|
||||
if (!/^(Прямое|Обратное) направление:/i.test(text)) return;
|
||||
const stops = [...text.matchAll(/«\s*([^»]+?)\s*»/g)].map(match => clean(match[1]));
|
||||
if (stops.length) directions.push({ label: text.split(':')[0], stops });
|
||||
});
|
||||
// Footnotes outside tables must not silently disappear during import.
|
||||
const outside = content.clone();
|
||||
outside.find('table').remove();
|
||||
const outsideText = clean(outside.text());
|
||||
if (/кроме|только|примечани|отмен|перенос|по пятниц|по понедельник|с \d{1,2}\.\d{1,2}|\d{1,2}:\d{2}/i.test(outsideText)) {
|
||||
warnings.push('Вне таблиц есть условия или изменения расписания. Проверьте их на сайте; автоматическое применение заблокировано.');
|
||||
}
|
||||
return { title, number: normalizeNumber(number).toUpperCase(), url: safeUrl(url), rows, directions, warnings };
|
||||
}
|
||||
|
||||
function parseNews(html) {
|
||||
const $ = cheerio.load(html);
|
||||
const news = [];
|
||||
$('article').each((_, node) => {
|
||||
const article = $(node);
|
||||
const title = clean(article.find('.entry-title').text());
|
||||
const href = article.find('.entry-title a').attr('href');
|
||||
if (!title || !href) return;
|
||||
let url;
|
||||
try { url = safeUrl(href); } catch { return; }
|
||||
const content = article.find('.entry-content').clone();
|
||||
content.find('script, style').remove();
|
||||
content.find('table').each((_, table) => {
|
||||
const rows = $(table).find('tr').toArray().map(tr => $(tr).children('td, th').toArray()
|
||||
.map(cell => clean($(cell).text())).filter(Boolean).join(' — ')).filter(Boolean);
|
||||
$(table).replaceWith($('<p>').text(rows.join('\n')));
|
||||
});
|
||||
content.find('br').replaceWith('\n');
|
||||
content.find('p, tr, div').append('\n');
|
||||
const text = content.text().split('\n').map(clean).filter(Boolean).join('\n').slice(0, 30000);
|
||||
const publishedDate = article.find('time').first().attr('datetime')?.slice(0, 10)
|
||||
|| url.match(/\/(\d{4})\/(\d{2})\/(\d{2})\//)?.slice(1).join('-') || null;
|
||||
news.push({ url, title, content: text, publishedDate, contentHash: hash({ title, text }) });
|
||||
});
|
||||
if (!news.length) throw sourceError('Не удалось распознать новости автопарка.');
|
||||
return news;
|
||||
}
|
||||
|
||||
module.exports = { HOME_URL, CATALOG_URL, clean, hash, normalizeName, normalizeNumber,
|
||||
sourceError, safeUrl, fetchHtml, parseCatalog, parseSchedule, parseNews, parseDeparture };
|
||||
Reference in New Issue
Block a user