fix(scheduled-runner): ретрай транзиентных ошибок отправки (522/5xx/429/сеть) вместо мгновенного failed + send_attempts

This commit is contained in:
ZeroDay
2026-07-07 06:50:46 +00:00
parent 4ff1bd51d9
commit 821d9ac66c
2 changed files with 34 additions and 6 deletions
+1
View File
@@ -224,6 +224,7 @@ const migrate = async () => {
// safe column alters (existing tables on prod may lack newer columns) // safe column alters (existing tables on prod may lack newer columns)
await query(`ALTER TABLE categories ADD COLUMN IF NOT EXISTS is_active BOOLEAN DEFAULT true`); await query(`ALTER TABLE categories ADD COLUMN IF NOT EXISTS is_active BOOLEAN DEFAULT true`);
await query(`ALTER TABLE scheduled_posts ADD COLUMN IF NOT EXISTS send_attempts INT NOT NULL DEFAULT 0`);
console.log('[DB] Migrations applied'); console.log('[DB] Migrations applied');
}; };
+30 -3
View File
@@ -401,6 +401,21 @@ async function publishOne(scheduledPost) {
// массового ретрая всё накопившееся улетело бы в канал пачкой. // массового ретрая всё накопившееся улетело бы в канал пачкой.
const SKIP_OLDER_THAN_H = 3; const SKIP_OLDER_THAN_H = 3;
// Ретрай транзиентных ошибок отправки (522/5xx/429/сетевые таймауты).
// Разовый сбой Telegram-прокси (CF Worker) не должен насмерть ронять слот.
const MAX_SEND_ATTEMPTS = 5;
const RETRY_BACKOFF_MIN = [2, 5, 10, 20, 30]; // минуты между попытками
function isTransientSendError(err) {
const status = err.response?.status;
if (status && status >= 500) return true; // 500..599 включая 522
if (status === 429) return true; // rate limit
const code = String(err.code || '');
if (['ECONNRESET','ETIMEDOUT','ECONNREFUSED','ECONNABORTED','EAI_AGAIN','EPIPE'].includes(code)) return true;
const m = String(err.message || '').toLowerCase();
if (m.includes('timeout') || m.includes('socket hang up') || m.includes('network')) return true;
return false;
}
async function runScheduled() { async function runScheduled() {
// 1) Помечаем слишком старые pending как skipped (не спамим канал задним числом) // 1) Помечаем слишком старые pending как skipped (не спамим канал задним числом)
const { rows: skipped } = await query( const { rows: skipped } = await query(
@@ -434,12 +449,24 @@ async function runScheduled() {
console.log(`[scheduled-runner] sent id=${sp.id} channel=${sp.channel_id} article=${sp.article_id}`); console.log(`[scheduled-runner] sent id=${sp.id} channel=${sp.channel_id} article=${sp.article_id}`);
} catch (err) { } catch (err) {
const msg = err.response?.data?.description || err.response?.data?.error?.error_msg || err.message; const msg = err.response?.data?.description || err.response?.data?.error?.error_msg || err.message;
const attempts = (sp.send_attempts || 0) + 1;
if (isTransientSendError(err) && attempts < MAX_SEND_ATTEMPTS) {
const delayMin = RETRY_BACKOFF_MIN[Math.min(attempts - 1, RETRY_BACKOFF_MIN.length - 1)];
const retryAt = new Date(Date.now() + delayMin * 60_000);
await query( await query(
`UPDATE scheduled_posts SET status='failed', error=$1 WHERE id=$2`, `UPDATE scheduled_posts SET status='pending', scheduled_at=$1, send_attempts=$2, error=$3 WHERE id=$4`,
[String(msg).slice(0, 1000), sp.id] [retryAt, attempts, `retry ${attempts}/${MAX_SEND_ATTEMPTS} через ${delayMin}м: ${String(msg).slice(0, 300)}`, sp.id]
);
results.push({ id: sp.id, ok: false, retry: attempts, error: msg });
console.warn(`[scheduled-runner] transient id=${sp.id}, retry ${attempts}/${MAX_SEND_ATTEMPTS} in ${delayMin}m: ${msg}`);
} else {
await query(
`UPDATE scheduled_posts SET status='failed', send_attempts=$1, error=$2 WHERE id=$3`,
[attempts, String(msg).slice(0, 1000), sp.id]
); );
results.push({ id: sp.id, ok: false, error: msg }); results.push({ id: sp.id, ok: false, error: msg });
console.error(`[scheduled-runner] failed id=${sp.id}: ${msg}`); console.error(`[scheduled-runner] failed id=${sp.id} (attempts=${attempts}): ${msg}`);
}
} }
} }
return { processed: rows.length, skipped: skipped.length, results }; return { processed: rows.length, skipped: skipped.length, results };