fix(scheduled-runner): ретрай транзиентных ошибок отправки (522/5xx/429/сеть) вместо мгновенного failed + send_attempts
This commit is contained in:
@@ -224,6 +224,7 @@ const migrate = async () => {
|
|||||||
|
|
||||||
// safe column alters (existing tables on prod may lack newer columns)
|
// safe column alters (existing tables on prod may lack newer columns)
|
||||||
await query(`ALTER TABLE categories ADD COLUMN IF NOT EXISTS is_active BOOLEAN DEFAULT true`);
|
await query(`ALTER TABLE categories ADD COLUMN IF NOT EXISTS is_active BOOLEAN DEFAULT true`);
|
||||||
|
await query(`ALTER TABLE scheduled_posts ADD COLUMN IF NOT EXISTS send_attempts INT NOT NULL DEFAULT 0`);
|
||||||
|
|
||||||
console.log('[DB] Migrations applied');
|
console.log('[DB] Migrations applied');
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -401,6 +401,21 @@ async function publishOne(scheduledPost) {
|
|||||||
// массового ретрая всё накопившееся улетело бы в канал пачкой.
|
// массового ретрая всё накопившееся улетело бы в канал пачкой.
|
||||||
const SKIP_OLDER_THAN_H = 3;
|
const SKIP_OLDER_THAN_H = 3;
|
||||||
|
|
||||||
|
// Ретрай транзиентных ошибок отправки (522/5xx/429/сетевые таймауты).
|
||||||
|
// Разовый сбой Telegram-прокси (CF Worker) не должен насмерть ронять слот.
|
||||||
|
const MAX_SEND_ATTEMPTS = 5;
|
||||||
|
const RETRY_BACKOFF_MIN = [2, 5, 10, 20, 30]; // минуты между попытками
|
||||||
|
function isTransientSendError(err) {
|
||||||
|
const status = err.response?.status;
|
||||||
|
if (status && status >= 500) return true; // 500..599 включая 522
|
||||||
|
if (status === 429) return true; // rate limit
|
||||||
|
const code = String(err.code || '');
|
||||||
|
if (['ECONNRESET','ETIMEDOUT','ECONNREFUSED','ECONNABORTED','EAI_AGAIN','EPIPE'].includes(code)) return true;
|
||||||
|
const m = String(err.message || '').toLowerCase();
|
||||||
|
if (m.includes('timeout') || m.includes('socket hang up') || m.includes('network')) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
async function runScheduled() {
|
async function runScheduled() {
|
||||||
// 1) Помечаем слишком старые pending как skipped (не спамим канал задним числом)
|
// 1) Помечаем слишком старые pending как skipped (не спамим канал задним числом)
|
||||||
const { rows: skipped } = await query(
|
const { rows: skipped } = await query(
|
||||||
@@ -434,12 +449,24 @@ async function runScheduled() {
|
|||||||
console.log(`[scheduled-runner] sent id=${sp.id} channel=${sp.channel_id} article=${sp.article_id}`);
|
console.log(`[scheduled-runner] sent id=${sp.id} channel=${sp.channel_id} article=${sp.article_id}`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err.response?.data?.description || err.response?.data?.error?.error_msg || err.message;
|
const msg = err.response?.data?.description || err.response?.data?.error?.error_msg || err.message;
|
||||||
|
const attempts = (sp.send_attempts || 0) + 1;
|
||||||
|
if (isTransientSendError(err) && attempts < MAX_SEND_ATTEMPTS) {
|
||||||
|
const delayMin = RETRY_BACKOFF_MIN[Math.min(attempts - 1, RETRY_BACKOFF_MIN.length - 1)];
|
||||||
|
const retryAt = new Date(Date.now() + delayMin * 60_000);
|
||||||
await query(
|
await query(
|
||||||
`UPDATE scheduled_posts SET status='failed', error=$1 WHERE id=$2`,
|
`UPDATE scheduled_posts SET status='pending', scheduled_at=$1, send_attempts=$2, error=$3 WHERE id=$4`,
|
||||||
[String(msg).slice(0, 1000), sp.id]
|
[retryAt, attempts, `retry ${attempts}/${MAX_SEND_ATTEMPTS} через ${delayMin}м: ${String(msg).slice(0, 300)}`, sp.id]
|
||||||
|
);
|
||||||
|
results.push({ id: sp.id, ok: false, retry: attempts, error: msg });
|
||||||
|
console.warn(`[scheduled-runner] transient id=${sp.id}, retry ${attempts}/${MAX_SEND_ATTEMPTS} in ${delayMin}m: ${msg}`);
|
||||||
|
} else {
|
||||||
|
await query(
|
||||||
|
`UPDATE scheduled_posts SET status='failed', send_attempts=$1, error=$2 WHERE id=$3`,
|
||||||
|
[attempts, String(msg).slice(0, 1000), sp.id]
|
||||||
);
|
);
|
||||||
results.push({ id: sp.id, ok: false, error: msg });
|
results.push({ id: sp.id, ok: false, error: msg });
|
||||||
console.error(`[scheduled-runner] failed id=${sp.id}: ${msg}`);
|
console.error(`[scheduled-runner] failed id=${sp.id} (attempts=${attempts}): ${msg}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return { processed: rows.length, skipped: skipped.length, results };
|
return { processed: rows.length, skipped: skipped.length, results };
|
||||||
|
|||||||
Reference in New Issue
Block a user