fix: close competition cutover integrity gaps
This commit is contained in:
1 parent
7ea2002975
commit
683d602d63
57 files changed
+4805
-1582
No files matched your search
@@ -15,6 +15,7 @@ const DRAIN_INTERVAL_MS = 5_000;
|
||||
const PROCESSING_LEASE_MS = 60_000;
|
||||
const MAX_EVENTS_PER_DRAIN = 20;
|
||||
const MAX_RETRY_DELAY_MS = 5 * 60_000;
|
||||
const MAX_DELIVERY_ATTEMPTS = 10;
|
||||
|
||||
interface ClaimedOutboxEvent {
|
||||
_id: mongoose.Types.ObjectId;
|
||||
@@ -130,18 +131,33 @@ export class CompetitionOutboxProcessor
|
||||
$unset: { lastError: 1 },
|
||||
}).exec();
|
||||
} catch (error) {
|
||||
const lastError = errorMessage(error).slice(0, 2_000);
|
||||
if (event.attemptCount >= MAX_DELIVERY_ATTEMPTS) {
|
||||
const terminal = await OutboxModel.updateOne(leaseFilter(event), {
|
||||
$set: {
|
||||
state: 'dead-letter',
|
||||
lastError,
|
||||
},
|
||||
}).exec();
|
||||
if (terminal.matchedCount > 0) {
|
||||
this.logger.error(
|
||||
`Competition outbox event ${event._id.toString()} moved to dead-letter after ${event.attemptCount} attempts: ${lastError}`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const availableAt = new Date(
|
||||
Date.now() + retryDelayMilliseconds(event.attemptCount),
|
||||
);
|
||||
await OutboxModel.updateOne(leaseFilter(event), {
|
||||
$set: {
|
||||
state: 'failed',
|
||||
lastError: errorMessage(error).slice(0, 2_000),
|
||||
lastError,
|
||||
availableAt,
|
||||
},
|
||||
}).exec();
|
||||
this.logger.warn(
|
||||
`Competition outbox event ${event._id.toString()} failed: ${errorMessage(error)}`,
|
||||
`Competition outbox event ${event._id.toString()} failed: ${lastError}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user