fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -1,4 +1,5 @@
import { Injectable, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { ConnectionService } from '../global/database/connection.service';
import { NatsService } from '../global/nats/nats.service';
import { resolveDakaoNatsNames } from './dakao.nats-names';
@@ -10,6 +11,7 @@ import {
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
const TERMINAL_STATES = ['sent', 'failed'] as const;
const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000;
@Injectable()
export class DakaoOutboxService {
@@ -50,35 +52,59 @@ export class DakaoOutboxService {
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
).exec();
await this.publishCommand({
subject: params.subject,
eventId: params.eventId,
envelope: params.envelope,
});
}
private async publishCommand<TPayload>(params: {
subject: string;
eventId: string;
envelope: DakaoCommandEnvelope<TPayload>;
publishLeaseOwnerIdentifier?: string;
}): Promise<void> {
const OutboxModel =
await this.connectionService.getModel('dakaoOutboxMessage');
const updateFilter = {
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
...(params.publishLeaseOwnerIdentifier
? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier }
: {}),
};
try {
await this.natsService.publishJetStream(params.subject, params.envelope, {
messageId: params.eventId,
});
await OutboxModel.updateOne(
{
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
},
updateFilter,
{
$set: {
state: 'published',
publishedAt: new Date(),
},
$inc: { publishAttempts: 1 },
$unset: { lastPublishError: '' },
$unset: {
lastPublishError: '',
publishLeaseOwnerIdentifier: '',
publishLeaseExpiresAt: '',
},
},
).exec();
} catch (err) {
await OutboxModel.updateOne(
{
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
},
updateFilter,
{
$inc: { publishAttempts: 1 },
$set: {
lastPublishError: (err as Error)?.message ?? String(err),
},
$unset: {
publishLeaseOwnerIdentifier: '',
publishLeaseExpiresAt: '',
},
},
).exec();
this.logger.error(
@@ -146,40 +172,77 @@ export class DakaoOutboxService {
public async sweepStale(params?: {
staleMs?: number;
publishLeaseDurationMilliseconds?: number;
limit?: number;
}): Promise<number> {
const OutboxModel =
await this.connectionService.getModel('dakaoOutboxMessage');
const currentTimestamp = new Date();
const staleBefore = new Date(
Date.now() - (params?.staleMs ?? 5 * 60 * 1000),
currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000),
);
const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`;
const publishLeaseExpiresAt = new Date(
currentTimestamp.getTime() +
(params?.publishLeaseDurationMilliseconds ??
DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS),
);
const rows = await OutboxModel.find({
state: { $in: ['pending', 'published'] },
updatedAt: { $lte: staleBefore },
$or: [
{ publishLeaseExpiresAt: { $exists: false } },
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
],
})
.sort({ updatedAt: 1 })
.limit(params?.limit ?? 20)
.lean()
.exec();
let swept = 0;
for (const row of rows) {
await this.enqueueAndPublish({
subject: row.subject,
eventId: row.eventId,
const claimedRow = await OutboxModel.findOneAndUpdate(
{
eventId: row.eventId,
state: { $in: ['pending', 'published'] },
$or: [
{ publishLeaseExpiresAt: { $exists: false } },
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
],
},
{
$set: {
publishLeaseOwnerIdentifier,
publishLeaseExpiresAt,
},
},
{ returnDocument: 'after' },
)
.lean()
.exec();
if (!claimedRow) continue;
await this.publishCommand({
subject: claimedRow.subject,
eventId: claimedRow.eventId,
publishLeaseOwnerIdentifier,
envelope: {
schema: 'dakao.command.v1',
eventId: row.eventId,
type: row.type,
eventSource: row.eventSource,
correlationId: row.correlationId,
eventId: claimedRow.eventId,
type: claimedRow.type,
eventSource: claimedRow.eventSource,
correlationId: claimedRow.correlationId,
resultSubjects:
row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type),
payload: row.payload,
claimedRow.resultSubjects ??
this.resolveDefaultResultSubjects(claimedRow.type),
payload: claimedRow.payload,
},
});
swept += 1;
}
return rows.length;
return swept;
}
private resolveDefaultResultSubjects(