fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -4,10 +4,13 @@ import {
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { DakaoOutboxService } from './dakao-outbox.service';
const SWEEP_INTERVAL_MS = 60 * 1000;
const STALE_OUTBOX_MS = 5 * 60 * 1000;
const SWEEP_LOCK_KEY = 'distributed-lock:dakao-outbox:sweep-stale-publish-leases';
const SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS = 55_000;
@Injectable()
export class DakaoOutboxSweeperService
@@ -16,7 +19,10 @@ export class DakaoOutboxSweeperService
private readonly logger = new Logger(DakaoOutboxSweeperService.name);
private timer: NodeJS.Timeout | undefined;
constructor(private readonly outboxService: DakaoOutboxService) {}
constructor(
private readonly outboxService: DakaoOutboxService,
private readonly distributedLockService: DistributedLockService,
) {}
onModuleInit() {
this.timer = setInterval(() => {
@@ -31,12 +37,23 @@ export class DakaoOutboxSweeperService
private async sweep(): Promise<void> {
try {
const swept = await this.outboxService.sweepStale({
staleMs: STALE_OUTBOX_MS,
limit: 20,
});
if (swept > 0) {
this.logger.warn(`Re-published ${swept} stale Dakao outbox messages.`);
const swept = await this.distributedLockService.runWithLock(
{
lockKey: SWEEP_LOCK_KEY,
lockExpirationDurationMilliseconds:
SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS,
},
async () =>
this.outboxService.sweepStale({
staleMs: STALE_OUTBOX_MS,
limit: 20,
}),
);
const sweptCount = swept ?? 0;
if (sweptCount > 0) {
this.logger.warn(
`Re-published ${sweptCount} stale Dakao outbox messages.`,
);
}
} catch (err) {
this.logger.error(
@@ -70,44 +70,55 @@ describe('DakaoOutboxService', () => {
process.env.NODE_ENV = 'staging';
try {
const exec = jest.fn().mockResolvedValue([
{
eventId: 'lolympus:chat.send-message:chat-log-1',
type: 'chat.send-message',
eventSource: 'lolympus',
subject: 'staging.dakao.chat.send-message.v1',
correlationId: 'chat-log-1',
payload: {
chatroomId: '18474390111480678',
segments: [{ kind: 'text', text: 'done' }],
},
const staleRow = {
eventId: 'lolympus:chat.send-message:chat-log-1',
type: 'chat.send-message',
eventSource: 'lolympus',
subject: 'staging.dakao.chat.send-message.v1',
correlationId: 'chat-log-1',
payload: {
chatroomId: '18474390111480678',
segments: [{ kind: 'text', text: 'done' }],
},
]);
};
const exec = jest.fn().mockResolvedValue([staleRow]);
const lean = jest.fn().mockReturnValue({ exec });
const limit = jest.fn().mockReturnValue({ lean });
const sort = jest.fn().mockReturnValue({ limit });
const claimExec = jest.fn().mockResolvedValue(staleRow);
const claimLean = jest.fn().mockReturnValue({ exec: claimExec });
const outboxModel = {
find: jest.fn().mockReturnValue({ sort }),
findOneAndUpdate: jest.fn().mockReturnValue({ lean: claimLean }),
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
};
const { service } = makeService(outboxModel);
const enqueue = jest
.spyOn(service, 'enqueueAndPublish')
.mockResolvedValue(undefined);
const { service, natsService } = makeService(outboxModel);
await expect(service.sweepStale({ staleMs: 1, limit: 1 })).resolves.toBe(
1,
);
expect(enqueue).toHaveBeenCalledWith(
expect(outboxModel.findOneAndUpdate).toHaveBeenCalledWith(
expect.objectContaining({
envelope: expect.objectContaining({
resultSubjects: {
succeeded:
'staging.lolympus.dakao.chat.send-message.succeeded.v1',
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
},
eventId: 'lolympus:chat.send-message:chat-log-1',
}),
expect.objectContaining({
$set: expect.objectContaining({
publishLeaseOwnerIdentifier: expect.any(String),
publishLeaseExpiresAt: expect.any(Date),
}),
}),
{ returnDocument: 'after' },
);
expect(natsService.publishJetStream).toHaveBeenCalledWith(
'staging.dakao.chat.send-message.v1',
expect.objectContaining({
resultSubjects: {
succeeded: 'staging.lolympus.dakao.chat.send-message.succeeded.v1',
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
},
}),
{ messageId: 'lolympus:chat.send-message:chat-log-1' },
);
} finally {
process.env.NODE_ENV = previousNodeEnv;
@@ -1,4 +1,5 @@
import { Injectable, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { ConnectionService } from '../global/database/connection.service';
import { NatsService } from '../global/nats/nats.service';
import { resolveDakaoNatsNames } from './dakao.nats-names';
@@ -10,6 +11,7 @@ import {
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
const TERMINAL_STATES = ['sent', 'failed'] as const;
const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000;
@Injectable()
export class DakaoOutboxService {
@@ -50,35 +52,59 @@ export class DakaoOutboxService {
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
).exec();
await this.publishCommand({
subject: params.subject,
eventId: params.eventId,
envelope: params.envelope,
});
}
private async publishCommand<TPayload>(params: {
subject: string;
eventId: string;
envelope: DakaoCommandEnvelope<TPayload>;
publishLeaseOwnerIdentifier?: string;
}): Promise<void> {
const OutboxModel =
await this.connectionService.getModel('dakaoOutboxMessage');
const updateFilter = {
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
...(params.publishLeaseOwnerIdentifier
? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier }
: {}),
};
try {
await this.natsService.publishJetStream(params.subject, params.envelope, {
messageId: params.eventId,
});
await OutboxModel.updateOne(
{
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
},
updateFilter,
{
$set: {
state: 'published',
publishedAt: new Date(),
},
$inc: { publishAttempts: 1 },
$unset: { lastPublishError: '' },
$unset: {
lastPublishError: '',
publishLeaseOwnerIdentifier: '',
publishLeaseExpiresAt: '',
},
},
).exec();
} catch (err) {
await OutboxModel.updateOne(
{
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
},
updateFilter,
{
$inc: { publishAttempts: 1 },
$set: {
lastPublishError: (err as Error)?.message ?? String(err),
},
$unset: {
publishLeaseOwnerIdentifier: '',
publishLeaseExpiresAt: '',
},
},
).exec();
this.logger.error(
@@ -146,40 +172,77 @@ export class DakaoOutboxService {
public async sweepStale(params?: {
staleMs?: number;
publishLeaseDurationMilliseconds?: number;
limit?: number;
}): Promise<number> {
const OutboxModel =
await this.connectionService.getModel('dakaoOutboxMessage');
const currentTimestamp = new Date();
const staleBefore = new Date(
Date.now() - (params?.staleMs ?? 5 * 60 * 1000),
currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000),
);
const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`;
const publishLeaseExpiresAt = new Date(
currentTimestamp.getTime() +
(params?.publishLeaseDurationMilliseconds ??
DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS),
);
const rows = await OutboxModel.find({
state: { $in: ['pending', 'published'] },
updatedAt: { $lte: staleBefore },
$or: [
{ publishLeaseExpiresAt: { $exists: false } },
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
],
})
.sort({ updatedAt: 1 })
.limit(params?.limit ?? 20)
.lean()
.exec();
let swept = 0;
for (const row of rows) {
await this.enqueueAndPublish({
subject: row.subject,
eventId: row.eventId,
const claimedRow = await OutboxModel.findOneAndUpdate(
{
eventId: row.eventId,
state: { $in: ['pending', 'published'] },
$or: [
{ publishLeaseExpiresAt: { $exists: false } },
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
],
},
{
$set: {
publishLeaseOwnerIdentifier,
publishLeaseExpiresAt,
},
},
{ returnDocument: 'after' },
)
.lean()
.exec();
if (!claimedRow) continue;
await this.publishCommand({
subject: claimedRow.subject,
eventId: claimedRow.eventId,
publishLeaseOwnerIdentifier,
envelope: {
schema: 'dakao.command.v1',
eventId: row.eventId,
type: row.type,
eventSource: row.eventSource,
correlationId: row.correlationId,
eventId: claimedRow.eventId,
type: claimedRow.type,
eventSource: claimedRow.eventSource,
correlationId: claimedRow.correlationId,
resultSubjects:
row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type),
payload: row.payload,
claimedRow.resultSubjects ??
this.resolveDefaultResultSubjects(claimedRow.type),
payload: claimedRow.payload,
},
});
swept += 1;
}
return rows.length;
return swept;
}
private resolveDefaultResultSubjects(