fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -4,10 +4,13 @@ import {
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { DakaoOutboxService } from './dakao-outbox.service';
|
||||
|
||||
const SWEEP_INTERVAL_MS = 60 * 1000;
|
||||
const STALE_OUTBOX_MS = 5 * 60 * 1000;
|
||||
const SWEEP_LOCK_KEY = 'distributed-lock:dakao-outbox:sweep-stale-publish-leases';
|
||||
const SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS = 55_000;
|
||||
|
||||
@Injectable()
|
||||
export class DakaoOutboxSweeperService
|
||||
@@ -16,7 +19,10 @@ export class DakaoOutboxSweeperService
|
||||
private readonly logger = new Logger(DakaoOutboxSweeperService.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
|
||||
constructor(private readonly outboxService: DakaoOutboxService) {}
|
||||
constructor(
|
||||
private readonly outboxService: DakaoOutboxService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
this.timer = setInterval(() => {
|
||||
@@ -31,12 +37,23 @@ export class DakaoOutboxSweeperService
|
||||
|
||||
private async sweep(): Promise<void> {
|
||||
try {
|
||||
const swept = await this.outboxService.sweepStale({
|
||||
staleMs: STALE_OUTBOX_MS,
|
||||
limit: 20,
|
||||
});
|
||||
if (swept > 0) {
|
||||
this.logger.warn(`Re-published ${swept} stale Dakao outbox messages.`);
|
||||
const swept = await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: SWEEP_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
},
|
||||
async () =>
|
||||
this.outboxService.sweepStale({
|
||||
staleMs: STALE_OUTBOX_MS,
|
||||
limit: 20,
|
||||
}),
|
||||
);
|
||||
const sweptCount = swept ?? 0;
|
||||
if (sweptCount > 0) {
|
||||
this.logger.warn(
|
||||
`Re-published ${sweptCount} stale Dakao outbox messages.`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
|
||||
@@ -70,44 +70,55 @@ describe('DakaoOutboxService', () => {
|
||||
process.env.NODE_ENV = 'staging';
|
||||
|
||||
try {
|
||||
const exec = jest.fn().mockResolvedValue([
|
||||
{
|
||||
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||
type: 'chat.send-message',
|
||||
eventSource: 'lolympus',
|
||||
subject: 'staging.dakao.chat.send-message.v1',
|
||||
correlationId: 'chat-log-1',
|
||||
payload: {
|
||||
chatroomId: '18474390111480678',
|
||||
segments: [{ kind: 'text', text: 'done' }],
|
||||
},
|
||||
const staleRow = {
|
||||
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||
type: 'chat.send-message',
|
||||
eventSource: 'lolympus',
|
||||
subject: 'staging.dakao.chat.send-message.v1',
|
||||
correlationId: 'chat-log-1',
|
||||
payload: {
|
||||
chatroomId: '18474390111480678',
|
||||
segments: [{ kind: 'text', text: 'done' }],
|
||||
},
|
||||
]);
|
||||
};
|
||||
const exec = jest.fn().mockResolvedValue([staleRow]);
|
||||
const lean = jest.fn().mockReturnValue({ exec });
|
||||
const limit = jest.fn().mockReturnValue({ lean });
|
||||
const sort = jest.fn().mockReturnValue({ limit });
|
||||
const claimExec = jest.fn().mockResolvedValue(staleRow);
|
||||
const claimLean = jest.fn().mockReturnValue({ exec: claimExec });
|
||||
const outboxModel = {
|
||||
find: jest.fn().mockReturnValue({ sort }),
|
||||
findOneAndUpdate: jest.fn().mockReturnValue({ lean: claimLean }),
|
||||
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
|
||||
};
|
||||
const { service } = makeService(outboxModel);
|
||||
const enqueue = jest
|
||||
.spyOn(service, 'enqueueAndPublish')
|
||||
.mockResolvedValue(undefined);
|
||||
const { service, natsService } = makeService(outboxModel);
|
||||
|
||||
await expect(service.sweepStale({ staleMs: 1, limit: 1 })).resolves.toBe(
|
||||
1,
|
||||
);
|
||||
|
||||
expect(enqueue).toHaveBeenCalledWith(
|
||||
expect(outboxModel.findOneAndUpdate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
envelope: expect.objectContaining({
|
||||
resultSubjects: {
|
||||
succeeded:
|
||||
'staging.lolympus.dakao.chat.send-message.succeeded.v1',
|
||||
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
|
||||
},
|
||||
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
$set: expect.objectContaining({
|
||||
publishLeaseOwnerIdentifier: expect.any(String),
|
||||
publishLeaseExpiresAt: expect.any(Date),
|
||||
}),
|
||||
}),
|
||||
{ returnDocument: 'after' },
|
||||
);
|
||||
expect(natsService.publishJetStream).toHaveBeenCalledWith(
|
||||
'staging.dakao.chat.send-message.v1',
|
||||
expect.objectContaining({
|
||||
resultSubjects: {
|
||||
succeeded: 'staging.lolympus.dakao.chat.send-message.succeeded.v1',
|
||||
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
|
||||
},
|
||||
}),
|
||||
{ messageId: 'lolympus:chat.send-message:chat-log-1' },
|
||||
);
|
||||
} finally {
|
||||
process.env.NODE_ENV = previousNodeEnv;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { NatsService } from '../global/nats/nats.service';
|
||||
import { resolveDakaoNatsNames } from './dakao.nats-names';
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
|
||||
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
|
||||
const TERMINAL_STATES = ['sent', 'failed'] as const;
|
||||
const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000;
|
||||
|
||||
@Injectable()
|
||||
export class DakaoOutboxService {
|
||||
@@ -50,35 +52,59 @@ export class DakaoOutboxService {
|
||||
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
|
||||
).exec();
|
||||
|
||||
await this.publishCommand({
|
||||
subject: params.subject,
|
||||
eventId: params.eventId,
|
||||
envelope: params.envelope,
|
||||
});
|
||||
}
|
||||
|
||||
private async publishCommand<TPayload>(params: {
|
||||
subject: string;
|
||||
eventId: string;
|
||||
envelope: DakaoCommandEnvelope<TPayload>;
|
||||
publishLeaseOwnerIdentifier?: string;
|
||||
}): Promise<void> {
|
||||
const OutboxModel =
|
||||
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||
const updateFilter = {
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
...(params.publishLeaseOwnerIdentifier
|
||||
? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier }
|
||||
: {}),
|
||||
};
|
||||
try {
|
||||
await this.natsService.publishJetStream(params.subject, params.envelope, {
|
||||
messageId: params.eventId,
|
||||
});
|
||||
await OutboxModel.updateOne(
|
||||
{
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
},
|
||||
updateFilter,
|
||||
{
|
||||
$set: {
|
||||
state: 'published',
|
||||
publishedAt: new Date(),
|
||||
},
|
||||
$inc: { publishAttempts: 1 },
|
||||
$unset: { lastPublishError: '' },
|
||||
$unset: {
|
||||
lastPublishError: '',
|
||||
publishLeaseOwnerIdentifier: '',
|
||||
publishLeaseExpiresAt: '',
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
} catch (err) {
|
||||
await OutboxModel.updateOne(
|
||||
{
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
},
|
||||
updateFilter,
|
||||
{
|
||||
$inc: { publishAttempts: 1 },
|
||||
$set: {
|
||||
lastPublishError: (err as Error)?.message ?? String(err),
|
||||
},
|
||||
$unset: {
|
||||
publishLeaseOwnerIdentifier: '',
|
||||
publishLeaseExpiresAt: '',
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
this.logger.error(
|
||||
@@ -146,40 +172,77 @@ export class DakaoOutboxService {
|
||||
|
||||
public async sweepStale(params?: {
|
||||
staleMs?: number;
|
||||
publishLeaseDurationMilliseconds?: number;
|
||||
limit?: number;
|
||||
}): Promise<number> {
|
||||
const OutboxModel =
|
||||
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||
const currentTimestamp = new Date();
|
||||
const staleBefore = new Date(
|
||||
Date.now() - (params?.staleMs ?? 5 * 60 * 1000),
|
||||
currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000),
|
||||
);
|
||||
const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`;
|
||||
const publishLeaseExpiresAt = new Date(
|
||||
currentTimestamp.getTime() +
|
||||
(params?.publishLeaseDurationMilliseconds ??
|
||||
DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS),
|
||||
);
|
||||
const rows = await OutboxModel.find({
|
||||
state: { $in: ['pending', 'published'] },
|
||||
updatedAt: { $lte: staleBefore },
|
||||
$or: [
|
||||
{ publishLeaseExpiresAt: { $exists: false } },
|
||||
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||
],
|
||||
})
|
||||
.sort({ updatedAt: 1 })
|
||||
.limit(params?.limit ?? 20)
|
||||
.lean()
|
||||
.exec();
|
||||
|
||||
let swept = 0;
|
||||
for (const row of rows) {
|
||||
await this.enqueueAndPublish({
|
||||
subject: row.subject,
|
||||
eventId: row.eventId,
|
||||
const claimedRow = await OutboxModel.findOneAndUpdate(
|
||||
{
|
||||
eventId: row.eventId,
|
||||
state: { $in: ['pending', 'published'] },
|
||||
$or: [
|
||||
{ publishLeaseExpiresAt: { $exists: false } },
|
||||
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||
],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
publishLeaseOwnerIdentifier,
|
||||
publishLeaseExpiresAt,
|
||||
},
|
||||
},
|
||||
{ returnDocument: 'after' },
|
||||
)
|
||||
.lean()
|
||||
.exec();
|
||||
if (!claimedRow) continue;
|
||||
|
||||
await this.publishCommand({
|
||||
subject: claimedRow.subject,
|
||||
eventId: claimedRow.eventId,
|
||||
publishLeaseOwnerIdentifier,
|
||||
envelope: {
|
||||
schema: 'dakao.command.v1',
|
||||
eventId: row.eventId,
|
||||
type: row.type,
|
||||
eventSource: row.eventSource,
|
||||
correlationId: row.correlationId,
|
||||
eventId: claimedRow.eventId,
|
||||
type: claimedRow.type,
|
||||
eventSource: claimedRow.eventSource,
|
||||
correlationId: claimedRow.correlationId,
|
||||
resultSubjects:
|
||||
row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type),
|
||||
payload: row.payload,
|
||||
claimedRow.resultSubjects ??
|
||||
this.resolveDefaultResultSubjects(claimedRow.type),
|
||||
payload: claimedRow.payload,
|
||||
},
|
||||
});
|
||||
swept += 1;
|
||||
}
|
||||
|
||||
return rows.length;
|
||||
return swept;
|
||||
}
|
||||
|
||||
private resolveDefaultResultSubjects(
|
||||
|
||||
Reference in new issue
Block a user