fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { NatsService } from '../global/nats/nats.service';
|
||||
import { resolveDakaoNatsNames } from './dakao.nats-names';
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
|
||||
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
|
||||
const TERMINAL_STATES = ['sent', 'failed'] as const;
|
||||
const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000;
|
||||
|
||||
@Injectable()
|
||||
export class DakaoOutboxService {
|
||||
@@ -50,35 +52,59 @@ export class DakaoOutboxService {
|
||||
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
|
||||
).exec();
|
||||
|
||||
await this.publishCommand({
|
||||
subject: params.subject,
|
||||
eventId: params.eventId,
|
||||
envelope: params.envelope,
|
||||
});
|
||||
}
|
||||
|
||||
private async publishCommand<TPayload>(params: {
|
||||
subject: string;
|
||||
eventId: string;
|
||||
envelope: DakaoCommandEnvelope<TPayload>;
|
||||
publishLeaseOwnerIdentifier?: string;
|
||||
}): Promise<void> {
|
||||
const OutboxModel =
|
||||
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||
const updateFilter = {
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
...(params.publishLeaseOwnerIdentifier
|
||||
? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier }
|
||||
: {}),
|
||||
};
|
||||
try {
|
||||
await this.natsService.publishJetStream(params.subject, params.envelope, {
|
||||
messageId: params.eventId,
|
||||
});
|
||||
await OutboxModel.updateOne(
|
||||
{
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
},
|
||||
updateFilter,
|
||||
{
|
||||
$set: {
|
||||
state: 'published',
|
||||
publishedAt: new Date(),
|
||||
},
|
||||
$inc: { publishAttempts: 1 },
|
||||
$unset: { lastPublishError: '' },
|
||||
$unset: {
|
||||
lastPublishError: '',
|
||||
publishLeaseOwnerIdentifier: '',
|
||||
publishLeaseExpiresAt: '',
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
} catch (err) {
|
||||
await OutboxModel.updateOne(
|
||||
{
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
},
|
||||
updateFilter,
|
||||
{
|
||||
$inc: { publishAttempts: 1 },
|
||||
$set: {
|
||||
lastPublishError: (err as Error)?.message ?? String(err),
|
||||
},
|
||||
$unset: {
|
||||
publishLeaseOwnerIdentifier: '',
|
||||
publishLeaseExpiresAt: '',
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
this.logger.error(
|
||||
@@ -146,40 +172,77 @@ export class DakaoOutboxService {
|
||||
|
||||
public async sweepStale(params?: {
|
||||
staleMs?: number;
|
||||
publishLeaseDurationMilliseconds?: number;
|
||||
limit?: number;
|
||||
}): Promise<number> {
|
||||
const OutboxModel =
|
||||
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||
const currentTimestamp = new Date();
|
||||
const staleBefore = new Date(
|
||||
Date.now() - (params?.staleMs ?? 5 * 60 * 1000),
|
||||
currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000),
|
||||
);
|
||||
const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`;
|
||||
const publishLeaseExpiresAt = new Date(
|
||||
currentTimestamp.getTime() +
|
||||
(params?.publishLeaseDurationMilliseconds ??
|
||||
DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS),
|
||||
);
|
||||
const rows = await OutboxModel.find({
|
||||
state: { $in: ['pending', 'published'] },
|
||||
updatedAt: { $lte: staleBefore },
|
||||
$or: [
|
||||
{ publishLeaseExpiresAt: { $exists: false } },
|
||||
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||
],
|
||||
})
|
||||
.sort({ updatedAt: 1 })
|
||||
.limit(params?.limit ?? 20)
|
||||
.lean()
|
||||
.exec();
|
||||
|
||||
let swept = 0;
|
||||
for (const row of rows) {
|
||||
await this.enqueueAndPublish({
|
||||
subject: row.subject,
|
||||
eventId: row.eventId,
|
||||
const claimedRow = await OutboxModel.findOneAndUpdate(
|
||||
{
|
||||
eventId: row.eventId,
|
||||
state: { $in: ['pending', 'published'] },
|
||||
$or: [
|
||||
{ publishLeaseExpiresAt: { $exists: false } },
|
||||
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||
],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
publishLeaseOwnerIdentifier,
|
||||
publishLeaseExpiresAt,
|
||||
},
|
||||
},
|
||||
{ returnDocument: 'after' },
|
||||
)
|
||||
.lean()
|
||||
.exec();
|
||||
if (!claimedRow) continue;
|
||||
|
||||
await this.publishCommand({
|
||||
subject: claimedRow.subject,
|
||||
eventId: claimedRow.eventId,
|
||||
publishLeaseOwnerIdentifier,
|
||||
envelope: {
|
||||
schema: 'dakao.command.v1',
|
||||
eventId: row.eventId,
|
||||
type: row.type,
|
||||
eventSource: row.eventSource,
|
||||
correlationId: row.correlationId,
|
||||
eventId: claimedRow.eventId,
|
||||
type: claimedRow.type,
|
||||
eventSource: claimedRow.eventSource,
|
||||
correlationId: claimedRow.correlationId,
|
||||
resultSubjects:
|
||||
row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type),
|
||||
payload: row.payload,
|
||||
claimedRow.resultSubjects ??
|
||||
this.resolveDefaultResultSubjects(claimedRow.type),
|
||||
payload: claimedRow.payload,
|
||||
},
|
||||
});
|
||||
swept += 1;
|
||||
}
|
||||
|
||||
return rows.length;
|
||||
return swept;
|
||||
}
|
||||
|
||||
private resolveDefaultResultSubjects(
|
||||
|
||||
Reference in new issue
Block a user