fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -56,20 +56,25 @@ export interface JetStreamConsumeOptions {
maxMessages?: number;
expiresMs?: number;
nakDelayMs?: number;
restartDelayMilliseconds?: number;
}
const JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS = 1_000;
@Injectable()
export class NatsService implements OnModuleDestroy {
private readonly logger = new Logger(NatsService.name);
private readonly stringCodec = StringCodec();
private readonly subscriptions = new Set<Subscription>();
private readonly jetStreamConsumers = new Set<ConsumerMessages>();
private isDestroying = false;
constructor(
@Inject(NATS_CLIENT) private readonly natsClient: NatsConnection,
) {}
async onModuleDestroy() {
this.isDestroying = true;
for (const consumer of this.jetStreamConsumers) {
try {
await consumer.close();
@@ -199,28 +204,36 @@ export class NatsService implements OnModuleDestroy {
options: JetStreamConsumeOptions,
handler: JetStreamMessageHandler<T>,
): Promise<void> {
try {
const consumer = await this.natsClient
.jetstream()
.consumers.get(options.stream, options.durable);
const messages = await consumer.consume({
max_messages: options.maxMessages ?? 10,
expires: options.expiresMs ?? 30000,
});
this.jetStreamConsumers.add(messages);
while (!this.isDestroying) {
try {
for await (const msg of messages) {
await this.dispatchJetStream(msg, handler, options);
const consumer = await this.natsClient
.jetstream()
.consumers.get(options.stream, options.durable);
const messages = await consumer.consume({
max_messages: options.maxMessages ?? 10,
expires: options.expiresMs ?? 30000,
});
this.jetStreamConsumers.add(messages);
try {
for await (const msg of messages) {
await this.dispatchJetStream(msg, handler, options);
}
} finally {
this.jetStreamConsumers.delete(messages);
}
} finally {
this.jetStreamConsumers.delete(messages);
} catch (err) {
if (this.isDestroying) return;
this.logger.error(
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
(err as Error)?.stack ?? err
}`,
);
}
} catch (err) {
this.logger.error(
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
(err as Error)?.stack ?? err
}`,
await this.sleep(
options.restartDelayMilliseconds ??
JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS,
);
}
}
@@ -313,4 +326,8 @@ export class NatsService implements OnModuleDestroy {
? RetentionPolicy.Workqueue
: RetentionPolicy.Limits;
}
private sleep(durationMilliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
}
}