fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -56,20 +56,25 @@ export interface JetStreamConsumeOptions {
|
||||
maxMessages?: number;
|
||||
expiresMs?: number;
|
||||
nakDelayMs?: number;
|
||||
restartDelayMilliseconds?: number;
|
||||
}
|
||||
|
||||
const JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS = 1_000;
|
||||
|
||||
@Injectable()
|
||||
export class NatsService implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(NatsService.name);
|
||||
private readonly stringCodec = StringCodec();
|
||||
private readonly subscriptions = new Set<Subscription>();
|
||||
private readonly jetStreamConsumers = new Set<ConsumerMessages>();
|
||||
private isDestroying = false;
|
||||
|
||||
constructor(
|
||||
@Inject(NATS_CLIENT) private readonly natsClient: NatsConnection,
|
||||
) {}
|
||||
|
||||
async onModuleDestroy() {
|
||||
this.isDestroying = true;
|
||||
for (const consumer of this.jetStreamConsumers) {
|
||||
try {
|
||||
await consumer.close();
|
||||
@@ -199,28 +204,36 @@ export class NatsService implements OnModuleDestroy {
|
||||
options: JetStreamConsumeOptions,
|
||||
handler: JetStreamMessageHandler<T>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const consumer = await this.natsClient
|
||||
.jetstream()
|
||||
.consumers.get(options.stream, options.durable);
|
||||
const messages = await consumer.consume({
|
||||
max_messages: options.maxMessages ?? 10,
|
||||
expires: options.expiresMs ?? 30000,
|
||||
});
|
||||
this.jetStreamConsumers.add(messages);
|
||||
|
||||
while (!this.isDestroying) {
|
||||
try {
|
||||
for await (const msg of messages) {
|
||||
await this.dispatchJetStream(msg, handler, options);
|
||||
const consumer = await this.natsClient
|
||||
.jetstream()
|
||||
.consumers.get(options.stream, options.durable);
|
||||
const messages = await consumer.consume({
|
||||
max_messages: options.maxMessages ?? 10,
|
||||
expires: options.expiresMs ?? 30000,
|
||||
});
|
||||
this.jetStreamConsumers.add(messages);
|
||||
|
||||
try {
|
||||
for await (const msg of messages) {
|
||||
await this.dispatchJetStream(msg, handler, options);
|
||||
}
|
||||
} finally {
|
||||
this.jetStreamConsumers.delete(messages);
|
||||
}
|
||||
} finally {
|
||||
this.jetStreamConsumers.delete(messages);
|
||||
} catch (err) {
|
||||
if (this.isDestroying) return;
|
||||
this.logger.error(
|
||||
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
|
||||
(err as Error)?.stack ?? err
|
||||
}`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
|
||||
(err as Error)?.stack ?? err
|
||||
}`,
|
||||
|
||||
await this.sleep(
|
||||
options.restartDelayMilliseconds ??
|
||||
JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -313,4 +326,8 @@ export class NatsService implements OnModuleDestroy {
|
||||
? RetentionPolicy.Workqueue
|
||||
: RetentionPolicy.Limits;
|
||||
}
|
||||
|
||||
private sleep(durationMilliseconds: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user