fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -4,10 +4,13 @@ import {
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { DakaoOutboxService } from './dakao-outbox.service';
|
||||
|
||||
const SWEEP_INTERVAL_MS = 60 * 1000;
|
||||
const STALE_OUTBOX_MS = 5 * 60 * 1000;
|
||||
const SWEEP_LOCK_KEY = 'distributed-lock:dakao-outbox:sweep-stale-publish-leases';
|
||||
const SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS = 55_000;
|
||||
|
||||
@Injectable()
|
||||
export class DakaoOutboxSweeperService
|
||||
@@ -16,7 +19,10 @@ export class DakaoOutboxSweeperService
|
||||
private readonly logger = new Logger(DakaoOutboxSweeperService.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
|
||||
constructor(private readonly outboxService: DakaoOutboxService) {}
|
||||
constructor(
|
||||
private readonly outboxService: DakaoOutboxService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
this.timer = setInterval(() => {
|
||||
@@ -31,12 +37,23 @@ export class DakaoOutboxSweeperService
|
||||
|
||||
private async sweep(): Promise<void> {
|
||||
try {
|
||||
const swept = await this.outboxService.sweepStale({
|
||||
staleMs: STALE_OUTBOX_MS,
|
||||
limit: 20,
|
||||
});
|
||||
if (swept > 0) {
|
||||
this.logger.warn(`Re-published ${swept} stale Dakao outbox messages.`);
|
||||
const swept = await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: SWEEP_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
},
|
||||
async () =>
|
||||
this.outboxService.sweepStale({
|
||||
staleMs: STALE_OUTBOX_MS,
|
||||
limit: 20,
|
||||
}),
|
||||
);
|
||||
const sweptCount = swept ?? 0;
|
||||
if (sweptCount > 0) {
|
||||
this.logger.warn(
|
||||
`Re-published ${sweptCount} stale Dakao outbox messages.`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
|
||||
Reference in new issue
Block a user