fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -1,8 +1,18 @@
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
import { TournamentCodeService } from './tournament-code.service';
const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches';
const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes';
const ENSURE_VENDOR_CODE_POOL_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool';
const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000;
const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000;
@Injectable()
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(TournamentCodeScheduler.name);
@@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private readonly tournamentCodeService: TournamentCodeService,
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
private readonly tournamentResultService: TournamentResultService,
private readonly distributedLockService: DistributedLockService,
) {}
public onModuleInit() {
@@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async attachPending() {
try {
const matches =
await this.tournamentCodeService.attachPendingMatchDocuments();
if (matches.length > 0) {
this.logger.debug(
`Attached tournament codes for ${matches.length} matches.`,
);
for (const match of matches) {
await this.tournamentResultService.scheduleInitialPoll(match);
}
}
await this.distributedLockService.runWithRenewedLock(
{
lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
const matches =
await this.tournamentCodeService.attachPendingMatchDocuments();
if (matches.length > 0) {
this.logger.debug(
`Attached tournament codes for ${matches.length} matches.`,
);
for (const match of matches) {
await this.tournamentResultService.scheduleInitialPoll(match);
}
}
},
);
} catch (error) {
this.logger.warn(
`Failed to attach pending tournament codes: ${
@@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async markExpired() {
try {
await this.tournamentCodeService.markExpired();
await this.distributedLockService.runWithRenewedLock(
{
lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.tournamentCodeService.markExpired();
},
);
} catch (error) {
this.logger.warn(
`Failed to mark expired tournament codes: ${
@@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async ensureVendorPool() {
try {
await this.autoProvisionService.ensurePool();
await this.distributedLockService.runWithRenewedLock(
{
lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.autoProvisionService.ensurePool();
},
);
} catch (error) {
this.logger.warn(
`Failed to auto-provision tournament codes: ${