fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -1,3 +1,4 @@
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { TournamentCodeScheduler } from './tournament-code.scheduler';
describe('TournamentCodeScheduler', () => {
@@ -18,16 +19,28 @@ describe('TournamentCodeScheduler', () => {
scheduleInitialPoll:
params?.scheduleInitialPoll ?? jest.fn().mockResolvedValue(null),
};
const distributedLockService = {
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
} as unknown as jest.Mocked<DistributedLockService>;
const scheduler = new TournamentCodeScheduler(
tournamentCodeService as never,
autoProvisionService as never,
tournamentResultService as never,
distributedLockService as never,
) as unknown as { attachPending(): Promise<void> };
return {
scheduler,
tournamentCodeService,
tournamentResultService,
distributedLockService,
};
}
@@ -59,4 +72,16 @@ describe('TournamentCodeScheduler', () => {
expect(tournamentResultService.scheduleInitialPoll).not.toHaveBeenCalled();
});
it('does not attach pending tournament codes when another instance owns the scheduler lock', async () => {
const { scheduler, tournamentCodeService, distributedLockService } =
createScheduler();
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
await scheduler.attachPending();
expect(
tournamentCodeService.attachPendingMatchDocuments,
).not.toHaveBeenCalled();
});
});
@@ -1,8 +1,18 @@
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
import { TournamentCodeService } from './tournament-code.service';
const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches';
const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes';
const ENSURE_VENDOR_CODE_POOL_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool';
const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000;
const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000;
@Injectable()
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(TournamentCodeScheduler.name);
@@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private readonly tournamentCodeService: TournamentCodeService,
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
private readonly tournamentResultService: TournamentResultService,
private readonly distributedLockService: DistributedLockService,
) {}
public onModuleInit() {
@@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async attachPending() {
try {
const matches =
await this.tournamentCodeService.attachPendingMatchDocuments();
if (matches.length > 0) {
this.logger.debug(
`Attached tournament codes for ${matches.length} matches.`,
);
for (const match of matches) {
await this.tournamentResultService.scheduleInitialPoll(match);
}
}
await this.distributedLockService.runWithRenewedLock(
{
lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
const matches =
await this.tournamentCodeService.attachPendingMatchDocuments();
if (matches.length > 0) {
this.logger.debug(
`Attached tournament codes for ${matches.length} matches.`,
);
for (const match of matches) {
await this.tournamentResultService.scheduleInitialPoll(match);
}
}
},
);
} catch (error) {
this.logger.warn(
`Failed to attach pending tournament codes: ${
@@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async markExpired() {
try {
await this.tournamentCodeService.markExpired();
await this.distributedLockService.runWithRenewedLock(
{
lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.tournamentCodeService.markExpired();
},
);
} catch (error) {
this.logger.warn(
`Failed to mark expired tournament codes: ${
@@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async ensureVendorPool() {
try {
await this.autoProvisionService.ensurePool();
await this.distributedLockService.runWithRenewedLock(
{
lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.autoProvisionService.ensurePool();
},
);
} catch (error) {
this.logger.warn(
`Failed to auto-provision tournament codes: ${