fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { TournamentCodeScheduler } from './tournament-code.scheduler';
|
||||
|
||||
describe('TournamentCodeScheduler', () => {
|
||||
@@ -18,16 +19,28 @@ describe('TournamentCodeScheduler', () => {
|
||||
scheduleInitialPoll:
|
||||
params?.scheduleInitialPoll ?? jest.fn().mockResolvedValue(null),
|
||||
};
|
||||
const distributedLockService = {
|
||||
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||
operation({
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
}),
|
||||
),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const scheduler = new TournamentCodeScheduler(
|
||||
tournamentCodeService as never,
|
||||
autoProvisionService as never,
|
||||
tournamentResultService as never,
|
||||
distributedLockService as never,
|
||||
) as unknown as { attachPending(): Promise<void> };
|
||||
|
||||
return {
|
||||
scheduler,
|
||||
tournamentCodeService,
|
||||
tournamentResultService,
|
||||
distributedLockService,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -59,4 +72,16 @@ describe('TournamentCodeScheduler', () => {
|
||||
|
||||
expect(tournamentResultService.scheduleInitialPoll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not attach pending tournament codes when another instance owns the scheduler lock', async () => {
|
||||
const { scheduler, tournamentCodeService, distributedLockService } =
|
||||
createScheduler();
|
||||
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
|
||||
|
||||
await scheduler.attachPending();
|
||||
|
||||
expect(
|
||||
tournamentCodeService.attachPendingMatchDocuments,
|
||||
).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,18 @@
|
||||
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
|
||||
import { TournamentCodeService } from './tournament-code.service';
|
||||
|
||||
const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches';
|
||||
const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes';
|
||||
const ENSURE_VENDOR_CODE_POOL_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool';
|
||||
const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000;
|
||||
const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000;
|
||||
|
||||
@Injectable()
|
||||
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(TournamentCodeScheduler.name);
|
||||
@@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly tournamentCodeService: TournamentCodeService,
|
||||
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
|
||||
private readonly tournamentResultService: TournamentResultService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public onModuleInit() {
|
||||
@@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async attachPending() {
|
||||
try {
|
||||
const matches =
|
||||
await this.tournamentCodeService.attachPendingMatchDocuments();
|
||||
if (matches.length > 0) {
|
||||
this.logger.debug(
|
||||
`Attached tournament codes for ${matches.length} matches.`,
|
||||
);
|
||||
for (const match of matches) {
|
||||
await this.tournamentResultService.scheduleInitialPoll(match);
|
||||
}
|
||||
}
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
const matches =
|
||||
await this.tournamentCodeService.attachPendingMatchDocuments();
|
||||
if (matches.length > 0) {
|
||||
this.logger.debug(
|
||||
`Attached tournament codes for ${matches.length} matches.`,
|
||||
);
|
||||
for (const match of matches) {
|
||||
await this.tournamentResultService.scheduleInitialPoll(match);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to attach pending tournament codes: ${
|
||||
@@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async markExpired() {
|
||||
try {
|
||||
await this.tournamentCodeService.markExpired();
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.tournamentCodeService.markExpired();
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to mark expired tournament codes: ${
|
||||
@@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async ensureVendorPool() {
|
||||
try {
|
||||
await this.autoProvisionService.ensurePool();
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.autoProvisionService.ensurePool();
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to auto-provision tournament codes: ${
|
||||
|
||||
Reference in new issue
Block a user