fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -1,8 +1,18 @@
|
||||
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
|
||||
import { TournamentCodeService } from './tournament-code.service';
|
||||
|
||||
const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches';
|
||||
const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes';
|
||||
const ENSURE_VENDOR_CODE_POOL_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool';
|
||||
const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000;
|
||||
const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000;
|
||||
|
||||
@Injectable()
|
||||
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(TournamentCodeScheduler.name);
|
||||
@@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly tournamentCodeService: TournamentCodeService,
|
||||
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
|
||||
private readonly tournamentResultService: TournamentResultService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public onModuleInit() {
|
||||
@@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async attachPending() {
|
||||
try {
|
||||
const matches =
|
||||
await this.tournamentCodeService.attachPendingMatchDocuments();
|
||||
if (matches.length > 0) {
|
||||
this.logger.debug(
|
||||
`Attached tournament codes for ${matches.length} matches.`,
|
||||
);
|
||||
for (const match of matches) {
|
||||
await this.tournamentResultService.scheduleInitialPoll(match);
|
||||
}
|
||||
}
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
const matches =
|
||||
await this.tournamentCodeService.attachPendingMatchDocuments();
|
||||
if (matches.length > 0) {
|
||||
this.logger.debug(
|
||||
`Attached tournament codes for ${matches.length} matches.`,
|
||||
);
|
||||
for (const match of matches) {
|
||||
await this.tournamentResultService.scheduleInitialPoll(match);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to attach pending tournament codes: ${
|
||||
@@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async markExpired() {
|
||||
try {
|
||||
await this.tournamentCodeService.markExpired();
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.tournamentCodeService.markExpired();
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to mark expired tournament codes: ${
|
||||
@@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async ensureVendorPool() {
|
||||
try {
|
||||
await this.autoProvisionService.ensurePool();
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.autoProvisionService.ensurePool();
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to auto-provision tournament codes: ${
|
||||
|
||||
Reference in new issue
Block a user