fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -11,6 +11,7 @@ import {
import { MatchStatsService } from '../community-stats/match-stats.service';
import { CommunityStatsService } from '../community-stats/community-stats.service';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import type {
MatchDocument,
MatchGameResult,
@@ -46,6 +47,12 @@ const POLL_EXPIRY_MS = 24 * 60 * 60_000;
const EXPAND_AFTER_MS = 60 * 60_000;
const EXPANDED_MEMBER_LIMIT_PER_TEAM = 2;
const TOURNAMENT_LOOKUP_START_MARGIN_SECONDS = 60;
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS =
30 * 60_000;
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS =
5 * 60_000;
const TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS =
5 * 60_000;
interface ManualGameAssignment {
gameNumber: number;
@@ -64,11 +71,29 @@ export class TournamentResultService {
private readonly riotJobQueueService: RiotJobQueueService,
private readonly communityStatsService: CommunityStatsService,
private readonly matchStatsService: MatchStatsService,
private readonly distributedLockService: DistributedLockService,
) {}
public async scheduleInitialPoll(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
priority: RiotJobPriority = 'high',
): Promise<EnqueueTournamentResultPollResult | null> {
const matchIdentifier = this.resolveMatchIdentifier(matchOrId);
return this.distributedLockService.runWithLock(
{
lockKey: this.tournamentResultPollProcessingLockKey(matchIdentifier),
lockExpirationDurationMilliseconds:
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockAcquireTimeoutMilliseconds: 1_000,
lockRetryDelayMilliseconds: 100,
},
async () => this.scheduleInitialPollWhileLockOwned(matchOrId, priority),
);
}
private async scheduleInitialPollWhileLockOwned(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
priority: RiotJobPriority,
): Promise<EnqueueTournamentResultPollResult | null> {
const match = await this.loadMatch(matchOrId);
if (match.status === 'finished') return null;
@@ -195,6 +220,29 @@ export class TournamentResultService {
}
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
const didProcessPollJob = await this.distributedLockService.runWithRenewedLock(
{
lockKey: this.tournamentResultPollProcessingLockKey(job.matchId),
lockExpirationDurationMilliseconds:
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.processPollJobWhileLockOwned(job);
return true;
},
);
if (didProcessPollJob) return;
this.logger.debug(
`Skipped tournament result poll job ${job.jobId}; another worker is processing match ${job.matchId}.`,
);
}
private async processPollJobWhileLockOwned(
job: PollTournamentResultJob,
): Promise<void> {
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
@@ -818,12 +866,28 @@ export class TournamentResultService {
private async reserveReadyDoubleEliminationCodes(
match: MatchDocument,
): Promise<void> {
await this.distributedLockService.runWithLock(
{
lockKey: this.tournamentResultReadyCodeReservationLockKey(
match._id.toString(),
),
lockExpirationDurationMilliseconds:
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
},
async () => this.reserveReadyDoubleEliminationCodesWhileLockOwned(match),
);
}
private async reserveReadyDoubleEliminationCodesWhileLockOwned(
match: MatchDocument,
): Promise<void> {
if ((match.gameMode ?? 'pair') !== 'double-elimination') return;
const readyBracketMatches = (match.doubleElimination?.bracketMatches ?? [])
.filter((bracketMatch) => bracketMatch.state === 'ready');
if (readyBracketMatches.length === 0) return;
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
const existingCodes =
@@ -871,8 +935,19 @@ export class TournamentResultService {
}
if (reservedIds.length === 0) return;
match.tournamentCodeIds.push(...reservedIds);
await match.save();
await MatchModel.updateOne(
{ _id: match._id },
{ $addToSet: { tournamentCodeIds: { $each: reservedIds } } },
).exec();
const existingTournamentCodeIds = new Set(
match.tournamentCodeIds.map((tournamentCodeId) =>
tournamentCodeId.toString(),
),
);
for (const reservedId of reservedIds) {
if (existingTournamentCodeIds.has(reservedId.toString())) continue;
match.tournamentCodeIds.push(reservedId);
}
}
private async markTournamentCodeUsedForManualResult(
@@ -1126,6 +1201,24 @@ export class TournamentResultService {
if (!match) throw new NotFoundException('Match not found.');
return match;
}
private resolveMatchIdentifier(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
): string {
return matchOrId instanceof mongoose.Types.ObjectId
? matchOrId.toString()
: matchOrId._id.toString();
}
private tournamentResultPollProcessingLockKey(matchIdentifier: string): string {
return `distributed-lock:tournament-result:process-poll-job:match:${matchIdentifier}`;
}
private tournamentResultReadyCodeReservationLockKey(
matchIdentifier: string,
): string {
return `distributed-lock:tournament-result:reserve-ready-double-elimination-codes:match:${matchIdentifier}`;
}
}
function isRiotMatchDto(value: unknown): value is RiotMatchDto {