fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -11,6 +11,7 @@ import {
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import type {
|
||||
MatchDocument,
|
||||
MatchGameResult,
|
||||
@@ -46,6 +47,12 @@ const POLL_EXPIRY_MS = 24 * 60 * 60_000;
|
||||
const EXPAND_AFTER_MS = 60 * 60_000;
|
||||
const EXPANDED_MEMBER_LIMIT_PER_TEAM = 2;
|
||||
const TOURNAMENT_LOOKUP_START_MARGIN_SECONDS = 60;
|
||||
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS =
|
||||
30 * 60_000;
|
||||
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS =
|
||||
5 * 60_000;
|
||||
const TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS =
|
||||
5 * 60_000;
|
||||
|
||||
interface ManualGameAssignment {
|
||||
gameNumber: number;
|
||||
@@ -64,11 +71,29 @@ export class TournamentResultService {
|
||||
private readonly riotJobQueueService: RiotJobQueueService,
|
||||
private readonly communityStatsService: CommunityStatsService,
|
||||
private readonly matchStatsService: MatchStatsService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public async scheduleInitialPoll(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
priority: RiotJobPriority = 'high',
|
||||
): Promise<EnqueueTournamentResultPollResult | null> {
|
||||
const matchIdentifier = this.resolveMatchIdentifier(matchOrId);
|
||||
return this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: this.tournamentResultPollProcessingLockKey(matchIdentifier),
|
||||
lockExpirationDurationMilliseconds:
|
||||
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockAcquireTimeoutMilliseconds: 1_000,
|
||||
lockRetryDelayMilliseconds: 100,
|
||||
},
|
||||
async () => this.scheduleInitialPollWhileLockOwned(matchOrId, priority),
|
||||
);
|
||||
}
|
||||
|
||||
private async scheduleInitialPollWhileLockOwned(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
priority: RiotJobPriority,
|
||||
): Promise<EnqueueTournamentResultPollResult | null> {
|
||||
const match = await this.loadMatch(matchOrId);
|
||||
if (match.status === 'finished') return null;
|
||||
@@ -195,6 +220,29 @@ export class TournamentResultService {
|
||||
}
|
||||
|
||||
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
|
||||
const didProcessPollJob = await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: this.tournamentResultPollProcessingLockKey(job.matchId),
|
||||
lockExpirationDurationMilliseconds:
|
||||
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.processPollJobWhileLockOwned(job);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
if (didProcessPollJob) return;
|
||||
this.logger.debug(
|
||||
`Skipped tournament result poll job ${job.jobId}; another worker is processing match ${job.matchId}.`,
|
||||
);
|
||||
}
|
||||
|
||||
private async processPollJobWhileLockOwned(
|
||||
job: PollTournamentResultJob,
|
||||
): Promise<void> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
@@ -818,12 +866,28 @@ export class TournamentResultService {
|
||||
|
||||
private async reserveReadyDoubleEliminationCodes(
|
||||
match: MatchDocument,
|
||||
): Promise<void> {
|
||||
await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: this.tournamentResultReadyCodeReservationLockKey(
|
||||
match._id.toString(),
|
||||
),
|
||||
lockExpirationDurationMilliseconds:
|
||||
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
},
|
||||
async () => this.reserveReadyDoubleEliminationCodesWhileLockOwned(match),
|
||||
);
|
||||
}
|
||||
|
||||
private async reserveReadyDoubleEliminationCodesWhileLockOwned(
|
||||
match: MatchDocument,
|
||||
): Promise<void> {
|
||||
if ((match.gameMode ?? 'pair') !== 'double-elimination') return;
|
||||
const readyBracketMatches = (match.doubleElimination?.bracketMatches ?? [])
|
||||
.filter((bracketMatch) => bracketMatch.state === 'ready');
|
||||
if (readyBracketMatches.length === 0) return;
|
||||
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
const existingCodes =
|
||||
@@ -871,8 +935,19 @@ export class TournamentResultService {
|
||||
}
|
||||
|
||||
if (reservedIds.length === 0) return;
|
||||
match.tournamentCodeIds.push(...reservedIds);
|
||||
await match.save();
|
||||
await MatchModel.updateOne(
|
||||
{ _id: match._id },
|
||||
{ $addToSet: { tournamentCodeIds: { $each: reservedIds } } },
|
||||
).exec();
|
||||
const existingTournamentCodeIds = new Set(
|
||||
match.tournamentCodeIds.map((tournamentCodeId) =>
|
||||
tournamentCodeId.toString(),
|
||||
),
|
||||
);
|
||||
for (const reservedId of reservedIds) {
|
||||
if (existingTournamentCodeIds.has(reservedId.toString())) continue;
|
||||
match.tournamentCodeIds.push(reservedId);
|
||||
}
|
||||
}
|
||||
|
||||
private async markTournamentCodeUsedForManualResult(
|
||||
@@ -1126,6 +1201,24 @@ export class TournamentResultService {
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
return match;
|
||||
}
|
||||
|
||||
private resolveMatchIdentifier(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
): string {
|
||||
return matchOrId instanceof mongoose.Types.ObjectId
|
||||
? matchOrId.toString()
|
||||
: matchOrId._id.toString();
|
||||
}
|
||||
|
||||
private tournamentResultPollProcessingLockKey(matchIdentifier: string): string {
|
||||
return `distributed-lock:tournament-result:process-poll-job:match:${matchIdentifier}`;
|
||||
}
|
||||
|
||||
private tournamentResultReadyCodeReservationLockKey(
|
||||
matchIdentifier: string,
|
||||
): string {
|
||||
return `distributed-lock:tournament-result:reserve-ready-double-elimination-codes:match:${matchIdentifier}`;
|
||||
}
|
||||
}
|
||||
|
||||
function isRiotMatchDto(value: unknown): value is RiotMatchDto {
|
||||
|
||||
Reference in new issue
Block a user