Harden match and tournament code concurrency
This commit is contained in:
1 parent
292e37aaac
commit
df7dcd3a9c
9 files changed
+2339
-128
No files matched your search
@@ -1,13 +1,17 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
Injectable,
|
||||
InternalServerErrorException,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import mongoose from 'mongoose';
|
||||
import {
|
||||
buildDoubleEliminationBracket,
|
||||
findNextReadyDoubleEliminationBracketMatches,
|
||||
} from '../double-elimination/double-elimination.util';
|
||||
import { AuctionRedisService } from '../auction/redis/auction-redis.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import type {
|
||||
DoubleEliminationBracketMatch,
|
||||
@@ -36,10 +40,15 @@ import {
|
||||
|
||||
const DEFAULT_REGION = 'kr';
|
||||
const DEFAULT_SPECTATOR_TYPE: TournamentSpectatorType = 'ALL';
|
||||
const TOURNAMENT_CODE_MATCH_LOCK_TTL_MS = 30_000;
|
||||
const TOURNAMENT_CODE_TRANSACTION_ATTEMPTS = 3;
|
||||
|
||||
@Injectable()
|
||||
export class TournamentCodeService {
|
||||
constructor(private readonly connectionService: ConnectionService) {}
|
||||
constructor(
|
||||
private readonly connectionService: ConnectionService,
|
||||
private readonly auctionRedisService: AuctionRedisService,
|
||||
) {}
|
||||
|
||||
public async bulkUpload(params: {
|
||||
communityId?: mongoose.Types.ObjectId;
|
||||
@@ -231,21 +240,34 @@ export class TournamentCodeService {
|
||||
|
||||
public async reserveForMatch(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
): Promise<MatchDocument> {
|
||||
const matchId = this.resolveMatchId(matchOrId);
|
||||
return this.withLockedMatchTransaction(matchId, (session) =>
|
||||
this.reserveForMatchInTransaction(matchId, session),
|
||||
);
|
||||
}
|
||||
|
||||
private async reserveForMatchInTransaction(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
session: mongoose.ClientSession,
|
||||
): Promise<MatchDocument> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
const match =
|
||||
matchOrId instanceof mongoose.Types.ObjectId
|
||||
? await MatchModel.findById(matchOrId)
|
||||
: await MatchModel.findById(matchOrId._id);
|
||||
const match = await this.execQuery<MatchDocument | null>(
|
||||
this.withQuerySession(MatchModel.findById(matchId), session),
|
||||
);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
|
||||
match.gameCount = match.gameCount || DEFAULT_TOURNAMENT_GAME_COUNT;
|
||||
match.tournamentCodeIds = match.tournamentCodeIds ?? [];
|
||||
this.ensureGameSchedule(match);
|
||||
if ((match.gameMode ?? 'pair') === 'double-elimination') {
|
||||
return this.reserveForDoubleEliminationMatch(match, TournamentCodeModel);
|
||||
return this.reserveForDoubleEliminationMatch(
|
||||
match,
|
||||
TournamentCodeModel,
|
||||
session,
|
||||
);
|
||||
}
|
||||
const shouldSaveMetadata =
|
||||
this.isMatchModified(match, 'gameCount') ||
|
||||
@@ -281,7 +303,7 @@ export class TournamentCodeService {
|
||||
},
|
||||
$unset: { releasedAt: 1 },
|
||||
},
|
||||
{ sort: { expiresAt: 1 }, returnDocument: 'after' },
|
||||
{ sort: { expiresAt: 1 }, returnDocument: 'after', session },
|
||||
).exec();
|
||||
if (!code) break;
|
||||
reservedIds.push(code._id);
|
||||
@@ -291,7 +313,7 @@ export class TournamentCodeService {
|
||||
match.tournamentCodeIds.push(...reservedIds);
|
||||
}
|
||||
if (reservedIds.length > 0 || shouldSaveMetadata) {
|
||||
await match.save();
|
||||
await match.save({ session });
|
||||
}
|
||||
return match;
|
||||
}
|
||||
@@ -301,13 +323,31 @@ export class TournamentCodeService {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
gameNumber: number;
|
||||
}): Promise<TournamentCodeSlotView> {
|
||||
return this.withLockedMatchTransaction(params.matchId, (session) =>
|
||||
this.replaceForMatchGameInTransaction(params, session),
|
||||
);
|
||||
}
|
||||
|
||||
private async replaceForMatchGameInTransaction(
|
||||
params: {
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
gameNumber: number;
|
||||
},
|
||||
session: mongoose.ClientSession,
|
||||
): Promise<TournamentCodeSlotView> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
const match = await MatchModel.findOne({
|
||||
_id: params.matchId,
|
||||
communityId: params.communityId,
|
||||
}).exec();
|
||||
const match = await this.execQuery<MatchDocument | null>(
|
||||
this.withQuerySession(
|
||||
MatchModel.findOne({
|
||||
_id: params.matchId,
|
||||
communityId: params.communityId,
|
||||
}),
|
||||
session,
|
||||
),
|
||||
);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
if (match.status !== 'team-built' && match.status !== 'in-progress') {
|
||||
throw new BadRequestException(
|
||||
@@ -348,13 +388,18 @@ export class TournamentCodeService {
|
||||
);
|
||||
}
|
||||
|
||||
const currentCode = await TournamentCodeModel.findOne({
|
||||
_id: { $in: match.tournamentCodeIds },
|
||||
matchId: match._id,
|
||||
...(doubleEliminationBracketMatch
|
||||
? { bracketMatchId: doubleEliminationBracketMatch.bracketMatchId }
|
||||
: { gameNumber: params.gameNumber }),
|
||||
}).exec();
|
||||
const currentCode = await this.execQuery<TournamentCodeDocument | null>(
|
||||
this.withQuerySession(
|
||||
TournamentCodeModel.findOne({
|
||||
_id: { $in: match.tournamentCodeIds },
|
||||
matchId: match._id,
|
||||
...(doubleEliminationBracketMatch
|
||||
? { bracketMatchId: doubleEliminationBracketMatch.bracketMatchId }
|
||||
: { gameNumber: params.gameNumber }),
|
||||
}),
|
||||
session,
|
||||
),
|
||||
);
|
||||
if (!currentCode) {
|
||||
throw new NotFoundException('Assigned tournament code not found.');
|
||||
}
|
||||
@@ -367,6 +412,25 @@ export class TournamentCodeService {
|
||||
const expiresAfter = new Date(
|
||||
Date.now() + TOURNAMENT_CODE_EXPIRY_MARGIN_MS,
|
||||
);
|
||||
await this.execQuery(
|
||||
this.withQuerySession(
|
||||
TournamentCodeModel.updateOne(
|
||||
{ _id: currentCode._id },
|
||||
{
|
||||
$set: { state: 'released', releasedAt: new Date() },
|
||||
$unset: {
|
||||
matchId: 1,
|
||||
gameNumber: 1,
|
||||
bracketMatchId: 1,
|
||||
reservedAt: 1,
|
||||
riotMatchId: 1,
|
||||
},
|
||||
},
|
||||
),
|
||||
session,
|
||||
),
|
||||
);
|
||||
|
||||
const replacement = await TournamentCodeModel.findOneAndUpdate(
|
||||
{
|
||||
state: 'available',
|
||||
@@ -389,7 +453,7 @@ export class TournamentCodeService {
|
||||
},
|
||||
$unset: { releasedAt: 1 },
|
||||
},
|
||||
{ sort: { expiresAt: 1 }, returnDocument: 'after' },
|
||||
{ sort: { expiresAt: 1 }, returnDocument: 'after', session },
|
||||
).exec();
|
||||
if (!replacement) {
|
||||
throw new BadRequestException(
|
||||
@@ -397,20 +461,12 @@ export class TournamentCodeService {
|
||||
);
|
||||
}
|
||||
|
||||
currentCode.state = 'released';
|
||||
currentCode.matchId = undefined;
|
||||
currentCode.gameNumber = undefined;
|
||||
currentCode.bracketMatchId = undefined;
|
||||
currentCode.reservedAt = undefined;
|
||||
currentCode.releasedAt = new Date();
|
||||
currentCode.riotMatchId = undefined;
|
||||
match.tournamentCodeIds = this.replaceTournamentCodeId(
|
||||
match.tournamentCodeIds,
|
||||
currentCode._id,
|
||||
replacement._id,
|
||||
);
|
||||
await currentCode.save();
|
||||
await match.save();
|
||||
await match.save({ session });
|
||||
|
||||
return {
|
||||
gameNumber: params.gameNumber,
|
||||
@@ -431,32 +487,46 @@ export class TournamentCodeService {
|
||||
|
||||
public async releaseFromMatch(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
): Promise<void> {
|
||||
const matchId = this.resolveMatchId(matchOrId);
|
||||
await this.withLockedMatchTransaction(matchId, (session) =>
|
||||
this.releaseFromMatchInTransaction(matchId, session),
|
||||
);
|
||||
}
|
||||
|
||||
private async releaseFromMatchInTransaction(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
session: mongoose.ClientSession,
|
||||
): Promise<void> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
const match =
|
||||
matchOrId instanceof mongoose.Types.ObjectId
|
||||
? await MatchModel.findById(matchOrId)
|
||||
: await MatchModel.findById(matchOrId._id);
|
||||
const match = await this.execQuery<MatchDocument | null>(
|
||||
this.withQuerySession(MatchModel.findById(matchId), session),
|
||||
);
|
||||
if (!match) return;
|
||||
if (match.tournamentCodeIds.length === 0) return;
|
||||
|
||||
await TournamentCodeModel.updateMany(
|
||||
{ _id: { $in: match.tournamentCodeIds }, state: 'reserved' },
|
||||
{
|
||||
$set: { state: 'released', releasedAt: new Date() },
|
||||
$unset: {
|
||||
matchId: 1,
|
||||
gameNumber: 1,
|
||||
bracketMatchId: 1,
|
||||
reservedAt: 1,
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
await this.execQuery(
|
||||
this.withQuerySession(
|
||||
TournamentCodeModel.updateMany(
|
||||
{ _id: { $in: match.tournamentCodeIds }, state: 'reserved' },
|
||||
{
|
||||
$set: { state: 'released', releasedAt: new Date() },
|
||||
$unset: {
|
||||
matchId: 1,
|
||||
gameNumber: 1,
|
||||
bracketMatchId: 1,
|
||||
reservedAt: 1,
|
||||
},
|
||||
},
|
||||
),
|
||||
session,
|
||||
),
|
||||
);
|
||||
|
||||
match.tournamentCodeIds = [];
|
||||
await match.save();
|
||||
await match.save({ session });
|
||||
}
|
||||
|
||||
public async markExpired(): Promise<number> {
|
||||
@@ -589,18 +659,128 @@ export class TournamentCodeService {
|
||||
return next;
|
||||
}
|
||||
|
||||
private resolveMatchId(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
): mongoose.Types.ObjectId {
|
||||
return matchOrId instanceof mongoose.Types.ObjectId
|
||||
? matchOrId
|
||||
: matchOrId._id;
|
||||
}
|
||||
|
||||
private async withLockedMatchTransaction<T>(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
operation: (session: mongoose.ClientSession) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const lockKey = this.matchCodeLockKey(matchId);
|
||||
const token = randomUUID();
|
||||
const acquired = await this.auctionRedisService.acquireTokenLock(
|
||||
lockKey,
|
||||
token,
|
||||
TOURNAMENT_CODE_MATCH_LOCK_TTL_MS,
|
||||
);
|
||||
if (!acquired) {
|
||||
throw new ConflictException(
|
||||
'Tournament code operation is already running for this match.',
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await this.runTransactionWithRetry(operation);
|
||||
} finally {
|
||||
await this.auctionRedisService.releaseTokenLock(lockKey, token);
|
||||
}
|
||||
}
|
||||
|
||||
private async runTransactionWithRetry<T>(
|
||||
operation: (session: mongoose.ClientSession) => Promise<T>,
|
||||
): Promise<T> {
|
||||
let lastError: unknown;
|
||||
for (
|
||||
let attempt = 1;
|
||||
attempt <= TOURNAMENT_CODE_TRANSACTION_ATTEMPTS;
|
||||
attempt += 1
|
||||
) {
|
||||
const session = await this.connectionService.createSession();
|
||||
if (!session) {
|
||||
throw new InternalServerErrorException('Failed to start DB session.');
|
||||
}
|
||||
|
||||
try {
|
||||
session.startTransaction();
|
||||
const result = await operation(session);
|
||||
await session.commitTransaction();
|
||||
return result;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await session.abortTransaction().catch(() => undefined);
|
||||
if (
|
||||
attempt >= TOURNAMENT_CODE_TRANSACTION_ATTEMPTS ||
|
||||
!this.isRetryableTransactionError(error)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
await session.endSession().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
private isRetryableTransactionError(error: unknown): boolean {
|
||||
const candidate = error as {
|
||||
code?: number;
|
||||
codeName?: string;
|
||||
hasErrorLabel?: (label: string) => boolean;
|
||||
};
|
||||
return Boolean(
|
||||
candidate?.hasErrorLabel?.('TransientTransactionError') ||
|
||||
candidate?.hasErrorLabel?.('UnknownTransactionCommitResult') ||
|
||||
candidate?.code === 112 ||
|
||||
candidate?.codeName === 'WriteConflict',
|
||||
);
|
||||
}
|
||||
|
||||
private matchCodeLockKey(matchId: mongoose.Types.ObjectId): string {
|
||||
return `tournament-code:match:${matchId.toString()}:lock`;
|
||||
}
|
||||
|
||||
private withQuerySession<T>(query: T, session: mongoose.ClientSession): T {
|
||||
const maybeQuery = query as {
|
||||
session?: (session: mongoose.ClientSession) => T;
|
||||
};
|
||||
if (maybeQuery && typeof maybeQuery.session === 'function') {
|
||||
return maybeQuery.session(session);
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
private async execQuery<T = unknown>(query: unknown): Promise<T> {
|
||||
const maybeQuery = query as { exec?: () => Promise<T> };
|
||||
if (maybeQuery && typeof maybeQuery.exec === 'function') {
|
||||
return maybeQuery.exec();
|
||||
}
|
||||
return query as Promise<T>;
|
||||
}
|
||||
|
||||
private async reserveForDoubleEliminationMatch(
|
||||
match: MatchDocument,
|
||||
TournamentCodeModel: mongoose.Model<TournamentCode>,
|
||||
session: mongoose.ClientSession,
|
||||
): Promise<MatchDocument> {
|
||||
const readyBracketMatches = findNextReadyDoubleEliminationBracketMatches(
|
||||
match.doubleElimination,
|
||||
);
|
||||
const existingCodes =
|
||||
match.tournamentCodeIds.length > 0
|
||||
? await TournamentCodeModel.find({
|
||||
_id: { $in: match.tournamentCodeIds },
|
||||
}).exec()
|
||||
? await this.execQuery<TournamentCodeDocument[]>(
|
||||
this.withQuerySession(
|
||||
TournamentCodeModel.find({
|
||||
_id: { $in: match.tournamentCodeIds },
|
||||
}),
|
||||
session,
|
||||
),
|
||||
)
|
||||
: [];
|
||||
const byBracketMatchId = new Map(
|
||||
existingCodes
|
||||
@@ -636,7 +816,7 @@ export class TournamentCodeService {
|
||||
},
|
||||
$unset: { releasedAt: 1 },
|
||||
},
|
||||
{ sort: { expiresAt: 1 }, returnDocument: 'after' },
|
||||
{ sort: { expiresAt: 1 }, returnDocument: 'after', session },
|
||||
).exec();
|
||||
if (!code) break;
|
||||
reservedIds.push(code._id);
|
||||
@@ -650,7 +830,7 @@ export class TournamentCodeService {
|
||||
reservedIds.length > 0 ||
|
||||
this.isMatchModified(match, 'doubleElimination')
|
||||
) {
|
||||
await match.save();
|
||||
await match.save({ session });
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user