Harden match and tournament code concurrency

This commit is contained in:
dahyeon committed 2026-06-27 12:28:05 -07:00
1 parent 292e37aaac
commit df7dcd3a9c
9 files changed
+2339 -128

No files matched your search

@@ -1,4 +1,5 @@
import { Module } from '@nestjs/common';
import { AuctionRedisModule } from '../auction/redis/auction-redis.module';
import { TournamentResultModule } from '../tournament-result/tournament-result.module';
import { TournamentCodeController } from './tournament-code.controller';
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
@@ -8,7 +9,7 @@ import { TournamentCodeService } from './tournament-code.service';
import { TournamentCodeVendorService } from './tournament-code-vendor.service';
@Module({
imports: [TournamentResultModule],
imports: [TournamentResultModule, AuctionRedisModule],
controllers: [TournamentCodeController, TournamentCodeReplacementController],
providers: [
TournamentCodeService,
@@ -20,9 +20,26 @@ type TournamentCodeServiceInternals = TournamentCodeService & {
attachPendingMatches(communityId?: mongoose.Types.ObjectId): Promise<number>;
};
function createAuctionRedisService() {
return {
acquireTokenLock: jest.fn().mockResolvedValue(true),
releaseTokenLock: jest.fn().mockResolvedValue(true),
};
}
function createSession() {
return {
startTransaction: jest.fn(),
commitTransaction: jest.fn().mockResolvedValue(undefined),
abortTransaction: jest.fn().mockResolvedValue(undefined),
endSession: jest.fn().mockResolvedValue(undefined),
};
}
function createService() {
return new TournamentCodeService(
{} as never,
createAuctionRedisService() as never,
) as unknown as TournamentCodeServiceInternals;
}
@@ -297,6 +314,7 @@ describe('TournamentCodeService round-robin scheduling', () => {
};
const service = new TournamentCodeService(
connectionService as never,
createAuctionRedisService() as never,
) as unknown as TournamentCodeServiceInternals;
const slots = await service.buildSlots(
@@ -356,6 +374,40 @@ describe('TournamentCodeService round-robin scheduling', () => {
});
describe('TournamentCodeService bulk upload', () => {
it('defines unique slot and riot match indexes for multi-instance safety', () => {
const indexes = TournamentCodeSchema.indexes();
expect(indexes).toContainEqual([
{ matchId: 1, gameNumber: 1 },
expect.objectContaining({
unique: true,
partialFilterExpression: {
matchId: { $exists: true },
gameNumber: { $exists: true },
state: { $in: ['reserved', 'used'] },
},
}),
]);
expect(indexes).toContainEqual([
{ matchId: 1, bracketMatchId: 1 },
expect.objectContaining({
unique: true,
partialFilterExpression: {
matchId: { $exists: true },
bracketMatchId: { $exists: true },
state: { $in: ['reserved', 'used'] },
},
}),
]);
expect(indexes).toContainEqual([
{ riotMatchId: 1 },
expect.objectContaining({
unique: true,
partialFilterExpression: { riotMatchId: { $type: 'string' } },
}),
]);
});
it('preserves tournament code casing at the schema layer', async () => {
const connection = mongoose.createConnection();
try {
@@ -393,6 +445,7 @@ describe('TournamentCodeService bulk upload', () => {
};
const service = new TournamentCodeService(
connectionService as never,
createAuctionRedisService() as never,
) as unknown as TournamentCodeServiceInternals;
service.attachPendingMatches = jest.fn().mockResolvedValue(0);
@@ -426,7 +479,10 @@ describe('TournamentCodeService pool stats', () => {
const connectionService = {
getModel: jest.fn().mockResolvedValue({ countDocuments }),
};
const service = new TournamentCodeService(connectionService as never);
const service = new TournamentCodeService(
connectionService as never,
createAuctionRedisService() as never,
);
await expect(service.countUsableAvailable()).resolves.toBe(12);
expect(countDocuments).toHaveBeenCalledWith({
@@ -477,7 +533,9 @@ describe('TournamentCodeService pending match attachment', () => {
exec: jest.fn().mockResolvedValue({ _id: codeId }),
}),
};
const session = createSession();
const connectionService = {
createSession: jest.fn().mockResolvedValue(session),
getModel: jest.fn().mockImplementation((modelName: string) => {
if (modelName === 'match') return Promise.resolve(MatchModel);
if (modelName === 'tournamentCode') {
@@ -486,7 +544,11 @@ describe('TournamentCodeService pending match attachment', () => {
throw new Error(`Unexpected model: ${modelName}`);
}),
};
const service = new TournamentCodeService(connectionService as never);
const auctionRedisService = createAuctionRedisService();
const service = new TournamentCodeService(
connectionService as never,
auctionRedisService as never,
);
const matches = await service.attachPendingMatchDocuments();
@@ -494,6 +556,17 @@ describe('TournamentCodeService pending match attachment', () => {
expect(MatchModel.findById).toHaveBeenCalledTimes(1);
expect(MatchModel.findById).toHaveBeenCalledWith(validMatch._id);
expect(TournamentCodeModel.findOneAndUpdate).toHaveBeenCalledTimes(1);
expect(auctionRedisService.acquireTokenLock).toHaveBeenCalledWith(
`tournament-code:match:${validMatch._id.toString()}:lock`,
expect.any(String),
expect.any(Number),
);
expect(session.startTransaction).toHaveBeenCalledTimes(1);
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
expect(auctionRedisService.releaseTokenLock).toHaveBeenCalledWith(
`tournament-code:match:${validMatch._id.toString()}:lock`,
expect.any(String),
);
expect(validMatch.tournamentCodeIds.map((id) => id.toString())).toEqual([
codeId.toString(),
]);
@@ -569,8 +642,11 @@ describe('TournamentCodeService code replacement', () => {
const TournamentCodeModel = {
findOne: jest.fn().mockReturnValue(queryResult(currentCode)),
findOneAndUpdate: jest.fn().mockReturnValue(queryResult(replacementCode)),
updateOne: jest.fn().mockReturnValue(queryResult({ modifiedCount: 1 })),
};
const session = createSession();
const connectionService = {
createSession: jest.fn().mockResolvedValue(session),
getModel: jest.fn().mockImplementation((modelName: string) => {
if (modelName === 'match') return Promise.resolve(MatchModel);
if (modelName === 'tournamentCode') {
@@ -579,8 +655,10 @@ describe('TournamentCodeService code replacement', () => {
throw new Error(`Unexpected model: ${modelName}`);
}),
};
const auctionRedisService = createAuctionRedisService();
const service = new TournamentCodeService(
connectionService as never,
auctionRedisService as never,
) as unknown as TournamentCodeServiceInternals;
return {
@@ -591,6 +669,8 @@ describe('TournamentCodeService code replacement', () => {
replacementCode,
MatchModel,
TournamentCodeModel,
auctionRedisService,
session,
service,
};
}
@@ -604,6 +684,8 @@ describe('TournamentCodeService code replacement', () => {
replacementCode,
MatchModel,
TournamentCodeModel,
auctionRedisService,
session,
service,
} = createReplacementFixture();
@@ -642,10 +724,30 @@ describe('TournamentCodeService code replacement', () => {
}),
expect.objectContaining({ returnDocument: 'after' }),
);
expect(currentCode?.state).toBe('released');
expect(currentCode?.matchId).toBeUndefined();
expect(currentCode?.gameNumber).toBeUndefined();
expect(currentCode?.save).toHaveBeenCalled();
expect(auctionRedisService.acquireTokenLock).toHaveBeenCalledWith(
`tournament-code:match:${matchId.toString()}:lock`,
expect.any(String),
expect.any(Number),
);
expect(session.startTransaction).toHaveBeenCalledTimes(1);
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
expect(auctionRedisService.releaseTokenLock).toHaveBeenCalledWith(
`tournament-code:match:${matchId.toString()}:lock`,
expect.any(String),
);
expect(TournamentCodeModel.updateOne).toHaveBeenCalledWith(
{ _id: currentCode?._id },
expect.objectContaining({
$set: expect.objectContaining({ state: 'released' }),
$unset: expect.objectContaining({
matchId: 1,
gameNumber: 1,
bracketMatchId: 1,
reservedAt: 1,
}),
}),
);
expect(currentCode?.save).not.toHaveBeenCalled();
expect(match.tournamentCodeIds.map((id) => id.toString())).toEqual([
replacementCode?._id.toString(),
]);
@@ -683,8 +785,8 @@ describe('TournamentCodeService code replacement', () => {
expect(fixture.TournamentCodeModel.findOneAndUpdate).not.toHaveBeenCalled();
});
it('leaves the current code untouched when no replacement code is available', async () => {
const { match, communityId, matchId, currentCode, service } =
it('aborts the transaction when no replacement code is available', async () => {
const { match, communityId, matchId, currentCode, session, service } =
createReplacementFixture({
replacementCode: null,
});
@@ -694,5 +796,6 @@ describe('TournamentCodeService code replacement', () => {
).rejects.toThrow('No available tournament code');
expect(currentCode?.save).not.toHaveBeenCalled();
expect(match.save).not.toHaveBeenCalled();
expect(session.abortTransaction).toHaveBeenCalledTimes(1);
});
});
@@ -1,13 +1,17 @@
import {
BadRequestException,
ConflictException,
Injectable,
InternalServerErrorException,
NotFoundException,
} from '@nestjs/common';
import { randomUUID } from 'crypto';
import mongoose from 'mongoose';
import {
buildDoubleEliminationBracket,
findNextReadyDoubleEliminationBracketMatches,
} from '../double-elimination/double-elimination.util';
import { AuctionRedisService } from '../auction/redis/auction-redis.service';
import { ConnectionService } from '../global/database/connection.service';
import type {
DoubleEliminationBracketMatch,
@@ -36,10 +40,15 @@ import {
const DEFAULT_REGION = 'kr';
const DEFAULT_SPECTATOR_TYPE: TournamentSpectatorType = 'ALL';
const TOURNAMENT_CODE_MATCH_LOCK_TTL_MS = 30_000;
const TOURNAMENT_CODE_TRANSACTION_ATTEMPTS = 3;
@Injectable()
export class TournamentCodeService {
constructor(private readonly connectionService: ConnectionService) {}
constructor(
private readonly connectionService: ConnectionService,
private readonly auctionRedisService: AuctionRedisService,
) {}
public async bulkUpload(params: {
communityId?: mongoose.Types.ObjectId;
@@ -231,21 +240,34 @@ export class TournamentCodeService {
public async reserveForMatch(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
): Promise<MatchDocument> {
const matchId = this.resolveMatchId(matchOrId);
return this.withLockedMatchTransaction(matchId, (session) =>
this.reserveForMatchInTransaction(matchId, session),
);
}
private async reserveForMatchInTransaction(
matchId: mongoose.Types.ObjectId,
session: mongoose.ClientSession,
): Promise<MatchDocument> {
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
const match =
matchOrId instanceof mongoose.Types.ObjectId
? await MatchModel.findById(matchOrId)
: await MatchModel.findById(matchOrId._id);
const match = await this.execQuery<MatchDocument | null>(
this.withQuerySession(MatchModel.findById(matchId), session),
);
if (!match) throw new NotFoundException('Match not found.');
match.gameCount = match.gameCount || DEFAULT_TOURNAMENT_GAME_COUNT;
match.tournamentCodeIds = match.tournamentCodeIds ?? [];
this.ensureGameSchedule(match);
if ((match.gameMode ?? 'pair') === 'double-elimination') {
return this.reserveForDoubleEliminationMatch(match, TournamentCodeModel);
return this.reserveForDoubleEliminationMatch(
match,
TournamentCodeModel,
session,
);
}
const shouldSaveMetadata =
this.isMatchModified(match, 'gameCount') ||
@@ -281,7 +303,7 @@ export class TournamentCodeService {
},
$unset: { releasedAt: 1 },
},
{ sort: { expiresAt: 1 }, returnDocument: 'after' },
{ sort: { expiresAt: 1 }, returnDocument: 'after', session },
).exec();
if (!code) break;
reservedIds.push(code._id);
@@ -291,7 +313,7 @@ export class TournamentCodeService {
match.tournamentCodeIds.push(...reservedIds);
}
if (reservedIds.length > 0 || shouldSaveMetadata) {
await match.save();
await match.save({ session });
}
return match;
}
@@ -301,13 +323,31 @@ export class TournamentCodeService {
matchId: mongoose.Types.ObjectId;
gameNumber: number;
}): Promise<TournamentCodeSlotView> {
return this.withLockedMatchTransaction(params.matchId, (session) =>
this.replaceForMatchGameInTransaction(params, session),
);
}
private async replaceForMatchGameInTransaction(
params: {
communityId: mongoose.Types.ObjectId;
matchId: mongoose.Types.ObjectId;
gameNumber: number;
},
session: mongoose.ClientSession,
): Promise<TournamentCodeSlotView> {
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
const match = await MatchModel.findOne({
_id: params.matchId,
communityId: params.communityId,
}).exec();
const match = await this.execQuery<MatchDocument | null>(
this.withQuerySession(
MatchModel.findOne({
_id: params.matchId,
communityId: params.communityId,
}),
session,
),
);
if (!match) throw new NotFoundException('Match not found.');
if (match.status !== 'team-built' && match.status !== 'in-progress') {
throw new BadRequestException(
@@ -348,13 +388,18 @@ export class TournamentCodeService {
);
}
const currentCode = await TournamentCodeModel.findOne({
_id: { $in: match.tournamentCodeIds },
matchId: match._id,
...(doubleEliminationBracketMatch
? { bracketMatchId: doubleEliminationBracketMatch.bracketMatchId }
: { gameNumber: params.gameNumber }),
}).exec();
const currentCode = await this.execQuery<TournamentCodeDocument | null>(
this.withQuerySession(
TournamentCodeModel.findOne({
_id: { $in: match.tournamentCodeIds },
matchId: match._id,
...(doubleEliminationBracketMatch
? { bracketMatchId: doubleEliminationBracketMatch.bracketMatchId }
: { gameNumber: params.gameNumber }),
}),
session,
),
);
if (!currentCode) {
throw new NotFoundException('Assigned tournament code not found.');
}
@@ -367,6 +412,25 @@ export class TournamentCodeService {
const expiresAfter = new Date(
Date.now() + TOURNAMENT_CODE_EXPIRY_MARGIN_MS,
);
await this.execQuery(
this.withQuerySession(
TournamentCodeModel.updateOne(
{ _id: currentCode._id },
{
$set: { state: 'released', releasedAt: new Date() },
$unset: {
matchId: 1,
gameNumber: 1,
bracketMatchId: 1,
reservedAt: 1,
riotMatchId: 1,
},
},
),
session,
),
);
const replacement = await TournamentCodeModel.findOneAndUpdate(
{
state: 'available',
@@ -389,7 +453,7 @@ export class TournamentCodeService {
},
$unset: { releasedAt: 1 },
},
{ sort: { expiresAt: 1 }, returnDocument: 'after' },
{ sort: { expiresAt: 1 }, returnDocument: 'after', session },
).exec();
if (!replacement) {
throw new BadRequestException(
@@ -397,20 +461,12 @@ export class TournamentCodeService {
);
}
currentCode.state = 'released';
currentCode.matchId = undefined;
currentCode.gameNumber = undefined;
currentCode.bracketMatchId = undefined;
currentCode.reservedAt = undefined;
currentCode.releasedAt = new Date();
currentCode.riotMatchId = undefined;
match.tournamentCodeIds = this.replaceTournamentCodeId(
match.tournamentCodeIds,
currentCode._id,
replacement._id,
);
await currentCode.save();
await match.save();
await match.save({ session });
return {
gameNumber: params.gameNumber,
@@ -431,32 +487,46 @@ export class TournamentCodeService {
public async releaseFromMatch(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
): Promise<void> {
const matchId = this.resolveMatchId(matchOrId);
await this.withLockedMatchTransaction(matchId, (session) =>
this.releaseFromMatchInTransaction(matchId, session),
);
}
private async releaseFromMatchInTransaction(
matchId: mongoose.Types.ObjectId,
session: mongoose.ClientSession,
): Promise<void> {
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
const match =
matchOrId instanceof mongoose.Types.ObjectId
? await MatchModel.findById(matchOrId)
: await MatchModel.findById(matchOrId._id);
const match = await this.execQuery<MatchDocument | null>(
this.withQuerySession(MatchModel.findById(matchId), session),
);
if (!match) return;
if (match.tournamentCodeIds.length === 0) return;
await TournamentCodeModel.updateMany(
{ _id: { $in: match.tournamentCodeIds }, state: 'reserved' },
{
$set: { state: 'released', releasedAt: new Date() },
$unset: {
matchId: 1,
gameNumber: 1,
bracketMatchId: 1,
reservedAt: 1,
},
},
).exec();
await this.execQuery(
this.withQuerySession(
TournamentCodeModel.updateMany(
{ _id: { $in: match.tournamentCodeIds }, state: 'reserved' },
{
$set: { state: 'released', releasedAt: new Date() },
$unset: {
matchId: 1,
gameNumber: 1,
bracketMatchId: 1,
reservedAt: 1,
},
},
),
session,
),
);
match.tournamentCodeIds = [];
await match.save();
await match.save({ session });
}
public async markExpired(): Promise<number> {
@@ -589,18 +659,128 @@ export class TournamentCodeService {
return next;
}
private resolveMatchId(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
): mongoose.Types.ObjectId {
return matchOrId instanceof mongoose.Types.ObjectId
? matchOrId
: matchOrId._id;
}
private async withLockedMatchTransaction<T>(
matchId: mongoose.Types.ObjectId,
operation: (session: mongoose.ClientSession) => Promise<T>,
): Promise<T> {
const lockKey = this.matchCodeLockKey(matchId);
const token = randomUUID();
const acquired = await this.auctionRedisService.acquireTokenLock(
lockKey,
token,
TOURNAMENT_CODE_MATCH_LOCK_TTL_MS,
);
if (!acquired) {
throw new ConflictException(
'Tournament code operation is already running for this match.',
);
}
try {
return await this.runTransactionWithRetry(operation);
} finally {
await this.auctionRedisService.releaseTokenLock(lockKey, token);
}
}
private async runTransactionWithRetry<T>(
operation: (session: mongoose.ClientSession) => Promise<T>,
): Promise<T> {
let lastError: unknown;
for (
let attempt = 1;
attempt <= TOURNAMENT_CODE_TRANSACTION_ATTEMPTS;
attempt += 1
) {
const session = await this.connectionService.createSession();
if (!session) {
throw new InternalServerErrorException('Failed to start DB session.');
}
try {
session.startTransaction();
const result = await operation(session);
await session.commitTransaction();
return result;
} catch (error) {
lastError = error;
await session.abortTransaction().catch(() => undefined);
if (
attempt >= TOURNAMENT_CODE_TRANSACTION_ATTEMPTS ||
!this.isRetryableTransactionError(error)
) {
throw error;
}
} finally {
await session.endSession().catch(() => undefined);
}
}
throw lastError;
}
private isRetryableTransactionError(error: unknown): boolean {
const candidate = error as {
code?: number;
codeName?: string;
hasErrorLabel?: (label: string) => boolean;
};
return Boolean(
candidate?.hasErrorLabel?.('TransientTransactionError') ||
candidate?.hasErrorLabel?.('UnknownTransactionCommitResult') ||
candidate?.code === 112 ||
candidate?.codeName === 'WriteConflict',
);
}
private matchCodeLockKey(matchId: mongoose.Types.ObjectId): string {
return `tournament-code:match:${matchId.toString()}:lock`;
}
private withQuerySession<T>(query: T, session: mongoose.ClientSession): T {
const maybeQuery = query as {
session?: (session: mongoose.ClientSession) => T;
};
if (maybeQuery && typeof maybeQuery.session === 'function') {
return maybeQuery.session(session);
}
return query;
}
private async execQuery<T = unknown>(query: unknown): Promise<T> {
const maybeQuery = query as { exec?: () => Promise<T> };
if (maybeQuery && typeof maybeQuery.exec === 'function') {
return maybeQuery.exec();
}
return query as Promise<T>;
}
private async reserveForDoubleEliminationMatch(
match: MatchDocument,
TournamentCodeModel: mongoose.Model<TournamentCode>,
session: mongoose.ClientSession,
): Promise<MatchDocument> {
const readyBracketMatches = findNextReadyDoubleEliminationBracketMatches(
match.doubleElimination,
);
const existingCodes =
match.tournamentCodeIds.length > 0
? await TournamentCodeModel.find({
_id: { $in: match.tournamentCodeIds },
}).exec()
? await this.execQuery<TournamentCodeDocument[]>(
this.withQuerySession(
TournamentCodeModel.find({
_id: { $in: match.tournamentCodeIds },
}),
session,
),
)
: [];
const byBracketMatchId = new Map(
existingCodes
@@ -636,7 +816,7 @@ export class TournamentCodeService {
},
$unset: { releasedAt: 1 },
},
{ sort: { expiresAt: 1 }, returnDocument: 'after' },
{ sort: { expiresAt: 1 }, returnDocument: 'after', session },
).exec();
if (!code) break;
reservedIds.push(code._id);
@@ -650,7 +830,7 @@ export class TournamentCodeService {
reservedIds.length > 0 ||
this.isMatchModified(match, 'doubleElimination')
) {
await match.save();
await match.save({ session });
}
return match;
}