Files
lolympus-tournament-support/apps/api/src/modules/global/database/migrations/20260627-tournament-code-unique-indexes.ts
T

758 lines
22 KiB
TypeScript

type TournamentCodeState =
| 'available'
| 'reserved'
| 'used'
| 'released'
| 'expired'
| 'voided'
| string;
type AnyDoc = Record<string, unknown> & { _id: unknown };
type IndexKey = Record<string, 1 | -1>;
type PartialFilterExpression = Record<string, unknown>;
type Projection = Record<string, 0 | 1>;
interface MigrationCursor {
limit(limit: number): AsyncIterable<AnyDoc>;
}
interface MigrationIndex {
key: IndexKey;
name: string;
partialFilterExpression?: PartialFilterExpression;
unique?: boolean;
}
interface MigrationCollection {
find(
filter: Record<string, unknown>,
options?: { projection?: Projection },
): MigrationCursor;
updateMany(
filter: Record<string, unknown>,
update: {
$pull?: Record<string, unknown>;
$set?: Record<string, unknown>;
$unset?: Record<string, ''>;
},
options?: { ordered?: boolean },
): Promise<{ matchedCount?: number; modifiedCount?: number } | unknown>;
indexes?(): Promise<MigrationIndex[]>;
dropIndex?(name: string): Promise<unknown>;
createIndex?(
key: IndexKey,
options: {
commitQuorum?: number | string;
maxTimeMS?: number;
name: string;
partialFilterExpression?: PartialFilterExpression;
unique?: boolean;
},
): Promise<string>;
}
interface MigrationDb {
collection(name: string): MigrationCollection;
}
interface TournamentCodeDoc extends AnyDoc {
bracketMatchId?: unknown;
code?: unknown;
createdAt?: unknown;
gameNumber?: unknown;
matchId?: unknown;
reservedAt?: unknown;
riotMatchId?: unknown;
state?: TournamentCodeState;
updatedAt?: unknown;
uploadedAt?: unknown;
usedAt?: unknown;
}
interface MatchDoc extends AnyDoc {
tournamentCodeIds?: unknown;
}
interface DesiredIndex {
key: IndexKey;
options: {
name: string;
partialFilterExpression?: PartialFilterExpression;
unique?: boolean;
};
}
interface DuplicateGroup {
codeDocs: TournamentCodeDoc[];
key: string;
kind: TournamentCodeDuplicateKind;
}
interface ReleasePlanEntry {
code: TournamentCodeDoc;
reasons: string[];
}
export type TournamentCodeDuplicateKind =
| 'gameNumber'
| 'bracketMatchId'
| 'riotMatchId';
export interface TournamentCodeDuplicateSummary {
bracketMatchId?: string;
code?: string;
gameNumber?: string;
id: string;
matchId?: string;
riotMatchId?: string;
state?: string;
}
export interface TournamentCodeDuplicateGroupSummary {
keeper: TournamentCodeDuplicateSummary;
key: string;
kind: TournamentCodeDuplicateKind;
losers: TournamentCodeDuplicateSummary[];
}
export interface TournamentCodeUniqueIndexMigrationOptions {
allowUsedDuplicateRelease?: boolean;
createIndexes?: boolean;
dryRun?: boolean;
duplicateSampleLimit?: number;
indexCommitQuorum?: number | string;
indexMaxTimeMS?: number;
now?: Date;
onProgress?: (progress: TournamentCodeUniqueIndexMigrationProgress) => void;
progressInterval?: number;
}
export interface TournamentCodeUniqueIndexMigrationStats {
blankRiotMatchIdsToUnset: number;
blankRiotMatchIdsUnset: number;
duplicateBracketMatchGroups: number;
duplicateCodesReleased: number;
duplicateCodesToRelease: number;
duplicateGameNumberGroups: number;
duplicateRiotMatchGroups: number;
indexesAlreadyCorrect: number;
indexesCreated: number;
indexesDropped: number;
matchesUpdated: number;
scannedCodes: number;
scannedMatches: number;
unsafeUsedDuplicateCodes: number;
unsafeUsedDuplicateGroups: number;
}
export interface TournamentCodeUniqueIndexMigrationResult {
sampleDuplicateGroups: TournamentCodeDuplicateGroupSummary[];
stats: TournamentCodeUniqueIndexMigrationStats;
}
export interface TournamentCodeUniqueIndexMigrationProgress {
action?: 'alreadyCorrect' | 'create' | 'created' | 'drop' | 'dropped' | 'list';
phase:
| 'scanMatches'
| 'scanTournamentCodes'
| 'cleanupBlankRiotMatchIds'
| 'cleanupDuplicateCodes'
| 'ensureIndexes';
scanned?: number;
indexName?: string;
total?: number;
}
const ACTIVE_CODE_STATES = new Set(['reserved', 'used']);
const MATCH_SCAN_FILTER = { 'tournamentCodeIds.0': { $exists: true } };
const MATCH_SCAN_PROJECTION: Projection = { _id: 1, tournamentCodeIds: 1 };
const TOURNAMENT_CODE_SCAN_FILTER = {
$or: [
{ state: { $in: ['reserved', 'used'] }, matchId: { $exists: true } },
{ riotMatchId: { $type: 'string' } },
],
};
const TOURNAMENT_CODE_SCAN_PROJECTION: Projection = {
_id: 1,
bracketMatchId: 1,
code: 1,
createdAt: 1,
gameNumber: 1,
matchId: 1,
reservedAt: 1,
riotMatchId: 1,
state: 1,
updatedAt: 1,
uploadedAt: 1,
usedAt: 1,
};
const DESIRED_TOURNAMENT_CODE_INDEXES: DesiredIndex[] = [
{
key: { matchId: 1, gameNumber: 1 },
options: {
name: 'matchId_1_gameNumber_1',
unique: true,
partialFilterExpression: {
matchId: { $exists: true },
gameNumber: { $exists: true },
state: { $in: ['reserved', 'used'] },
},
},
},
{
key: { matchId: 1, bracketMatchId: 1 },
options: {
name: 'matchId_1_bracketMatchId_1',
unique: true,
partialFilterExpression: {
matchId: { $exists: true },
bracketMatchId: { $exists: true },
state: { $in: ['reserved', 'used'] },
},
},
},
{
key: { riotMatchId: 1 },
options: {
name: 'riotMatchId_1',
unique: true,
partialFilterExpression: { riotMatchId: { $type: 'string' } },
},
},
];
export async function migrateTournamentCodeUniqueIndexes(
db: MigrationDb,
options: TournamentCodeUniqueIndexMigrationOptions = {},
): Promise<TournamentCodeUniqueIndexMigrationResult> {
const dryRun = options.dryRun ?? true;
const now = options.now ?? new Date();
const tournamentCodeCollection = db.collection('tournamentCode');
const matchCollection = db.collection('match');
const matchScan = await readReferencedTournamentCodeIds(
matchCollection,
options,
);
const codeDocs = await readTournamentCodes(tournamentCodeCollection, options);
const groups = findDuplicateGroups(codeDocs, matchScan.referencedCodeIds);
const releasePlan = buildReleasePlan(groups, matchScan.referencedCodeIds);
const blankRiotMatchCodeIds = codeDocs
.filter(hasBlankRiotMatchId)
.map((code) => code._id);
const unsafePlanEntries = releasePlan.filter(
(entry) => entry.code.state === 'used',
);
const unsafeGroupCount = groups.filter((group) => {
const keeper = chooseKeeper(group, matchScan.referencedCodeIds);
const keeperId = stringifyId(keeper._id);
return group.codeDocs.some(
(code) => stringifyId(code._id) !== keeperId && code.state === 'used',
);
}).length;
const stats: TournamentCodeUniqueIndexMigrationStats = {
blankRiotMatchIdsToUnset: blankRiotMatchCodeIds.length,
blankRiotMatchIdsUnset: 0,
duplicateBracketMatchGroups: groups.filter(
(group) => group.kind === 'bracketMatchId',
).length,
duplicateCodesReleased: 0,
duplicateCodesToRelease: releasePlan.length,
duplicateGameNumberGroups: groups.filter(
(group) => group.kind === 'gameNumber',
).length,
duplicateRiotMatchGroups: groups.filter(
(group) => group.kind === 'riotMatchId',
).length,
indexesAlreadyCorrect: 0,
indexesCreated: 0,
indexesDropped: 0,
matchesUpdated: 0,
scannedCodes: codeDocs.length,
scannedMatches: matchScan.scannedMatches,
unsafeUsedDuplicateCodes: unsafePlanEntries.length,
unsafeUsedDuplicateGroups: unsafeGroupCount,
};
if (
!dryRun &&
unsafePlanEntries.length > 0 &&
options.allowUsedDuplicateRelease !== true
) {
throw new Error(
`Refusing to release ${unsafePlanEntries.length} used duplicate tournament codes. ` +
'Review the duplicate groups manually or re-run with --allow-used-duplicate-release.',
);
}
if (!dryRun) {
if (blankRiotMatchCodeIds.length > 0) {
emitProgress(options, 'cleanupBlankRiotMatchIds', {
total: blankRiotMatchCodeIds.length,
});
const result = await tournamentCodeCollection.updateMany(
{ _id: { $in: blankRiotMatchCodeIds } },
{ $unset: { riotMatchId: '' } },
);
stats.blankRiotMatchIdsUnset = readModifiedCount(result);
}
if (releasePlan.length > 0) {
const releaseIds = releasePlan.map((entry) => entry.code._id);
emitProgress(options, 'cleanupDuplicateCodes', {
total: releaseIds.length,
});
const releaseResult = await tournamentCodeCollection.updateMany(
{ _id: { $in: releaseIds } },
{
$set: { releasedAt: now, state: 'released' },
$unset: {
bracketMatchId: '',
gameNumber: '',
matchId: '',
reservedAt: '',
riotMatchId: '',
usedAt: '',
},
},
);
stats.duplicateCodesReleased = readModifiedCount(releaseResult);
const matchResult = await matchCollection.updateMany(
{ tournamentCodeIds: { $in: releaseIds } },
{ $pull: { tournamentCodeIds: { $in: releaseIds } } },
);
stats.matchesUpdated = readModifiedCount(matchResult);
}
if (options.createIndexes ?? true) {
emitProgress(options, 'ensureIndexes');
const indexStats = await ensureTournamentCodeUniqueIndexes(
tournamentCodeCollection,
options,
);
stats.indexesAlreadyCorrect = indexStats.indexesAlreadyCorrect;
stats.indexesCreated = indexStats.indexesCreated;
stats.indexesDropped = indexStats.indexesDropped;
}
}
return {
sampleDuplicateGroups: summarizeGroups(
groups,
matchScan.referencedCodeIds,
options.duplicateSampleLimit ?? 20,
),
stats,
};
}
async function readTournamentCodes(
collection: MigrationCollection,
options: TournamentCodeUniqueIndexMigrationOptions,
): Promise<TournamentCodeDoc[]> {
const docs: TournamentCodeDoc[] = [];
for await (const doc of collection
.find(TOURNAMENT_CODE_SCAN_FILTER, {
projection: TOURNAMENT_CODE_SCAN_PROJECTION,
})
.limit(0)) {
docs.push(doc as TournamentCodeDoc);
emitScanProgress(options, 'scanTournamentCodes', docs.length);
}
emitProgress(options, 'scanTournamentCodes', { scanned: docs.length });
return docs;
}
async function readReferencedTournamentCodeIds(
collection: MigrationCollection,
options: TournamentCodeUniqueIndexMigrationOptions,
): Promise<{ referencedCodeIds: Set<string>; scannedMatches: number }> {
const referencedCodeIds = new Set<string>();
let scannedMatches = 0;
for await (const doc of collection
.find(MATCH_SCAN_FILTER, { projection: MATCH_SCAN_PROJECTION })
.limit(0)) {
scannedMatches += 1;
emitScanProgress(options, 'scanMatches', scannedMatches);
const ids = (doc as MatchDoc).tournamentCodeIds;
if (!Array.isArray(ids)) continue;
for (const id of ids) {
const stringId = stringifyId(id);
if (stringId) referencedCodeIds.add(stringId);
}
}
emitProgress(options, 'scanMatches', { scanned: scannedMatches });
return { referencedCodeIds, scannedMatches };
}
function emitScanProgress(
options: TournamentCodeUniqueIndexMigrationOptions,
phase: TournamentCodeUniqueIndexMigrationProgress['phase'],
scanned: number,
) {
const interval = options.progressInterval ?? 5_000;
if (interval < 1 || scanned % interval !== 0) return;
emitProgress(options, phase, { scanned });
}
function emitProgress(
options: TournamentCodeUniqueIndexMigrationOptions,
phase: TournamentCodeUniqueIndexMigrationProgress['phase'],
values: Omit<TournamentCodeUniqueIndexMigrationProgress, 'phase'> = {},
) {
options.onProgress?.({ phase, ...values });
}
function findDuplicateGroups(
docs: TournamentCodeDoc[],
referencedCodeIds: Set<string>,
): DuplicateGroup[] {
return [
...duplicateGroupsFor(docs, referencedCodeIds, 'gameNumber'),
...duplicateGroupsFor(docs, referencedCodeIds, 'bracketMatchId'),
...duplicateGroupsFor(docs, referencedCodeIds, 'riotMatchId'),
];
}
function duplicateGroupsFor(
docs: TournamentCodeDoc[],
referencedCodeIds: Set<string>,
kind: TournamentCodeDuplicateKind,
): DuplicateGroup[] {
const groups = new Map<string, TournamentCodeDoc[]>();
for (const doc of docs) {
const key = duplicateKey(doc, kind);
if (!key) continue;
const group = groups.get(key) ?? [];
group.push(doc);
groups.set(key, group);
}
return [...groups.entries()]
.filter(([, groupDocs]) => groupDocs.length > 1)
.map(([key, groupDocs]) => ({
codeDocs: sortCodesByKeeperPriority(groupDocs, referencedCodeIds, kind),
key,
kind,
}));
}
function duplicateKey(
doc: TournamentCodeDoc,
kind: TournamentCodeDuplicateKind,
): string | null {
if (kind === 'riotMatchId') {
if (typeof doc.riotMatchId !== 'string') return null;
if (doc.riotMatchId.trim() === '') return null;
return `riotMatchId:${doc.riotMatchId}`;
}
if (!ACTIVE_CODE_STATES.has(String(doc.state))) return null;
if (!hasField(doc, 'matchId')) return null;
if (kind === 'gameNumber') {
if (!hasField(doc, 'gameNumber')) return null;
return `matchId:${stringifyKeyPart(doc.matchId)}:gameNumber:${stringifyKeyPart(
doc.gameNumber,
)}`;
}
if (!hasField(doc, 'bracketMatchId')) return null;
return `matchId:${stringifyKeyPart(
doc.matchId,
)}:bracketMatchId:${stringifyKeyPart(doc.bracketMatchId)}`;
}
function buildReleasePlan(
groups: DuplicateGroup[],
referencedCodeIds: Set<string>,
): ReleasePlanEntry[] {
const byCodeId = new Map<string, ReleasePlanEntry>();
for (const group of groups) {
const keeper = chooseKeeper(group, referencedCodeIds);
const keeperId = stringifyId(keeper._id);
for (const code of group.codeDocs) {
const codeId = stringifyId(code._id);
if (!codeId || codeId === keeperId) continue;
const entry = byCodeId.get(codeId) ?? { code, reasons: [] };
entry.reasons.push(`${group.kind}:${group.key}`);
byCodeId.set(codeId, entry);
}
}
return [...byCodeId.values()];
}
function summarizeGroups(
groups: DuplicateGroup[],
referencedCodeIds: Set<string>,
limit: number,
): TournamentCodeDuplicateGroupSummary[] {
return groups.slice(0, limit).map((group) => {
const keeper = chooseKeeper(group, referencedCodeIds);
const keeperId = stringifyId(keeper._id);
return {
keeper: summarizeCode(keeper),
key: group.key,
kind: group.kind,
losers: group.codeDocs
.filter((code) => stringifyId(code._id) !== keeperId)
.map(summarizeCode),
};
});
}
function summarizeCode(doc: TournamentCodeDoc): TournamentCodeDuplicateSummary {
return {
bracketMatchId:
doc.bracketMatchId === undefined ? undefined : String(doc.bracketMatchId),
code: doc.code === undefined ? undefined : String(doc.code),
gameNumber: doc.gameNumber === undefined ? undefined : String(doc.gameNumber),
id: stringifyId(doc._id) ?? '',
matchId: doc.matchId === undefined ? undefined : stringifyId(doc.matchId),
riotMatchId:
doc.riotMatchId === undefined ? undefined : String(doc.riotMatchId),
state: doc.state === undefined ? undefined : String(doc.state),
};
}
function chooseKeeper(
group: DuplicateGroup,
referencedCodeIds: Set<string>,
): TournamentCodeDoc {
return sortCodesByKeeperPriority(
group.codeDocs,
referencedCodeIds,
group.kind,
)[0];
}
function sortCodesByKeeperPriority(
docs: TournamentCodeDoc[],
referencedCodeIds: Set<string>,
kind: TournamentCodeDuplicateKind,
): TournamentCodeDoc[] {
return [...docs].sort((left, right) =>
compareCodePriority(left, right, referencedCodeIds, kind),
);
}
function compareCodePriority(
left: TournamentCodeDoc,
right: TournamentCodeDoc,
referencedCodeIds: Set<string>,
kind: TournamentCodeDuplicateKind,
): number {
if (kind !== 'riotMatchId') {
const leftReferenced = referencedCodeIds.has(stringifyId(left._id) ?? '');
const rightReferenced = referencedCodeIds.has(stringifyId(right._id) ?? '');
if (leftReferenced !== rightReferenced) return leftReferenced ? -1 : 1;
}
const leftStateRank = stateRank(left.state);
const rightStateRank = stateRank(right.state);
if (leftStateRank !== rightStateRank) return leftStateRank - rightStateRank;
if (kind === 'riotMatchId') {
const leftReferenced = referencedCodeIds.has(stringifyId(left._id) ?? '');
const rightReferenced = referencedCodeIds.has(stringifyId(right._id) ?? '');
if (leftReferenced !== rightReferenced) return leftReferenced ? -1 : 1;
}
const leftHasRiotMatch = hasNonBlankRiotMatchId(left);
const rightHasRiotMatch = hasNonBlankRiotMatchId(right);
if (leftHasRiotMatch !== rightHasRiotMatch) {
return leftHasRiotMatch ? -1 : 1;
}
const leftDate = latestTimestamp(left);
const rightDate = latestTimestamp(right);
if (leftDate !== rightDate) return rightDate - leftDate;
return (stringifyId(left._id) ?? '').localeCompare(stringifyId(right._id) ?? '');
}
function stateRank(state: unknown): number {
if (state === 'used') return 0;
if (state === 'reserved') return 1;
if (state === 'available') return 2;
if (state === 'released') return 3;
if (state === 'expired') return 4;
if (state === 'voided') return 5;
return 10;
}
function latestTimestamp(doc: TournamentCodeDoc): number {
return Math.max(
toTimestamp(doc.usedAt),
toTimestamp(doc.reservedAt),
toTimestamp(doc.updatedAt),
toTimestamp(doc.createdAt),
toTimestamp(doc.uploadedAt),
0,
);
}
function toTimestamp(value: unknown): number {
if (value instanceof Date) return value.getTime();
if (typeof value === 'number') return value;
if (typeof value !== 'string') return 0;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? 0 : parsed;
}
function hasBlankRiotMatchId(doc: TournamentCodeDoc): boolean {
return typeof doc.riotMatchId === 'string' && doc.riotMatchId.trim() === '';
}
function hasNonBlankRiotMatchId(doc: TournamentCodeDoc): boolean {
return typeof doc.riotMatchId === 'string' && doc.riotMatchId.trim() !== '';
}
async function ensureTournamentCodeUniqueIndexes(
collection: MigrationCollection,
options: TournamentCodeUniqueIndexMigrationOptions,
): Promise<{
indexesAlreadyCorrect: number;
indexesCreated: number;
indexesDropped: number;
}> {
if (!collection.indexes || !collection.dropIndex || !collection.createIndex) {
throw new Error('Migration collection does not support index operations.');
}
emitProgress(options, 'ensureIndexes', { action: 'list' });
let indexes = await listCollectionIndexes(collection);
let indexesAlreadyCorrect = 0;
let indexesCreated = 0;
let indexesDropped = 0;
for (const desired of DESIRED_TOURNAMENT_CODE_INDEXES) {
const matchingIndexes = indexes.filter(
(index) =>
index.name === desired.options.name ||
sameObject(index.key, desired.key),
);
const correctIndex = matchingIndexes.find((index) =>
sameIndex(index, desired),
);
if (correctIndex) {
emitProgress(options, 'ensureIndexes', {
action: 'alreadyCorrect',
indexName: desired.options.name,
});
indexesAlreadyCorrect += 1;
continue;
}
for (const index of matchingIndexes) {
if (index.name === '_id_') continue;
emitProgress(options, 'ensureIndexes', {
action: 'drop',
indexName: index.name,
});
await collection.dropIndex(index.name);
emitProgress(options, 'ensureIndexes', {
action: 'dropped',
indexName: index.name,
});
indexesDropped += 1;
}
emitProgress(options, 'ensureIndexes', {
action: 'create',
indexName: desired.options.name,
});
await collection.createIndex(desired.key, {
...desired.options,
...(options.indexCommitQuorum !== undefined
? { commitQuorum: options.indexCommitQuorum }
: {}),
...(options.indexMaxTimeMS !== undefined
? { maxTimeMS: options.indexMaxTimeMS }
: {}),
});
emitProgress(options, 'ensureIndexes', {
action: 'created',
indexName: desired.options.name,
});
indexesCreated += 1;
emitProgress(options, 'ensureIndexes', { action: 'list' });
indexes = await listCollectionIndexes(collection);
}
return { indexesAlreadyCorrect, indexesCreated, indexesDropped };
}
async function listCollectionIndexes(
collection: MigrationCollection,
): Promise<MigrationIndex[]> {
if (!collection.indexes) {
throw new Error('Migration collection does not support index operations.');
}
try {
return await collection.indexes();
} catch (error) {
if (isNamespaceNotFoundError(error)) return [];
throw error;
}
}
function isNamespaceNotFoundError(error: unknown): boolean {
const candidate = error as {
code?: number;
codeName?: string;
message?: string;
};
return Boolean(
candidate?.code === 26 ||
candidate?.codeName === 'NamespaceNotFound' ||
candidate?.message?.includes('ns does not exist'),
);
}
function sameIndex(index: MigrationIndex, desired: DesiredIndex): boolean {
return (
sameObject(index.key, desired.key) &&
index.unique === desired.options.unique &&
sameObject(
index.partialFilterExpression ?? {},
desired.options.partialFilterExpression ?? {},
)
);
}
function sameObject(left: unknown, right: unknown): boolean {
return JSON.stringify(left) === JSON.stringify(right);
}
function readModifiedCount(result: unknown): number {
if (!result || typeof result !== 'object') return 0;
const modifiedCount = (result as { modifiedCount?: unknown }).modifiedCount;
return typeof modifiedCount === 'number' ? modifiedCount : 0;
}
function hasField(doc: Record<string, unknown>, key: string): boolean {
return Object.prototype.hasOwnProperty.call(doc, key);
}
function stringifyKeyPart(value: unknown): string {
if (value === null) return '<null>';
if (value === undefined) return '<undefined>';
return stringifyId(value) ?? String(value);
}
function stringifyId(value: unknown): string | undefined {
if (value === undefined || value === null) return undefined;
return typeof value === 'string' ? value : String(value);
}