Files
lolympus-tournament-support/apps/api/src/scripts/auction-redis-rehearsal.ts
T

604 lines
20 KiB
TypeScript

import { ConfigService } from '@nestjs/config';
import { randomUUID } from 'crypto';
import fs from 'fs';
import mongoose from 'mongoose';
import path from 'path';
import { createClient } from 'redis';
import { AuctionDeadlineScheduler } from '../modules/auction/auction-deadline.scheduler';
import { AuctionDurabilityScheduler } from '../modules/auction/auction-durability.scheduler';
import { AuctionService } from '../modules/auction/auction.service';
import { AuctionRedisCommandService } from '../modules/auction/redis/auction-redis-command.service';
import { buildAuctionRedisKeys } from '../modules/auction/redis/auction-redis.keys';
import { AuctionRedisService } from '../modules/auction/redis/auction-redis.service';
import { AuctionRedisStateReaderService } from '../modules/auction/redis/auction-redis-state-reader.service';
import { ConnectionService, resolveMongoDbName } from '../modules/global/database/connection.service';
import {
assertDistinctRedisInstances,
resolveRedisClientOptions,
} from '../modules/global/redis/redis.config';
import { SecretService } from '../modules/global/secret/secret.service';
type RedisClient = ReturnType<typeof createClient>;
const REHEARSAL_PREFIX = 'redis-sot-rehearsal';
const WAIT_FOR_EXPIRY_MS = 6200;
interface RehearsalResources {
auctionIds: mongoose.Types.ObjectId[];
communityId?: mongoose.Types.ObjectId;
matchId?: mongoose.Types.ObjectId;
userIds: mongoose.Types.ObjectId[];
}
function loadProjectEnv(): void {
const candidates = [
path.resolve(process.cwd(), 'project.env'),
path.resolve(process.cwd(), '../project.env'),
path.resolve(process.cwd(), 'apps/project.env'),
];
for (const candidate of candidates) {
if (!fs.existsSync(candidate)) continue;
for (const line of fs.readFileSync(candidate, 'utf8').split(/\r?\n/u)) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const separatorIndex = trimmed.indexOf('=');
if (separatorIndex < 0) continue;
const key = trimmed.slice(0, separatorIndex).trim();
const value = trimmed
.slice(separatorIndex + 1)
.trim()
.replace(/^['"]|['"]$/gu, '');
if (key && process.env[key] === undefined) {
process.env[key] = value;
}
}
}
}
async function readFirstSecret(
secretService: SecretService,
keys: string[],
): Promise<string | undefined> {
for (const key of keys) {
const value = await secretService.get<string>(key, '');
const trimmed = value.trim();
if (trimmed) return trimmed;
}
return undefined;
}
function requireSafeNodeEnv(): void {
if (process.env.NODE_ENV === 'prod' && process.env.ALLOW_PROD_AUCTION_REHEARSAL !== 'true') {
throw new Error(
'Refusing to run auction Redis rehearsal against prod. Set ALLOW_PROD_AUCTION_REHEARSAL=true only for an explicit production drill.',
);
}
}
function logStep(message: string): void {
console.log(`[auction-rehearsal] ${message}`);
}
async function createRedisClientFromSecrets(
secretService: SecretService,
kind: 'common' | 'auction',
): Promise<RedisClient> {
const prefix = kind === 'auction' ? 'AUCTION_REDIS' : 'REDIS';
const urlKeys = kind === 'auction' ? [`${prefix}_URL`] : ['REDIS_URL', 'GARNET_URL'];
const endpointKeys =
kind === 'auction' ? [`${prefix}_ENDPOINT`] : ['REDIS_ENDPOINT', 'GARNET_ENDPOINT'];
const usernameKeys =
kind === 'auction'
? [`${prefix}_USERNAME`, `${prefix}_USERID`]
: ['REDIS_USERNAME', 'REDIS_USERID', 'GARNET_USERNAME', 'GARNET_USERID'];
const passwordKeys =
kind === 'auction' ? [`${prefix}_PASSWORD`] : ['REDIS_PASSWORD', 'GARNET_PASSWORD'];
const [url, endpoint, username, password] = await Promise.all([
readFirstSecret(secretService, urlKeys),
readFirstSecret(secretService, endpointKeys),
readFirstSecret(secretService, usernameKeys),
readFirstSecret(secretService, passwordKeys),
]);
const client = createClient(
resolveRedisClientOptions({
endpoint,
endpointEnvName: `${prefix}_ENDPOINT`,
password,
url,
urlEnvName: `${prefix}_URL`,
username,
}),
);
client.on('error', (error) => {
console.error(`[auction-rehearsal] ${kind} Redis error:`, error);
});
await client.connect();
return client;
}
async function assertRedisTopology(
secretService: SecretService,
): Promise<{ auctionRedisClient: RedisClient; commonRedisClient: RedisClient }> {
const [
redisUrl,
redisEndpoint,
auctionRedisUrl,
auctionRedisEndpoint,
] = await Promise.all([
readFirstSecret(secretService, ['REDIS_URL', 'GARNET_URL']),
readFirstSecret(secretService, ['REDIS_ENDPOINT', 'GARNET_ENDPOINT']),
readFirstSecret(secretService, ['AUCTION_REDIS_URL']),
readFirstSecret(secretService, ['AUCTION_REDIS_ENDPOINT']),
]);
const redisOptions = resolveRedisClientOptions({
endpoint: redisEndpoint,
endpointEnvName: 'REDIS_ENDPOINT',
url: redisUrl,
urlEnvName: 'REDIS_URL',
});
const auctionRedisOptions = resolveRedisClientOptions({
endpoint: auctionRedisEndpoint,
endpointEnvName: 'AUCTION_REDIS_ENDPOINT',
url: auctionRedisUrl,
urlEnvName: 'AUCTION_REDIS_URL',
});
assertDistinctRedisInstances(redisOptions.url, auctionRedisOptions.url);
const [commonRedisClient, auctionRedisClient] = await Promise.all([
createRedisClientFromSecrets(secretService, 'common'),
createRedisClientFromSecrets(secretService, 'auction'),
]);
await Promise.all([
commonRedisClient.sendCommand(['PING']),
auctionRedisClient.sendCommand(['PING']),
auctionRedisClient.sendCommand(['TIME']),
]);
return { auctionRedisClient, commonRedisClient };
}
async function cleanupRedis(
auctionRedisService: AuctionRedisService,
matchId: mongoose.Types.ObjectId,
): Promise<void> {
const keys = buildAuctionRedisKeys(matchId.toString());
await auctionRedisService.raw().sendCommand([
'DEL',
keys.liveStateKey,
keys.structureStateKey,
keys.appliedTransitionIdsKey,
keys.bidHistoryKey,
keys.bidRequestResultsKey,
keys.bidRequestResultOrderKey,
keys.actionRequestResultsKey,
keys.actionRequestResultOrderKey,
keys.rehydrateLockKey,
keys.stateTransitionLockKey,
]);
await auctionRedisService.raw().sendCommand([
'ZREM',
keys.deadlineQueueKey,
matchId.toString(),
]);
let cursor = '0';
do {
const reply = await auctionRedisService.raw().sendCommand([
'SCAN',
cursor,
'MATCH',
`${keys.durabilityPayloadKeyPrefix}*`,
'COUNT',
'100',
]);
if (!Array.isArray(reply) || reply.length < 2) return;
cursor = String(reply[0]);
const payloadKeys = Array.isArray(reply[1]) ? reply[1].map(String) : [];
if (payloadKeys.length > 0) {
await auctionRedisService.raw().sendCommand([
'ZREM',
keys.durabilityRetryQueueKey,
...payloadKeys,
]);
await auctionRedisService.raw().sendCommand([
'ZREM',
keys.durabilityDeadLetterQueueKey,
...payloadKeys,
]);
await auctionRedisService.raw().sendCommand(['DEL', ...payloadKeys]);
}
} while (cursor !== '0');
}
async function closeRedisClient(
client: RedisClient,
label: 'auction' | 'common',
): Promise<void> {
try {
const result = await Promise.race([
client.quit().then(() => 'closed' as const),
new Promise<'timeout'>((resolve) => {
setTimeout(() => resolve('timeout'), 3_000);
}),
]);
if (result === 'timeout') {
client.destroy();
logStep(`${label} Redis quit timed out; destroyed client`);
}
} catch {
client.destroy();
logStep(`${label} Redis quit failed; destroyed client`);
}
}
async function cleanupMongo(
connectionService: ConnectionService,
resources: RehearsalResources,
): Promise<void> {
const AuctionModel = await connectionService.getModel('auction');
const AuctionHistoryModel = await connectionService.getModel('auctionHistory');
const SideEffectModel = await connectionService.getModel(
'auctionSideEffectApplication',
);
const MatchModel = await connectionService.getModel('match');
const UserModel = await connectionService.getModel('user');
const CommunityModel = await connectionService.getModel('community');
const auctionIds = new Set(resources.auctionIds.map((id) => id.toString()));
if (resources.matchId) {
const existingAuctions = await AuctionModel.find({
matchId: resources.matchId,
})
.select({ _id: 1 })
.lean()
.exec();
for (const auction of existingAuctions) {
auctionIds.add(String(auction._id));
}
await SideEffectModel.deleteMany({ matchId: resources.matchId }).exec();
await MatchModel.deleteOne({ _id: resources.matchId }).exec();
}
if (auctionIds.size > 0) {
const ids = [...auctionIds].map((id) => new mongoose.Types.ObjectId(id));
await AuctionHistoryModel.deleteMany({ auctionId: { $in: ids } }).exec();
await AuctionModel.deleteMany({ _id: { $in: ids } }).exec();
}
if (resources.userIds.length > 0) {
await UserModel.deleteMany({ _id: { $in: resources.userIds } }).exec();
}
if (resources.communityId) {
await CommunityModel.deleteOne({ _id: resources.communityId }).exec();
}
}
async function seedMatch(
connectionService: ConnectionService,
): Promise<RehearsalResources> {
const CommunityModel = await connectionService.getModel('community');
const UserModel = await connectionService.getModel('user');
const MatchModel = await connectionService.getModel('match');
const suffix = `${Date.now()}-${randomUUID().slice(0, 8)}`;
const communityId = new mongoose.Types.ObjectId();
const ownerUserId = new mongoose.Types.ObjectId();
const community = await CommunityModel.create({
_id: communityId,
description: 'Temporary auction Redis rehearsal community',
name: `${REHEARSAL_PREFIX}-${suffix}`,
ownerUserId,
riotPlatform: 'kr',
riotRegion: 'asia',
});
const userIds = Array.from({ length: 10 }, (_, index) =>
index === 0 ? ownerUserId : new mongoose.Types.ObjectId(),
);
await UserModel.insertMany(
userIds.map((userId, index) => ({
_id: userId,
communityId: community._id,
preferredPositions: [{ position: 'MID', priority: 1 }],
riotAccounts: [
{
championMasteries: [],
championRecords: {},
gameName: `Rehearsal${index + 1}`,
isPrimary: true,
lastRefreshRequestedAt: new Date(),
profileIconId: 1,
puuid: `${REHEARSAL_PREFIX}-puuid-${suffix}-${index + 1}`,
tagLine: 'TEST',
tierHistory: [],
tiers: [],
},
],
role: index === 0 ? 'owner' : 'member',
})),
);
const match = await MatchModel.create({
communityId: community._id,
description: 'Temporary Redis SoT rehearsal match',
gameCount: 1,
gameMode: 'pair',
maxParticipants: 10,
participants: userIds.map((userId, index) => ({
excluded: false,
joinedAt: new Date(Date.now() + index),
preferredPositions: [{ position: 'MID', priority: 1 }],
userId,
})),
scheduledAt: new Date(Date.now() + 60_000),
shortCode: `R${Math.random().toString(36).slice(2, 6)}`.slice(0, 6).toUpperCase(),
status: 'recruiting',
title: `${REHEARSAL_PREFIX}-${suffix}`,
});
return {
auctionIds: [],
communityId: community._id,
matchId: match._id,
userIds,
};
}
async function flushDurability(
durabilityScheduler: AuctionDurabilityScheduler,
): Promise<void> {
for (let attempt = 0; attempt < 5; attempt += 1) {
const result = await durabilityScheduler.processDueDurabilityPayloads(50);
if (result.applied === 0 && result.failed === 0) return;
}
}
function assertState(
state: Record<string, unknown> | null,
expectedPhase: string,
): Record<string, unknown> {
if (!state) throw new Error(`Expected auction state phase=${expectedPhase}, got null`);
if (state.phase !== expectedPhase) {
throw new Error(
`Expected auction phase=${expectedPhase}, got ${String(state.phase)}`,
);
}
return state;
}
async function main(): Promise<void> {
loadProjectEnv();
process.env.NODE_ENV = process.env.NODE_ENV || 'staging';
process.env.SECRET_SOURCE = process.env.SECRET_SOURCE || 'infisical';
requireSafeNodeEnv();
const keep = process.argv.includes('--keep');
const configService = new ConfigService(process.env);
const secretService = new SecretService(configService);
await secretService.ready();
logStep(`secrets loaded, nodeEnv=${process.env.NODE_ENV}, db=${resolveMongoDbName()}`);
const { auctionRedisClient, commonRedisClient } =
await assertRedisTopology(secretService);
const connectionService = new ConnectionService(secretService);
const auctionRedisService = new AuctionRedisService(auctionRedisClient);
const auctionRedisCommandService = new AuctionRedisCommandService(
auctionRedisService,
);
const auctionRedisStateReaderService = new AuctionRedisStateReaderService(
auctionRedisService,
);
const dummyTournamentCodeService = {
releaseFromMatch: async () => undefined,
reserveForMatch: async () => undefined,
};
const dummyTournamentResultService = {
cancelPolling: async () => undefined,
scheduleInitialPoll: async () => undefined,
};
const dummySideEffectService = {
mirrorMatchSideEffectStatus: async () => true,
};
const dummyEventBus = {
broadcastMatchSideEffect: () => undefined,
broadcastState: () => undefined,
};
const auctionService = new AuctionService(
connectionService,
{
enrichRiotAccount: async () => ({
account: null,
league: 'fresh',
masteries: 'fresh',
recentMatches: 'fresh',
summoner: 'fresh',
}),
} as never,
dummyTournamentCodeService as never,
dummyTournamentResultService as never,
auctionRedisService,
auctionRedisCommandService,
auctionRedisStateReaderService,
);
const durabilityScheduler = new AuctionDurabilityScheduler(
connectionService,
auctionRedisService,
dummyTournamentCodeService as never,
dummyTournamentResultService as never,
dummySideEffectService as never,
dummyEventBus as never,
);
const deadlineScheduler = new AuctionDeadlineScheduler(
auctionRedisService,
auctionRedisCommandService,
auctionService,
);
const resources: RehearsalResources = { auctionIds: [], userIds: [] };
try {
Object.assign(resources, await seedMatch(connectionService));
if (!resources.matchId || !resources.communityId) {
throw new Error('Failed to seed rehearsal match.');
}
logStep(`seeded match=${resources.matchId.toString()}`);
const auction = await auctionService.start({
bidRoundExtensionSeconds: 1,
captains: [
{ teamIndex: 1, userId: resources.userIds[0] },
{ teamIndex: 2, userId: resources.userIds[1] },
],
communityId: resources.communityId,
initialTeamBidPointAmount: 100,
manualTopTierAuctionPriorityUserIds: [],
matchId: resources.matchId,
maximumBidRoundDurationSeconds: 5,
startAuctionRequestId: randomUUID(),
topTierAuctionPriorityThreshold: 'NONE',
});
resources.auctionIds.push(auction._id);
assertState(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
),
'presenting',
);
logStep('start -> presenting OK');
await auctionService.openBidding(resources.matchId, {
auctionActionRequestId: randomUUID(),
});
await flushDurability(durabilityScheduler);
assertState(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
),
'bidding',
);
logStep('open-bidding -> bidding OK');
const bidResult = await auctionService.placeBid({
amount: 10,
bidRequestId: randomUUID(),
matchId: resources.matchId,
teamIndex: 1,
});
if (!('revision' in bidResult)) throw new Error('Bid did not return state.');
logStep('bid OK');
await auctionService.markSold({
assignedPosition: 'MID',
auctionActionRequestId: randomUUID(),
matchId: resources.matchId,
});
await flushDurability(durabilityScheduler);
assertState(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
),
'sold',
);
logStep('sold transition OK');
await auctionService.next(resources.matchId, {
auctionActionRequestId: randomUUID(),
});
await flushDurability(durabilityScheduler);
assertState(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
),
'presenting',
);
logStep('next -> presenting OK');
await auctionService.openBidding(resources.matchId, {
auctionActionRequestId: randomUUID(),
});
await flushDurability(durabilityScheduler);
logStep(`waiting ${WAIT_FOR_EXPIRY_MS}ms for Redis TIME deadline`);
await new Promise((resolve) => setTimeout(resolve, WAIT_FOR_EXPIRY_MS));
const deadlineResult = await deadlineScheduler.processDueBidRoundDeadlines(20);
if (deadlineResult.completed < 1) {
throw new Error(
`Expected deadline completion, got ${JSON.stringify(deadlineResult)}`,
);
}
await flushDurability(durabilityScheduler);
assertState(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
),
'passed',
);
logStep('automatic expiry -> passed OK');
const historyBeforeUndo = await auctionService.listHistory({
communityId: resources.communityId,
matchId: resources.matchId,
});
const undoTarget = historyBeforeUndo[historyBeforeUndo.length - 1]?.sequence;
if (!undoTarget) throw new Error('No history sequence to undo.');
await auctionService.undoTo({
auctionActionRequestId: randomUUID(),
communityId: resources.communityId,
matchId: resources.matchId,
sequence: undoTarget,
});
await flushDurability(durabilityScheduler);
logStep(`undo to sequence=${undoTarget} OK`);
await auctionService.reset({
auctionActionRequestId: randomUUID(),
communityId: resources.communityId,
matchId: resources.matchId,
});
await flushDurability(durabilityScheduler);
assertState(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
),
'waiting',
);
logStep('reset -> waiting + side effect OK');
const restartedAuction = await auctionService.start({
bidRoundExtensionSeconds: 1,
captains: [
{ teamIndex: 1, userId: resources.userIds[0] },
{ teamIndex: 2, userId: resources.userIds[1] },
],
communityId: resources.communityId,
initialTeamBidPointAmount: 100,
manualTopTierAuctionPriorityUserIds: [],
matchId: resources.matchId,
maximumBidRoundDurationSeconds: 5,
startAuctionRequestId: randomUUID(),
topTierAuctionPriorityThreshold: 'NONE',
});
resources.auctionIds.push(restartedAuction._id);
assertState(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
),
'presenting',
);
logStep('restart after reset OK');
logStep('REHEARSAL PASSED');
} finally {
if (!keep) {
if (resources.matchId) {
await cleanupRedis(auctionRedisService, resources.matchId);
}
await cleanupMongo(connectionService, resources);
logStep('cleanup complete');
} else {
logStep('kept rehearsal data because --keep was provided');
}
await closeRedisClient(auctionRedisClient, 'auction');
await closeRedisClient(commonRedisClient, 'common');
await connectionService.onModuleDestroy();
await mongoose.disconnect().catch(() => undefined);
logStep('connections closed');
}
}
void main().then(() => {
process.exit(0);
}).catch((error) => {
console.error('[auction-rehearsal] FAILED:', error);
process.exit(1);
});