Files
lolympus-tournament-support/apps/api/src/modules/auction/auction-durability.scheduler.ts
T

1392 lines
46 KiB
TypeScript

import {
Injectable,
Logger,
OnModuleDestroy,
OnModuleInit,
Optional,
} from '@nestjs/common';
import { createHash, randomUUID } from 'crypto';
import type mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import type {
AuctionMatchSideEffectStatus,
MatchTeamMember,
Position,
} from '../global/database/schema';
import { TournamentCodeService } from '../tournament-code/tournament-code.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { AuctionEventBusService } from './auction-event-bus.service';
import {
AuctionSideEffectService,
buildMatchSideEffectDelta,
buildSideEffectLeaseAcquireUpdate,
type MatchSideEffectDelta,
} from './auction-side-effect.service';
import { AuctionRedisService } from './redis/auction-redis.service';
import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service';
import {
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
buildAuctionRedisKeys,
} from './redis/auction-redis.keys';
// [AUCTION-DEBUG] 경매 "중복 멤버" 조사용 임시 계측. 조사 종료 후 제거 대상.
import { recordNewRelicCustomEvent } from '../global/observability/new-relic';
const AUCTION_DURABILITY_TICK_INTERVAL_MS = 1000;
const AUCTION_DURABILITY_BATCH_LIMIT = 20;
const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000;
const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000;
const AUCTION_DURABILITY_MAX_ATTEMPTS = 10;
const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000;
const AUCTION_DURABILITY_LEADER_LOCK_KEY =
'auction:auctionDurabilityScheduler:leader';
const AUCTION_DURABILITY_LEADER_LOCK_TTL_MS = 2500;
const AUCTION_REDIS_COMPLETED_RETAIN_SECONDS = 7 * 24 * 60 * 60;
const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000;
const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete';
const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset';
const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects';
export interface AuctionDurabilityPayload {
actorUserId?: unknown;
afterAuctionMongoProjection: Record<string, unknown>;
auctionActionType: string;
auctionHistorySequence: number;
auctionId: unknown;
auctionLifecycleId: string;
auctionStateTransitionId: string;
committedAtRedisTimestampMS?: number;
historyMutation?: Record<string, unknown>;
matchId: unknown;
matchSideEffectMutation: Record<string, unknown>;
shouldBroadcastState?: boolean;
broadcastStateRevision?: number;
revision: number;
snapshotBefore: Record<string, unknown>;
}
export interface AuctionProjectionIdentity {
auctionLifecycleId?: string;
revision?: number;
}
type ParsedAuctionHistoryMutation =
| { type: 'append' }
| { targetHistorySequence: number; type: 'deleteAfterSequenceAndAppendUndo' }
| { resetMode: 'clear-history' | 'preserve-audit'; type: 'reset' };
type ParsedMatchSideEffectMutation =
| { type: typeof SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE }
| { type: typeof RESET_MATCH_SIDE_EFFECT_TYPE }
| { type: typeof UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE };
interface CleanupSideEffectMatchDocument {
competitionId?: unknown;
gameSchedule?: unknown[];
participants: Array<{ excluded?: boolean }>;
result?: unknown;
resultPollState?: unknown;
save(): Promise<unknown>;
status: string;
teamBuiltAt?: Date;
teams?: unknown[];
tournamentCodeIds?: unknown[];
}
@Injectable()
export class AuctionDurabilityScheduler
implements OnModuleInit, OnModuleDestroy
{
private readonly logger = new Logger(AuctionDurabilityScheduler.name);
private tickTimer?: NodeJS.Timeout;
private tickInProgress = false;
constructor(
private readonly connectionService: ConnectionService,
private readonly auctionRedisService: AuctionRedisService,
@Optional()
private readonly tournamentCodeService?: TournamentCodeService,
@Optional()
private readonly tournamentResultService?: TournamentResultService,
@Optional()
private readonly auctionSideEffectService?: AuctionSideEffectService,
@Optional()
private readonly auctionEventBusService?: AuctionEventBusService,
@Optional()
private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService,
) {}
public onModuleInit(): void {
this.tickTimer = setInterval(
() => void this.safeTick(),
AUCTION_DURABILITY_TICK_INTERVAL_MS,
);
this.tickTimer.unref?.();
}
public onModuleDestroy(): void {
if (this.tickTimer) clearInterval(this.tickTimer);
}
public async readDueDurabilityPayloadKeys(
currentRedisTimestampMS: number,
batchLimit: number,
): Promise<string[]> {
const reply = await this.auctionRedisService
.raw()
.sendCommand([
'ZRANGEBYSCORE',
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
'-inf',
String(currentRedisTimestampMS),
'LIMIT',
'0',
String(batchLimit),
]);
return Array.isArray(reply) ? reply.map(String) : [];
}
public async processDueDurabilityPayloads(batchLimit: number): Promise<{
applied: number;
failed: number;
skipped: number;
}> {
const currentRedisTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
const payloadKeys = await this.readDueDurabilityPayloadKeys(
currentRedisTimestampMS,
batchLimit,
);
let applied = 0;
let failed = 0;
let skipped = 0;
for (const payloadKey of payloadKeys) {
try {
const payload = await this.readDurabilityPayload(payloadKey);
if (!payload) {
skipped += 1;
await this.clearDurabilityPayload(payloadKey);
continue;
}
await this.applyDurabilityPayload(payload);
applied += 1;
// [AUCTION-DEBUG] 제거 대상. 영속 지연(lag): Redis 커밋 → Mongo 영속까지 걸린 시간 + 배치 백로그.
// 사용자가 본 "되돌리기/이력이 2~3 뒤처짐"을 수치화한다(지연이 클수록 undo
// 패널이 stale 시퀀스를 보여 엉뚱한 스냅샷 복원 가능성↑).
recordNewRelicCustomEvent('AuctionServerDurability', {
matchId: typeof payload.matchId === 'string' ? payload.matchId : '',
auctionStateTransitionId: payload.auctionStateTransitionId,
auctionActionType: payload.auctionActionType,
revision: payload.revision,
auctionHistorySequence: payload.auctionHistorySequence,
persistLatencyMS:
typeof payload.committedAtRedisTimestampMS === 'number'
? currentRedisTimestampMS - payload.committedAtRedisTimestampMS
: -1,
batchBacklog: payloadKeys.length,
});
await this.clearDurabilityPayload(payloadKey);
} catch (error) {
failed += 1;
const failureDisposition = await this.recordDurabilityPayloadFailure(
payloadKey,
currentRedisTimestampMS,
error,
);
this.logger.warn(
`Auction durability payload failed for ${payloadKey} (${failureDisposition}): ${
error instanceof Error ? error.message : error
}`,
);
}
}
return { applied, failed, skipped };
}
public async readDurabilityPayload(
payloadKey: string,
): Promise<AuctionDurabilityPayload | null> {
const reply = await this.auctionRedisService
.raw()
.sendCommand(['HGETALL', payloadKey]);
const hash = parseRedisHash(reply);
const payloadJson = hash.payloadJson;
if (!payloadJson) return null;
const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload;
const committedAtRedisTimestampMS = Number(
hash.committedAtRedisTimestampMS,
);
if (Number.isFinite(committedAtRedisTimestampMS)) {
payload.committedAtRedisTimestampMS = committedAtRedisTimestampMS;
}
payload.shouldBroadcastState = hash.shouldBroadcastState === '1';
const broadcastStateRevision = Number(hash.broadcastStateRevision);
if (Number.isFinite(broadcastStateRevision)) {
payload.broadcastStateRevision = broadcastStateRevision;
}
return payload;
}
public async applyDurabilityPayload(
payload: AuctionDurabilityPayload,
): Promise<void> {
const AuctionHistoryModel =
await this.connectionService.getModel('auctionHistory');
const AuctionModel = await this.connectionService.getModel('auction');
const AuctionSideEffectApplicationModel =
await this.connectionService.getModel('auctionSideEffectApplication');
const session = await this.connectionService.createSession();
const sideEffectMutation = parseMatchSideEffectMutation(
payload.matchSideEffectMutation,
);
try {
session?.startTransaction();
const historyMutation = parseAuctionHistoryMutation(
payload.historyMutation,
);
if (typeof AuctionModel.findById === 'function') {
const currentProjection =
await readAuctionProjectionIdentityForMutation(
AuctionModel,
payload.auctionId,
session,
);
if (
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
) {
// 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다.
// 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다.
if (
historyMutation.type === 'append' &&
currentProjection.auctionLifecycleId === payload.auctionLifecycleId
) {
await this.applyStaleAppendHistoryRecovery({
AuctionHistoryModel,
payload,
session,
});
}
if (session) await session.commitTransaction();
return;
}
}
await this.applyAuctionHistoryMutation({
AuctionHistoryModel,
AuctionModel,
payload,
session,
});
const projectionOperation = buildAuctionProjectionGuardedUpdate(payload);
await execMaybe(
AuctionModel.updateOne(
projectionOperation.filter as never,
projectionOperation.update as never,
{ session } as never,
),
);
if (sideEffectMutation) {
const sideEffectOperation = buildSideEffectApplicationUpsert(
payload,
sideEffectMutation,
);
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
sideEffectOperation.filter as never,
sideEffectOperation.update as never,
{
session,
upsert: true,
} as never,
),
);
}
if (session) await session.commitTransaction();
} catch (error) {
if (session) await session.abortTransaction();
throw error;
} finally {
if (session) await session.endSession();
}
await this.broadcastLatestRedisAuctionStateIfRequested(payload);
if (sideEffectMutation) {
await this.applyMatchSideEffect(payload, sideEffectMutation);
}
}
/**
* 노드 크래시 복구용 백업 broadcast(1차 전달 아님).
*
* 정상 경로에서는 요청을 처리한 게이트웨이(수동 전환) 또는
* `AuctionDeadlineScheduler`(자동 만료)가 commit 직후 동기로 `state`를 방송한다.
* 이 메서드는 "방송을 하기로 한 인스턴스가 broadcast 전에 죽은" 경우를 위해
* outbox를 처리하면서 최신 state를 한 번 더 방송하는 안전망이다.
* 이 호출은 위 try 블록의 Mongo 트랜잭션이 성공해야(early return/throw 없이)
* 도달하므로 1차 전달로 의존해서는 안 된다. 중복 방송은 클라 revision 가드로 무해.
*/
private async broadcastLatestRedisAuctionStateIfRequested(
payload: AuctionDurabilityPayload,
): Promise<void> {
if (!payload.shouldBroadcastState) return;
if (!this.auctionRedisStateReaderService || !this.auctionEventBusService) {
return;
}
const matchId = String(payload.matchId);
const state =
await this.auctionRedisStateReaderService.readAuctionStateView(matchId);
if (!state) return;
const stateRevision = Number(state.revision);
if (
Number.isFinite(payload.broadcastStateRevision) &&
Number.isFinite(stateRevision) &&
stateRevision < Number(payload.broadcastStateRevision)
) {
return;
}
this.auctionEventBusService.broadcastState(matchId, state);
}
private async applyMatchSideEffect(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) {
await this.applySyncMatchTeamsSideEffect(payload, mutation);
return;
}
if (mutation.type === RESET_MATCH_SIDE_EFFECT_TYPE) {
await this.applyResetMatchSideEffect(payload, mutation);
return;
}
if (mutation.type === UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE) {
await this.applyUndoCompletedMatchSideEffect(payload, mutation);
}
}
private async applySyncMatchTeamsSideEffect(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
if (!this.tournamentCodeService || !this.tournamentResultService) {
throw new Error('Auction side-effect services are not available.');
}
const AuctionModel = await this.connectionService.getModel('auction');
const AuctionSideEffectApplicationModel =
await this.connectionService.getModel('auctionSideEffectApplication');
const MatchModel = await this.connectionService.getModel('match');
const currentRedisTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
const ownerToken = randomUUID();
const identity = buildSideEffectApplicationIdentityFromPayload(
payload,
mutation,
);
const leaseResult = await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
{
...identity,
$or: [
{ state: 'pending' },
{ state: 'failed' },
{
state: 'processing',
processingLeaseExpiresAtTimestampMS: {
$lte: currentRedisTimestampMS,
},
},
],
} as never,
buildSideEffectLeaseAcquireUpdate({
currentRedisTimestampMS,
leaseDurationMS: AUCTION_SIDE_EFFECT_LEASE_DURATION_MS,
ownerToken,
}) as never,
),
);
if (!hasMatchedDocument(leaseResult)) return;
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
{
AuctionModel,
currentServerTimestampMS: currentRedisTimestampMS,
expectedMatchSideEffectRevision: Number(
payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
),
payload,
status: 'processing',
},
);
if (!processingDelta) return;
const expectedMatchSideEffectRevision =
processingDelta.matchSideEffectRevision;
try {
const match = (await execMaybe(
MatchModel.findById(payload.matchId) as never,
)) as {
competitionId?: unknown;
participants: Array<{
preferredPositions?: Array<{ position?: Position }>;
userId: { toString(): string };
}>;
save(): Promise<unknown>;
status: string;
teamBuiltAt?: Date;
teams: ReturnType<typeof buildMatchTeamsFromProjection>;
} | null;
if (!match)
throw new Error(`Match not found: ${String(payload.matchId)}`);
if (match.competitionId) {
await this.tournamentResultService.scheduleInitialPoll(match as never);
} else {
const matchTeams = buildMatchTeamsFromProjection(
payload.afterAuctionMongoProjection,
match,
);
// [AUCTION-DEBUG] 제거 대상. 최종 팀에 같은 유저가 2번 들어갔는지 무조건 검사
// (출처가 markSold든 undo 복원이든 projection이든 여기서 결정적으로 잡힌다).
const allMemberUserIds = matchTeams.flatMap((team) =>
team.members.map((member) => member.userId.toString()),
);
const distinctMemberCount = new Set(allMemberUserIds).size;
recordNewRelicCustomEvent('AuctionServerTeamBuild', {
matchId: typeof payload.matchId === 'string' ? payload.matchId : '',
auctionStateTransitionId: payload.auctionStateTransitionId,
teamCount: matchTeams.length,
totalMemberCount: allMemberUserIds.length,
distinctMemberCount,
hasDuplicateMember: allMemberUserIds.length !== distinctMemberCount,
duplicateUserId:
allMemberUserIds.find(
(id, idx) => allMemberUserIds.indexOf(id) !== idx,
) ?? null,
});
if (matchTeams.length === 0) {
throw new Error(
`Auction complete side effect produced no match teams for match ${String(payload.matchId)}.`,
);
}
const finalizedAt = match.teamBuiltAt ?? new Date();
match.teams = matchTeams;
match.status = 'team-built';
match.teamBuiltAt = undefined;
await match.save();
const preparedMatch =
(await this.tournamentCodeService.reserveForMatch(match as never)) ??
match;
preparedMatch.teamBuiltAt = finalizedAt;
await preparedMatch.save();
await this.tournamentResultService.scheduleInitialPoll(
preparedMatch as never,
);
}
const completedAtTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
$set: {
completedAtTimestampMS,
state: 'completed',
},
$unset: {
lastError: '',
processingLeaseExpiresAtTimestampMS: '',
processingOwnerToken: '',
processingStartedAtTimestampMS: '',
},
} as never,
),
);
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: completedAtTimestampMS,
errorMessage: undefined,
expectedMatchSideEffectRevision,
payload,
status: 'completed',
});
await this.expireCompletedAuctionRedisRuntime(payload);
} catch (error) {
const failureTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
$set: {
lastError: error instanceof Error ? error.message : String(error),
state: 'failed',
},
$unset: {
processingLeaseExpiresAtTimestampMS: '',
processingOwnerToken: '',
processingStartedAtTimestampMS: '',
},
} as never,
),
);
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: failureTimestampMS,
errorMessage: error instanceof Error ? error.message : String(error),
expectedMatchSideEffectRevision,
payload,
status: 'failed',
});
throw error;
}
}
private async applyStaleAppendHistoryRecovery(params: {
AuctionHistoryModel: {
findOne?: (...args: never[]) => unknown;
updateOne: (...args: never[]) => unknown;
};
payload: AuctionDurabilityPayload;
session: mongoose.ClientSession | null | undefined;
}): Promise<void> {
const invalidated =
await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params);
if (invalidated) return;
const historyOperation = buildAuctionHistoryUpsert(
params.payload,
).updateOne;
await execMaybe(
params.AuctionHistoryModel.updateOne(
historyOperation.filter as never,
historyOperation.update as never,
{
session: params.session,
upsert: true,
} as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
);
}
private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: {
AuctionHistoryModel: {
findOne?: (...args: never[]) => unknown;
};
payload: AuctionDurabilityPayload;
session: mongoose.ClientSession | null | undefined;
}): Promise<boolean> {
if (!params.AuctionHistoryModel.findOne) return true;
// 늦게 도착한 append는 history gap을 메울 수 있지만,
// 이후 undo/reset이 의도적으로 삭제한 sequence를 되살리면 안 된다.
const query = params.AuctionHistoryModel.findOne({
auctionId: params.payload.auctionId,
sequence: { $gt: params.payload.auctionHistorySequence },
$or: [
{
'historyMutation.targetHistorySequence': {
$lt: params.payload.auctionHistorySequence,
},
'historyMutation.type': 'deleteAfterSequenceAndAppendUndo',
},
{
'historyMutation.resetMode': { $ne: 'preserve-audit' },
'historyMutation.type': 'reset',
},
],
} as never);
const scoped =
query && typeof query === 'object' && 'session' in query
? (query as { session: (...args: never[]) => unknown }).session(
params.session as never,
)
: query;
const leaned =
scoped && typeof scoped === 'object' && 'lean' in scoped
? (scoped as { lean: () => unknown }).lean()
: scoped;
const result =
leaned && typeof leaned === 'object' && 'exec' in leaned
? await (leaned as { exec: () => Promise<unknown> }).exec()
: await Promise.resolve(leaned);
return Boolean(result);
}
private async applyResetMatchSideEffect(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
await this.applyMatchCleanupSideEffect({
mutation,
payload,
updateMatch: async (match) => {
await this.tournamentCodeService?.releaseFromMatch(match as never);
await this.tournamentResultService?.cancelPolling(match as never);
match.status = 'recruiting';
match.teams = [];
match.tournamentCodeIds = [];
match.gameSchedule = [];
match.participants.forEach((participant) => {
participant.excluded = false;
});
match.teamBuiltAt = undefined;
match.result = undefined;
match.resultPollState = undefined;
await match.save();
},
});
}
private async applyUndoCompletedMatchSideEffect(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
await this.applyMatchCleanupSideEffect({
mutation,
payload,
updateMatch: async (match) => {
await this.tournamentCodeService?.releaseFromMatch(match as never);
await this.tournamentResultService?.cancelPolling(match as never);
match.status = 'team-built';
match.result = undefined;
match.resultPollState = undefined;
await match.save();
},
});
}
private async applyMatchCleanupSideEffect(params: {
mutation: ParsedMatchSideEffectMutation;
payload: AuctionDurabilityPayload;
updateMatch: (match: CleanupSideEffectMatchDocument) => Promise<void>;
}): Promise<void> {
if (!this.tournamentCodeService || !this.tournamentResultService) {
throw new Error('Auction side-effect services are not available.');
}
const AuctionModel = await this.connectionService.getModel('auction');
const AuctionSideEffectApplicationModel =
await this.connectionService.getModel('auctionSideEffectApplication');
const MatchModel = await this.connectionService.getModel('match');
const currentRedisTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
const ownerToken = randomUUID();
const identity = buildSideEffectApplicationIdentityFromPayload(
params.payload,
params.mutation,
);
const leaseResult = await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
{
...identity,
$or: [
{ state: 'pending' },
{ state: 'failed' },
{
state: 'processing',
processingLeaseExpiresAtTimestampMS: {
$lte: currentRedisTimestampMS,
},
},
],
} as never,
buildSideEffectLeaseAcquireUpdate({
currentRedisTimestampMS,
leaseDurationMS: AUCTION_SIDE_EFFECT_LEASE_DURATION_MS,
ownerToken,
}) as never,
),
);
if (!hasMatchedDocument(leaseResult)) return;
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
{
AuctionModel,
currentServerTimestampMS: currentRedisTimestampMS,
expectedMatchSideEffectRevision: Number(
params.payload.afterAuctionMongoProjection.matchSideEffectRevision ??
0,
),
payload: params.payload,
status: 'processing',
},
);
if (!processingDelta) return;
const expectedMatchSideEffectRevision =
processingDelta.matchSideEffectRevision;
try {
const match = (await execMaybe(
MatchModel.findById(params.payload.matchId) as never,
)) as CleanupSideEffectMatchDocument | null;
if (!match) {
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
}
if (match.competitionId) {
throw new Error(
'Cannot reset or undo Match teams after the immutable Competition has been created.',
);
}
await params.updateMatch(match);
const completedAtTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
$set: {
completedAtTimestampMS,
state: 'completed',
},
$unset: {
lastError: '',
processingLeaseExpiresAtTimestampMS: '',
processingOwnerToken: '',
processingStartedAtTimestampMS: '',
},
} as never,
),
);
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: completedAtTimestampMS,
errorMessage: undefined,
expectedMatchSideEffectRevision,
payload: params.payload,
status: 'completed',
});
} catch (error) {
const failureTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
const errorMessage =
error instanceof Error ? error.message : String(error);
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
$set: {
lastError: errorMessage,
state: 'failed',
},
$unset: {
processingLeaseExpiresAtTimestampMS: '',
processingOwnerToken: '',
processingStartedAtTimestampMS: '',
},
} as never,
),
);
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: failureTimestampMS,
errorMessage,
expectedMatchSideEffectRevision,
payload: params.payload,
status: 'failed',
});
throw error;
}
}
private async applyAuctionHistoryMutation(params: {
AuctionHistoryModel: {
deleteMany?: (...args: never[]) => unknown;
updateOne: (...args: never[]) => unknown;
};
AuctionModel: {
findById?: (...args: never[]) => {
lean?: () => { exec?: () => Promise<unknown> };
session?: (...args: never[]) => {
lean?: () => { exec?: () => Promise<unknown> };
select?: (...args: never[]) => {
lean?: () => { exec?: () => Promise<unknown> };
session?: (...args: never[]) => unknown;
};
};
select?: (...args: never[]) => {
lean?: () => { exec?: () => Promise<unknown> };
session?: (...args: never[]) => unknown;
};
};
};
payload: AuctionDurabilityPayload;
session: mongoose.ClientSession | null | undefined;
}): Promise<void> {
const mutation = parseAuctionHistoryMutation(
params.payload.historyMutation,
);
if (isDestructiveHistoryMutation(mutation)) {
if (mutation.type === 'deleteAfterSequenceAndAppendUndo') {
if (!params.AuctionHistoryModel.deleteMany) {
throw new Error('Auction history deleteMany is not available.');
}
await execMaybe(
params.AuctionHistoryModel.deleteMany(
{
auctionId: params.payload.auctionId,
sequence: { $gt: mutation.targetHistorySequence },
} as never,
{ session: params.session } as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
);
}
if (mutation.type === 'reset' && mutation.resetMode === 'clear-history') {
if (!params.AuctionHistoryModel.deleteMany) {
throw new Error('Auction history deleteMany is not available.');
}
await execMaybe(
params.AuctionHistoryModel.deleteMany(
{
auctionId: params.payload.auctionId,
} as never,
{ session: params.session } as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
);
}
}
const historyOperation = buildAuctionHistoryUpsert(
params.payload,
).updateOne;
await execMaybe(
params.AuctionHistoryModel.updateOne(
historyOperation.filter as never,
historyOperation.update as never,
{
session: params.session,
upsert: true,
} as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
);
}
private async persistAndBroadcastMatchSideEffectStatus(params: {
AuctionModel: {
findOneAndUpdate(
filter: never,
update: never,
options: never,
): Promise<unknown> | { exec(): Promise<unknown> };
};
currentServerTimestampMS: number;
errorMessage?: string;
expectedMatchSideEffectRevision: number;
payload: AuctionDurabilityPayload;
status: AuctionMatchSideEffectStatus;
}): Promise<MatchSideEffectDelta | null> {
const updatedAuction = (await execMaybe(
params.AuctionModel.findOneAndUpdate(
{
_id: params.payload.auctionId,
auctionLifecycleId: params.payload.auctionLifecycleId,
matchSideEffectRevision: params.expectedMatchSideEffectRevision,
} as never,
{
$inc: { matchSideEffectRevision: 1 },
$set: {
...(params.errorMessage
? { matchSideEffectError: params.errorMessage }
: {}),
matchSideEffectStatus: params.status,
},
...(params.errorMessage
? {}
: { $unset: { matchSideEffectError: '' } }),
} as never,
{
projection: {
auctionLifecycleId: 1,
matchSideEffectRevision: 1,
matchSideEffectStatus: 1,
},
returnDocument: 'after',
} as never,
),
)) as {
auctionLifecycleId?: string;
matchSideEffectRevision?: number;
matchSideEffectStatus?: AuctionMatchSideEffectStatus;
} | null;
if (!updatedAuction?.auctionLifecycleId) return null;
const delta = buildMatchSideEffectDelta({
auctionLifecycleId: updatedAuction.auctionLifecycleId,
currentServerTimestampMS: params.currentServerTimestampMS,
matchSideEffectRevision: Number(
updatedAuction.matchSideEffectRevision ?? 0,
),
matchSideEffectStatus:
updatedAuction.matchSideEffectStatus ?? params.status,
});
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
auctionLifecycleId: delta.auctionLifecycleId,
matchId: String(params.payload.matchId),
matchSideEffectRevision: delta.matchSideEffectRevision,
matchSideEffectStatus: delta.matchSideEffectStatus,
});
this.auctionEventBusService?.broadcastMatchSideEffect(
String(params.payload.matchId),
delta,
);
return delta;
}
private async safeTick(): Promise<void> {
if (this.tickInProgress) return;
this.tickInProgress = true;
const leaderLockToken = randomUUID();
let leaderLockAcquired = false;
try {
leaderLockAcquired = await this.auctionRedisService.acquireTokenLock(
AUCTION_DURABILITY_LEADER_LOCK_KEY,
leaderLockToken,
AUCTION_DURABILITY_LEADER_LOCK_TTL_MS,
);
if (!leaderLockAcquired) return;
await this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT);
} catch (error) {
this.logger.warn(
`Auction durability tick failed: ${
error instanceof Error ? error.message : error
}`,
);
} finally {
if (leaderLockAcquired) {
await this.auctionRedisService.releaseTokenLock(
AUCTION_DURABILITY_LEADER_LOCK_KEY,
leaderLockToken,
);
}
this.tickInProgress = false;
}
}
private async clearDurabilityPayload(payloadKey: string): Promise<void> {
await this.auctionRedisService
.raw()
.sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]);
await this.auctionRedisService.raw().sendCommand(['DEL', payloadKey]);
}
private async requeueDurabilityPayload(
payloadKey: string,
retryAtRedisTimestampMS: number,
): Promise<void> {
await this.auctionRedisService
.raw()
.sendCommand([
'ZADD',
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
String(retryAtRedisTimestampMS),
payloadKey,
]);
}
private async recordDurabilityPayloadFailure(
payloadKey: string,
currentRedisTimestampMS: number,
error: unknown,
): Promise<'dead-lettered' | 'requeued'> {
const errorMessage = formatErrorMessageForRedis(error);
const attemptReply = await this.auctionRedisService
.raw()
.sendCommand(['HINCRBY', payloadKey, 'attempts', '1']);
const attempts = Number(attemptReply);
await this.auctionRedisService
.raw()
.sendCommand([
'HSET',
payloadKey,
'lastError',
errorMessage,
'lastFailedAtRedisTimestampMS',
String(currentRedisTimestampMS),
]);
if (
Number.isFinite(attempts) &&
attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS
) {
await this.auctionRedisService
.raw()
.sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]);
await this.auctionRedisService
.raw()
.sendCommand([
'HSET',
payloadKey,
'deadLetteredAtRedisTimestampMS',
String(currentRedisTimestampMS),
]);
await this.auctionRedisService
.raw()
.sendCommand([
'ZADD',
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
String(currentRedisTimestampMS),
payloadKey,
]);
return 'dead-lettered';
}
await this.requeueDurabilityPayload(
payloadKey,
currentRedisTimestampMS + calculateDurabilityRetryDelayMS(attempts),
);
return 'requeued';
}
private async expireCompletedAuctionRedisRuntime(
payload: AuctionDurabilityPayload,
): Promise<void> {
const matchId = String(payload.matchId);
const keys = buildAuctionRedisKeys(matchId);
await this.auctionRedisService
.raw()
.sendCommand([
'EVAL',
[
'local expectedAuctionLifecycleId = ARGV[2]',
"local currentAuctionLifecycleId = redis.call('HGET', KEYS[1], 'auctionLifecycleId')",
'if currentAuctionLifecycleId ~= expectedAuctionLifecycleId then',
' return 0',
'end',
"redis.call('ZREM', KEYS[2], ARGV[1])",
'for index = 1, #KEYS do',
' if index ~= 2 then',
" redis.call('EXPIRE', KEYS[index], ARGV[3])",
' end',
'end',
'return 1',
].join('\n'),
'11',
keys.liveStateKey,
keys.deadlineQueueKey,
keys.structureStateKey,
keys.appliedTransitionIdsKey,
keys.bidHistoryKey,
keys.bidRequestResultsKey,
keys.bidRequestResultOrderKey,
keys.actionRequestResultsKey,
keys.actionRequestResultOrderKey,
keys.rehydrateLockKey,
keys.stateTransitionLockKey,
matchId,
payload.auctionLifecycleId,
String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS),
]);
}
}
export function buildAuctionHistoryUpsert(payload: AuctionDurabilityPayload) {
return {
updateOne: {
filter: {
auctionId: payload.auctionId,
auctionStateTransitionId: payload.auctionStateTransitionId,
},
update: {
$setOnInsert: {
action: payload.auctionActionType,
actorUserId: payload.actorUserId,
auctionId: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
auctionStateTransitionId: payload.auctionStateTransitionId,
historyMutation: payload.historyMutation,
matchId: payload.matchId,
matchSideEffectMutation: payload.matchSideEffectMutation,
revision: payload.revision,
sequence: payload.auctionHistorySequence,
snapshotBefore: payload.snapshotBefore,
takenAt: new Date(payload.committedAtRedisTimestampMS ?? 0),
},
},
upsert: true,
},
};
}
export function parseAuctionHistoryMutation(
mutation: unknown,
): ParsedAuctionHistoryMutation {
if (!mutation || typeof mutation !== 'object') return { type: 'append' };
const record = mutation as Record<string, unknown>;
if (record.type === 'deleteAfterSequenceAndAppendUndo') {
const targetHistorySequence = Number(record.targetHistorySequence);
if (!Number.isFinite(targetHistorySequence) || targetHistorySequence < 0) {
throw new Error('Invalid undo history mutation target sequence.');
}
return {
targetHistorySequence,
type: 'deleteAfterSequenceAndAppendUndo',
};
}
if (record.type === 'reset') {
const resetMode =
record.resetMode === 'preserve-audit'
? 'preserve-audit'
: 'clear-history';
return { resetMode, type: 'reset' };
}
return { type: 'append' };
}
function isDestructiveHistoryMutation(
mutation: ParsedAuctionHistoryMutation,
): boolean {
return (
mutation.type === 'deleteAfterSequenceAndAppendUndo' ||
(mutation.type === 'reset' && mutation.resetMode === 'clear-history')
);
}
async function readAuctionProjectionIdentityForMutation(
AuctionModel: {
findById?: (...args: never[]) => unknown;
},
auctionId: unknown,
session: mongoose.ClientSession | null | undefined,
): Promise<AuctionProjectionIdentity> {
if (!AuctionModel.findById) return {};
const query = AuctionModel.findById(auctionId as never);
const selected =
query && typeof query === 'object' && 'select' in query
? (query as { select: (...args: never[]) => unknown }).select({
auctionLifecycleId: 1,
revision: 1,
} as never)
: query;
const scoped =
selected && typeof selected === 'object' && 'session' in selected
? (selected as { session: (...args: never[]) => unknown }).session(
session as never,
)
: selected;
const leaned =
scoped && typeof scoped === 'object' && 'lean' in scoped
? (scoped as { lean: () => unknown }).lean()
: scoped;
const projection =
leaned && typeof leaned === 'object' && 'exec' in leaned
? await (leaned as { exec: () => Promise<unknown> }).exec()
: await Promise.resolve(leaned);
if (!projection || typeof projection !== 'object') return {};
const record = projection as Record<string, unknown>;
return {
auctionLifecycleId:
typeof record.auctionLifecycleId === 'string'
? record.auctionLifecycleId
: undefined,
revision:
typeof record.revision === 'number'
? record.revision
: Number(record.revision),
};
}
export function buildAuctionProjectionGuardedUpdate(
payload: AuctionDurabilityPayload,
) {
return {
filter: {
_id: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
revision: { $lt: payload.revision },
},
update: {
$set: payload.afterAuctionMongoProjection,
},
};
}
export function shouldApplyAuctionProjection(
current: AuctionProjectionIdentity,
payload: Pick<AuctionDurabilityPayload, 'auctionLifecycleId' | 'revision'>,
): boolean {
if (current.auctionLifecycleId !== payload.auctionLifecycleId) return false;
return (current.revision ?? -1) < payload.revision;
}
export function shouldProcessAuctionDurabilityPayload(
current: AuctionProjectionIdentity,
payload: Pick<AuctionDurabilityPayload, 'auctionLifecycleId' | 'revision'>,
): boolean {
if (current.auctionLifecycleId !== payload.auctionLifecycleId) return false;
return (current.revision ?? -1) <= payload.revision;
}
export function buildSideEffectApplicationUpsert(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
) {
const identity = buildSideEffectApplicationIdentityFromPayload(
payload,
mutation,
);
return {
filter: identity,
update: {
$setOnInsert: {
...identity,
auctionLifecycleId: payload.auctionLifecycleId,
matchId: payload.matchId,
attempts: 0,
payload: mutation,
payloadChecksum: hashStableJson(mutation),
state: 'pending',
},
},
};
}
function parseMatchSideEffectMutation(
value: Record<string, unknown> | undefined,
): ParsedMatchSideEffectMutation | null {
if (value?.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) {
return { type: SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE };
}
if (value?.type === RESET_MATCH_SIDE_EFFECT_TYPE) {
return { type: RESET_MATCH_SIDE_EFFECT_TYPE };
}
if (value?.type === UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE) {
return { type: UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE };
}
return null;
}
function buildSideEffectApplicationIdentityFromPayload(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
) {
return {
auctionId: payload.auctionId,
auctionStateTransitionId: payload.auctionStateTransitionId,
sideEffectType: mutation.type,
};
}
function hashStableJson(value: unknown): string {
return createHash('sha256')
.update(
JSON.stringify(
value,
Object.keys(value as Record<string, unknown>).sort(),
),
)
.digest('hex');
}
function hasMatchedDocument(result: unknown): boolean {
if (!result || typeof result !== 'object') return false;
const record = result as Record<string, unknown>;
const matchedCount = Number(record.matchedCount ?? record.modifiedCount ?? 0);
return matchedCount > 0;
}
function buildMatchTeamsFromProjection(
projection: Record<string, unknown>,
match: {
participants: Array<{
preferredPositions?: Array<{ position?: string }>;
userId: { toString(): string };
}>;
},
) {
const auctionTeams = Array.isArray(projection.auctionTeams)
? (projection.auctionTeams as Array<Record<string, unknown>>)
: [];
const participantByUserId = new Map(
match.participants.map((participant) => [
participant.userId.toString(),
participant,
]),
);
return auctionTeams.map((team) => {
const teamIndex = Number(team.teamIndex);
const captainUserId = team.captainUserId as MatchTeamMember['userId'];
const captain = participantByUserId.get(captainUserId.toString());
const rawMembers = Array.isArray(team.members)
? (team.members as Array<Record<string, unknown>>)
: [];
const members: MatchTeamMember[] = [
{
assignedPosition:
(captain?.preferredPositions?.[0]?.position as
| Position
| undefined) ?? 'MID',
cost: 0,
userId: captainUserId,
},
...rawMembers
.filter(
(member) => member.userId?.toString() !== captainUserId.toString(),
)
.map((member) => ({
assignedPosition: (member.assignedPosition ?? 'MID') as Position,
cost: Number(member.memberAcquisitionBidPointAmount ?? 0),
userId: member.userId as MatchTeamMember['userId'],
})),
];
return {
captainUserId,
members,
name: `${teamIndex}팀`,
teamIndex,
};
});
}
function parseRedisHash(reply: unknown): Record<string, string> {
if (!reply) return {};
if (Array.isArray(reply)) {
const result: Record<string, string> = {};
for (let index = 0; index < reply.length; index += 2) {
result[String(reply[index])] = String(reply[index + 1]);
}
return result;
}
if (typeof reply === 'object') {
return Object.fromEntries(
Object.entries(reply as Record<string, unknown>).map(([key, value]) => [
key,
String(value),
]),
);
}
return {};
}
function formatErrorMessageForRedis(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message.slice(0, AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH);
}
function calculateDurabilityRetryDelayMS(attempts: number): number {
if (!Number.isFinite(attempts) || attempts <= 1) {
return AUCTION_DURABILITY_RETRY_DELAY_MS;
}
return Math.min(
AUCTION_DURABILITY_MAX_RETRY_DELAY_MS,
AUCTION_DURABILITY_RETRY_DELAY_MS * 2 ** (attempts - 1),
);
}
async function execMaybe<T>(
queryOrPromise: Promise<T> | { exec(): Promise<T> },
) {
if (
queryOrPromise &&
typeof (queryOrPromise as { exec?: unknown }).exec === 'function'
) {
return (queryOrPromise as { exec(): Promise<T> }).exec();
}
return queryOrPromise;
}