import { Injectable, Logger, OnModuleDestroy, OnModuleInit, Optional, } from '@nestjs/common'; import { createHash, randomUUID } from 'crypto'; import type mongoose from 'mongoose'; import { ConnectionService } from '../global/database/connection.service'; import type { AuctionMatchSideEffectStatus, MatchTeamMember, Position, } from '../global/database/schema'; import { TournamentCodeService } from '../tournament-code/tournament-code.service'; import { TournamentResultService } from '../tournament-result/tournament-result.service'; import { AuctionEventBusService } from './auction-event-bus.service'; import { AuctionSideEffectService, buildMatchSideEffectDelta, buildSideEffectLeaseAcquireUpdate, type MatchSideEffectDelta, } from './auction-side-effect.service'; import { AuctionRedisService } from './redis/auction-redis.service'; import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service'; import { AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, AUCTION_DURABILITY_RETRY_QUEUE_KEY, buildAuctionRedisKeys, } from './redis/auction-redis.keys'; // [AUCTION-DEBUG] 경매 "중복 멤버" 조사용 임시 계측. 조사 종료 후 제거 대상. import { recordNewRelicCustomEvent } from '../global/observability/new-relic'; const AUCTION_DURABILITY_TICK_INTERVAL_MS = 1000; const AUCTION_DURABILITY_BATCH_LIMIT = 20; const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000; const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000; const AUCTION_DURABILITY_MAX_ATTEMPTS = 10; const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000; const AUCTION_DURABILITY_LEADER_LOCK_KEY = 'auction:auctionDurabilityScheduler:leader'; const AUCTION_DURABILITY_LEADER_LOCK_TTL_MS = 2500; const AUCTION_REDIS_COMPLETED_RETAIN_SECONDS = 7 * 24 * 60 * 60; const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000; const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete'; const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset'; const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects'; export interface AuctionDurabilityPayload { actorUserId?: unknown; afterAuctionMongoProjection: Record; auctionActionType: string; auctionHistorySequence: number; auctionId: unknown; auctionLifecycleId: string; auctionStateTransitionId: string; committedAtRedisTimestampMS?: number; historyMutation?: Record; matchId: unknown; matchSideEffectMutation: Record; shouldBroadcastState?: boolean; broadcastStateRevision?: number; revision: number; snapshotBefore: Record; } export interface AuctionProjectionIdentity { auctionLifecycleId?: string; revision?: number; } type ParsedAuctionHistoryMutation = | { type: 'append' } | { targetHistorySequence: number; type: 'deleteAfterSequenceAndAppendUndo' } | { resetMode: 'clear-history' | 'preserve-audit'; type: 'reset' }; type ParsedMatchSideEffectMutation = | { type: typeof SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE } | { type: typeof RESET_MATCH_SIDE_EFFECT_TYPE } | { type: typeof UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE }; interface CleanupSideEffectMatchDocument { competitionId?: unknown; gameSchedule?: unknown[]; participants: Array<{ excluded?: boolean }>; result?: unknown; resultPollState?: unknown; save(): Promise; status: string; teamBuiltAt?: Date; teams?: unknown[]; tournamentCodeIds?: unknown[]; } @Injectable() export class AuctionDurabilityScheduler implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(AuctionDurabilityScheduler.name); private tickTimer?: NodeJS.Timeout; private tickInProgress = false; constructor( private readonly connectionService: ConnectionService, private readonly auctionRedisService: AuctionRedisService, @Optional() private readonly tournamentCodeService?: TournamentCodeService, @Optional() private readonly tournamentResultService?: TournamentResultService, @Optional() private readonly auctionSideEffectService?: AuctionSideEffectService, @Optional() private readonly auctionEventBusService?: AuctionEventBusService, @Optional() private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService, ) {} public onModuleInit(): void { this.tickTimer = setInterval( () => void this.safeTick(), AUCTION_DURABILITY_TICK_INTERVAL_MS, ); this.tickTimer.unref?.(); } public onModuleDestroy(): void { if (this.tickTimer) clearInterval(this.tickTimer); } public async readDueDurabilityPayloadKeys( currentRedisTimestampMS: number, batchLimit: number, ): Promise { const reply = await this.auctionRedisService .raw() .sendCommand([ 'ZRANGEBYSCORE', AUCTION_DURABILITY_RETRY_QUEUE_KEY, '-inf', String(currentRedisTimestampMS), 'LIMIT', '0', String(batchLimit), ]); return Array.isArray(reply) ? reply.map(String) : []; } public async processDueDurabilityPayloads(batchLimit: number): Promise<{ applied: number; failed: number; skipped: number; }> { const currentRedisTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const payloadKeys = await this.readDueDurabilityPayloadKeys( currentRedisTimestampMS, batchLimit, ); let applied = 0; let failed = 0; let skipped = 0; for (const payloadKey of payloadKeys) { try { const payload = await this.readDurabilityPayload(payloadKey); if (!payload) { skipped += 1; await this.clearDurabilityPayload(payloadKey); continue; } await this.applyDurabilityPayload(payload); applied += 1; // [AUCTION-DEBUG] 제거 대상. 영속 지연(lag): Redis 커밋 → Mongo 영속까지 걸린 시간 + 배치 백로그. // 사용자가 본 "되돌리기/이력이 2~3 뒤처짐"을 수치화한다(지연이 클수록 undo // 패널이 stale 시퀀스를 보여 엉뚱한 스냅샷 복원 가능성↑). recordNewRelicCustomEvent('AuctionServerDurability', { matchId: typeof payload.matchId === 'string' ? payload.matchId : '', auctionStateTransitionId: payload.auctionStateTransitionId, auctionActionType: payload.auctionActionType, revision: payload.revision, auctionHistorySequence: payload.auctionHistorySequence, persistLatencyMS: typeof payload.committedAtRedisTimestampMS === 'number' ? currentRedisTimestampMS - payload.committedAtRedisTimestampMS : -1, batchBacklog: payloadKeys.length, }); await this.clearDurabilityPayload(payloadKey); } catch (error) { failed += 1; const failureDisposition = await this.recordDurabilityPayloadFailure( payloadKey, currentRedisTimestampMS, error, ); this.logger.warn( `Auction durability payload failed for ${payloadKey} (${failureDisposition}): ${ error instanceof Error ? error.message : error }`, ); } } return { applied, failed, skipped }; } public async readDurabilityPayload( payloadKey: string, ): Promise { const reply = await this.auctionRedisService .raw() .sendCommand(['HGETALL', payloadKey]); const hash = parseRedisHash(reply); const payloadJson = hash.payloadJson; if (!payloadJson) return null; const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload; const committedAtRedisTimestampMS = Number( hash.committedAtRedisTimestampMS, ); if (Number.isFinite(committedAtRedisTimestampMS)) { payload.committedAtRedisTimestampMS = committedAtRedisTimestampMS; } payload.shouldBroadcastState = hash.shouldBroadcastState === '1'; const broadcastStateRevision = Number(hash.broadcastStateRevision); if (Number.isFinite(broadcastStateRevision)) { payload.broadcastStateRevision = broadcastStateRevision; } return payload; } public async applyDurabilityPayload( payload: AuctionDurabilityPayload, ): Promise { const AuctionHistoryModel = await this.connectionService.getModel('auctionHistory'); const AuctionModel = await this.connectionService.getModel('auction'); const AuctionSideEffectApplicationModel = await this.connectionService.getModel('auctionSideEffectApplication'); const session = await this.connectionService.createSession(); const sideEffectMutation = parseMatchSideEffectMutation( payload.matchSideEffectMutation, ); try { session?.startTransaction(); const historyMutation = parseAuctionHistoryMutation( payload.historyMutation, ); if (typeof AuctionModel.findById === 'function') { const currentProjection = await readAuctionProjectionIdentityForMutation( AuctionModel, payload.auctionId, session, ); if ( !shouldProcessAuctionDurabilityPayload(currentProjection, payload) ) { // 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다. // 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다. if ( historyMutation.type === 'append' && currentProjection.auctionLifecycleId === payload.auctionLifecycleId ) { await this.applyStaleAppendHistoryRecovery({ AuctionHistoryModel, payload, session, }); } if (session) await session.commitTransaction(); return; } } await this.applyAuctionHistoryMutation({ AuctionHistoryModel, AuctionModel, payload, session, }); const projectionOperation = buildAuctionProjectionGuardedUpdate(payload); await execMaybe( AuctionModel.updateOne( projectionOperation.filter as never, projectionOperation.update as never, { session } as never, ), ); if (sideEffectMutation) { const sideEffectOperation = buildSideEffectApplicationUpsert( payload, sideEffectMutation, ); await execMaybe( AuctionSideEffectApplicationModel.updateOne( sideEffectOperation.filter as never, sideEffectOperation.update as never, { session, upsert: true, } as never, ), ); } if (session) await session.commitTransaction(); } catch (error) { if (session) await session.abortTransaction(); throw error; } finally { if (session) await session.endSession(); } await this.broadcastLatestRedisAuctionStateIfRequested(payload); if (sideEffectMutation) { await this.applyMatchSideEffect(payload, sideEffectMutation); } } /** * 노드 크래시 복구용 백업 broadcast(1차 전달 아님). * * 정상 경로에서는 요청을 처리한 게이트웨이(수동 전환) 또는 * `AuctionDeadlineScheduler`(자동 만료)가 commit 직후 동기로 `state`를 방송한다. * 이 메서드는 "방송을 하기로 한 인스턴스가 broadcast 전에 죽은" 경우를 위해 * outbox를 처리하면서 최신 state를 한 번 더 방송하는 안전망이다. * 이 호출은 위 try 블록의 Mongo 트랜잭션이 성공해야(early return/throw 없이) * 도달하므로 1차 전달로 의존해서는 안 된다. 중복 방송은 클라 revision 가드로 무해. */ private async broadcastLatestRedisAuctionStateIfRequested( payload: AuctionDurabilityPayload, ): Promise { if (!payload.shouldBroadcastState) return; if (!this.auctionRedisStateReaderService || !this.auctionEventBusService) { return; } const matchId = String(payload.matchId); const state = await this.auctionRedisStateReaderService.readAuctionStateView(matchId); if (!state) return; const stateRevision = Number(state.revision); if ( Number.isFinite(payload.broadcastStateRevision) && Number.isFinite(stateRevision) && stateRevision < Number(payload.broadcastStateRevision) ) { return; } this.auctionEventBusService.broadcastState(matchId, state); } private async applyMatchSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) { await this.applySyncMatchTeamsSideEffect(payload, mutation); return; } if (mutation.type === RESET_MATCH_SIDE_EFFECT_TYPE) { await this.applyResetMatchSideEffect(payload, mutation); return; } if (mutation.type === UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE) { await this.applyUndoCompletedMatchSideEffect(payload, mutation); } } private async applySyncMatchTeamsSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { if (!this.tournamentCodeService || !this.tournamentResultService) { throw new Error('Auction side-effect services are not available.'); } const AuctionModel = await this.connectionService.getModel('auction'); const AuctionSideEffectApplicationModel = await this.connectionService.getModel('auctionSideEffectApplication'); const MatchModel = await this.connectionService.getModel('match'); const currentRedisTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const ownerToken = randomUUID(); const identity = buildSideEffectApplicationIdentityFromPayload( payload, mutation, ); const leaseResult = await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, $or: [ { state: 'pending' }, { state: 'failed' }, { state: 'processing', processingLeaseExpiresAtTimestampMS: { $lte: currentRedisTimestampMS, }, }, ], } as never, buildSideEffectLeaseAcquireUpdate({ currentRedisTimestampMS, leaseDurationMS: AUCTION_SIDE_EFFECT_LEASE_DURATION_MS, ownerToken, }) as never, ), ); if (!hasMatchedDocument(leaseResult)) return; const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus( { AuctionModel, currentServerTimestampMS: currentRedisTimestampMS, expectedMatchSideEffectRevision: Number( payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0, ), payload, status: 'processing', }, ); if (!processingDelta) return; const expectedMatchSideEffectRevision = processingDelta.matchSideEffectRevision; try { const match = (await execMaybe( MatchModel.findById(payload.matchId) as never, )) as { competitionId?: unknown; participants: Array<{ preferredPositions?: Array<{ position?: Position }>; userId: { toString(): string }; }>; save(): Promise; status: string; teamBuiltAt?: Date; teams: ReturnType; } | null; if (!match) throw new Error(`Match not found: ${String(payload.matchId)}`); if (match.competitionId) { await this.tournamentResultService.scheduleInitialPoll(match as never); } else { const matchTeams = buildMatchTeamsFromProjection( payload.afterAuctionMongoProjection, match, ); // [AUCTION-DEBUG] 제거 대상. 최종 팀에 같은 유저가 2번 들어갔는지 무조건 검사 // (출처가 markSold든 undo 복원이든 projection이든 여기서 결정적으로 잡힌다). const allMemberUserIds = matchTeams.flatMap((team) => team.members.map((member) => member.userId.toString()), ); const distinctMemberCount = new Set(allMemberUserIds).size; recordNewRelicCustomEvent('AuctionServerTeamBuild', { matchId: typeof payload.matchId === 'string' ? payload.matchId : '', auctionStateTransitionId: payload.auctionStateTransitionId, teamCount: matchTeams.length, totalMemberCount: allMemberUserIds.length, distinctMemberCount, hasDuplicateMember: allMemberUserIds.length !== distinctMemberCount, duplicateUserId: allMemberUserIds.find( (id, idx) => allMemberUserIds.indexOf(id) !== idx, ) ?? null, }); if (matchTeams.length === 0) { throw new Error( `Auction complete side effect produced no match teams for match ${String(payload.matchId)}.`, ); } const finalizedAt = match.teamBuiltAt ?? new Date(); match.teams = matchTeams; match.status = 'team-built'; match.teamBuiltAt = undefined; await match.save(); const preparedMatch = (await this.tournamentCodeService.reserveForMatch(match as never)) ?? match; preparedMatch.teamBuiltAt = finalizedAt; await preparedMatch.save(); await this.tournamentResultService.scheduleInitialPoll( preparedMatch as never, ); } const completedAtTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( identity as never, { $set: { completedAtTimestampMS, state: 'completed', }, $unset: { lastError: '', processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: completedAtTimestampMS, errorMessage: undefined, expectedMatchSideEffectRevision, payload, status: 'completed', }); await this.expireCompletedAuctionRedisRuntime(payload); } catch (error) { const failureTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( identity as never, { $set: { lastError: error instanceof Error ? error.message : String(error), state: 'failed', }, $unset: { processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: failureTimestampMS, errorMessage: error instanceof Error ? error.message : String(error), expectedMatchSideEffectRevision, payload, status: 'failed', }); throw error; } } private async applyStaleAppendHistoryRecovery(params: { AuctionHistoryModel: { findOne?: (...args: never[]) => unknown; updateOne: (...args: never[]) => unknown; }; payload: AuctionDurabilityPayload; session: mongoose.ClientSession | null | undefined; }): Promise { const invalidated = await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params); if (invalidated) return; const historyOperation = buildAuctionHistoryUpsert( params.payload, ).updateOne; await execMaybe( params.AuctionHistoryModel.updateOne( historyOperation.filter as never, historyOperation.update as never, { session: params.session, upsert: true, } as never, ) as Promise | { exec(): Promise }, ); } private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: { AuctionHistoryModel: { findOne?: (...args: never[]) => unknown; }; payload: AuctionDurabilityPayload; session: mongoose.ClientSession | null | undefined; }): Promise { if (!params.AuctionHistoryModel.findOne) return true; // 늦게 도착한 append는 history gap을 메울 수 있지만, // 이후 undo/reset이 의도적으로 삭제한 sequence를 되살리면 안 된다. const query = params.AuctionHistoryModel.findOne({ auctionId: params.payload.auctionId, sequence: { $gt: params.payload.auctionHistorySequence }, $or: [ { 'historyMutation.targetHistorySequence': { $lt: params.payload.auctionHistorySequence, }, 'historyMutation.type': 'deleteAfterSequenceAndAppendUndo', }, { 'historyMutation.resetMode': { $ne: 'preserve-audit' }, 'historyMutation.type': 'reset', }, ], } as never); const scoped = query && typeof query === 'object' && 'session' in query ? (query as { session: (...args: never[]) => unknown }).session( params.session as never, ) : query; const leaned = scoped && typeof scoped === 'object' && 'lean' in scoped ? (scoped as { lean: () => unknown }).lean() : scoped; const result = leaned && typeof leaned === 'object' && 'exec' in leaned ? await (leaned as { exec: () => Promise }).exec() : await Promise.resolve(leaned); return Boolean(result); } private async applyResetMatchSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { await this.applyMatchCleanupSideEffect({ mutation, payload, updateMatch: async (match) => { await this.tournamentCodeService?.releaseFromMatch(match as never); await this.tournamentResultService?.cancelPolling(match as never); match.status = 'recruiting'; match.teams = []; match.tournamentCodeIds = []; match.gameSchedule = []; match.participants.forEach((participant) => { participant.excluded = false; }); match.teamBuiltAt = undefined; match.result = undefined; match.resultPollState = undefined; await match.save(); }, }); } private async applyUndoCompletedMatchSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { await this.applyMatchCleanupSideEffect({ mutation, payload, updateMatch: async (match) => { await this.tournamentCodeService?.releaseFromMatch(match as never); await this.tournamentResultService?.cancelPolling(match as never); match.status = 'team-built'; match.result = undefined; match.resultPollState = undefined; await match.save(); }, }); } private async applyMatchCleanupSideEffect(params: { mutation: ParsedMatchSideEffectMutation; payload: AuctionDurabilityPayload; updateMatch: (match: CleanupSideEffectMatchDocument) => Promise; }): Promise { if (!this.tournamentCodeService || !this.tournamentResultService) { throw new Error('Auction side-effect services are not available.'); } const AuctionModel = await this.connectionService.getModel('auction'); const AuctionSideEffectApplicationModel = await this.connectionService.getModel('auctionSideEffectApplication'); const MatchModel = await this.connectionService.getModel('match'); const currentRedisTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const ownerToken = randomUUID(); const identity = buildSideEffectApplicationIdentityFromPayload( params.payload, params.mutation, ); const leaseResult = await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, $or: [ { state: 'pending' }, { state: 'failed' }, { state: 'processing', processingLeaseExpiresAtTimestampMS: { $lte: currentRedisTimestampMS, }, }, ], } as never, buildSideEffectLeaseAcquireUpdate({ currentRedisTimestampMS, leaseDurationMS: AUCTION_SIDE_EFFECT_LEASE_DURATION_MS, ownerToken, }) as never, ), ); if (!hasMatchedDocument(leaseResult)) return; const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus( { AuctionModel, currentServerTimestampMS: currentRedisTimestampMS, expectedMatchSideEffectRevision: Number( params.payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0, ), payload: params.payload, status: 'processing', }, ); if (!processingDelta) return; const expectedMatchSideEffectRevision = processingDelta.matchSideEffectRevision; try { const match = (await execMaybe( MatchModel.findById(params.payload.matchId) as never, )) as CleanupSideEffectMatchDocument | null; if (!match) { throw new Error(`Match not found: ${String(params.payload.matchId)}`); } if (match.competitionId) { throw new Error( 'Cannot reset or undo Match teams after the immutable Competition has been created.', ); } await params.updateMatch(match); const completedAtTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( identity as never, { $set: { completedAtTimestampMS, state: 'completed', }, $unset: { lastError: '', processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: completedAtTimestampMS, errorMessage: undefined, expectedMatchSideEffectRevision, payload: params.payload, status: 'completed', }); } catch (error) { const failureTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const errorMessage = error instanceof Error ? error.message : String(error); await execMaybe( AuctionSideEffectApplicationModel.updateOne( identity as never, { $set: { lastError: errorMessage, state: 'failed', }, $unset: { processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: failureTimestampMS, errorMessage, expectedMatchSideEffectRevision, payload: params.payload, status: 'failed', }); throw error; } } private async applyAuctionHistoryMutation(params: { AuctionHistoryModel: { deleteMany?: (...args: never[]) => unknown; updateOne: (...args: never[]) => unknown; }; AuctionModel: { findById?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; session?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; select?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; session?: (...args: never[]) => unknown; }; }; select?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; session?: (...args: never[]) => unknown; }; }; }; payload: AuctionDurabilityPayload; session: mongoose.ClientSession | null | undefined; }): Promise { const mutation = parseAuctionHistoryMutation( params.payload.historyMutation, ); if (isDestructiveHistoryMutation(mutation)) { if (mutation.type === 'deleteAfterSequenceAndAppendUndo') { if (!params.AuctionHistoryModel.deleteMany) { throw new Error('Auction history deleteMany is not available.'); } await execMaybe( params.AuctionHistoryModel.deleteMany( { auctionId: params.payload.auctionId, sequence: { $gt: mutation.targetHistorySequence }, } as never, { session: params.session } as never, ) as Promise | { exec(): Promise }, ); } if (mutation.type === 'reset' && mutation.resetMode === 'clear-history') { if (!params.AuctionHistoryModel.deleteMany) { throw new Error('Auction history deleteMany is not available.'); } await execMaybe( params.AuctionHistoryModel.deleteMany( { auctionId: params.payload.auctionId, } as never, { session: params.session } as never, ) as Promise | { exec(): Promise }, ); } } const historyOperation = buildAuctionHistoryUpsert( params.payload, ).updateOne; await execMaybe( params.AuctionHistoryModel.updateOne( historyOperation.filter as never, historyOperation.update as never, { session: params.session, upsert: true, } as never, ) as Promise | { exec(): Promise }, ); } private async persistAndBroadcastMatchSideEffectStatus(params: { AuctionModel: { findOneAndUpdate( filter: never, update: never, options: never, ): Promise | { exec(): Promise }; }; currentServerTimestampMS: number; errorMessage?: string; expectedMatchSideEffectRevision: number; payload: AuctionDurabilityPayload; status: AuctionMatchSideEffectStatus; }): Promise { const updatedAuction = (await execMaybe( params.AuctionModel.findOneAndUpdate( { _id: params.payload.auctionId, auctionLifecycleId: params.payload.auctionLifecycleId, matchSideEffectRevision: params.expectedMatchSideEffectRevision, } as never, { $inc: { matchSideEffectRevision: 1 }, $set: { ...(params.errorMessage ? { matchSideEffectError: params.errorMessage } : {}), matchSideEffectStatus: params.status, }, ...(params.errorMessage ? {} : { $unset: { matchSideEffectError: '' } }), } as never, { projection: { auctionLifecycleId: 1, matchSideEffectRevision: 1, matchSideEffectStatus: 1, }, returnDocument: 'after', } as never, ), )) as { auctionLifecycleId?: string; matchSideEffectRevision?: number; matchSideEffectStatus?: AuctionMatchSideEffectStatus; } | null; if (!updatedAuction?.auctionLifecycleId) return null; const delta = buildMatchSideEffectDelta({ auctionLifecycleId: updatedAuction.auctionLifecycleId, currentServerTimestampMS: params.currentServerTimestampMS, matchSideEffectRevision: Number( updatedAuction.matchSideEffectRevision ?? 0, ), matchSideEffectStatus: updatedAuction.matchSideEffectStatus ?? params.status, }); await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({ auctionLifecycleId: delta.auctionLifecycleId, matchId: String(params.payload.matchId), matchSideEffectRevision: delta.matchSideEffectRevision, matchSideEffectStatus: delta.matchSideEffectStatus, }); this.auctionEventBusService?.broadcastMatchSideEffect( String(params.payload.matchId), delta, ); return delta; } private async safeTick(): Promise { if (this.tickInProgress) return; this.tickInProgress = true; const leaderLockToken = randomUUID(); let leaderLockAcquired = false; try { leaderLockAcquired = await this.auctionRedisService.acquireTokenLock( AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken, AUCTION_DURABILITY_LEADER_LOCK_TTL_MS, ); if (!leaderLockAcquired) return; await this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT); } catch (error) { this.logger.warn( `Auction durability tick failed: ${ error instanceof Error ? error.message : error }`, ); } finally { if (leaderLockAcquired) { await this.auctionRedisService.releaseTokenLock( AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken, ); } this.tickInProgress = false; } } private async clearDurabilityPayload(payloadKey: string): Promise { await this.auctionRedisService .raw() .sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]); await this.auctionRedisService.raw().sendCommand(['DEL', payloadKey]); } private async requeueDurabilityPayload( payloadKey: string, retryAtRedisTimestampMS: number, ): Promise { await this.auctionRedisService .raw() .sendCommand([ 'ZADD', AUCTION_DURABILITY_RETRY_QUEUE_KEY, String(retryAtRedisTimestampMS), payloadKey, ]); } private async recordDurabilityPayloadFailure( payloadKey: string, currentRedisTimestampMS: number, error: unknown, ): Promise<'dead-lettered' | 'requeued'> { const errorMessage = formatErrorMessageForRedis(error); const attemptReply = await this.auctionRedisService .raw() .sendCommand(['HINCRBY', payloadKey, 'attempts', '1']); const attempts = Number(attemptReply); await this.auctionRedisService .raw() .sendCommand([ 'HSET', payloadKey, 'lastError', errorMessage, 'lastFailedAtRedisTimestampMS', String(currentRedisTimestampMS), ]); if ( Number.isFinite(attempts) && attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS ) { await this.auctionRedisService .raw() .sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]); await this.auctionRedisService .raw() .sendCommand([ 'HSET', payloadKey, 'deadLetteredAtRedisTimestampMS', String(currentRedisTimestampMS), ]); await this.auctionRedisService .raw() .sendCommand([ 'ZADD', AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, String(currentRedisTimestampMS), payloadKey, ]); return 'dead-lettered'; } await this.requeueDurabilityPayload( payloadKey, currentRedisTimestampMS + calculateDurabilityRetryDelayMS(attempts), ); return 'requeued'; } private async expireCompletedAuctionRedisRuntime( payload: AuctionDurabilityPayload, ): Promise { const matchId = String(payload.matchId); const keys = buildAuctionRedisKeys(matchId); await this.auctionRedisService .raw() .sendCommand([ 'EVAL', [ 'local expectedAuctionLifecycleId = ARGV[2]', "local currentAuctionLifecycleId = redis.call('HGET', KEYS[1], 'auctionLifecycleId')", 'if currentAuctionLifecycleId ~= expectedAuctionLifecycleId then', ' return 0', 'end', "redis.call('ZREM', KEYS[2], ARGV[1])", 'for index = 1, #KEYS do', ' if index ~= 2 then', " redis.call('EXPIRE', KEYS[index], ARGV[3])", ' end', 'end', 'return 1', ].join('\n'), '11', keys.liveStateKey, keys.deadlineQueueKey, keys.structureStateKey, keys.appliedTransitionIdsKey, keys.bidHistoryKey, keys.bidRequestResultsKey, keys.bidRequestResultOrderKey, keys.actionRequestResultsKey, keys.actionRequestResultOrderKey, keys.rehydrateLockKey, keys.stateTransitionLockKey, matchId, payload.auctionLifecycleId, String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS), ]); } } export function buildAuctionHistoryUpsert(payload: AuctionDurabilityPayload) { return { updateOne: { filter: { auctionId: payload.auctionId, auctionStateTransitionId: payload.auctionStateTransitionId, }, update: { $setOnInsert: { action: payload.auctionActionType, actorUserId: payload.actorUserId, auctionId: payload.auctionId, auctionLifecycleId: payload.auctionLifecycleId, auctionStateTransitionId: payload.auctionStateTransitionId, historyMutation: payload.historyMutation, matchId: payload.matchId, matchSideEffectMutation: payload.matchSideEffectMutation, revision: payload.revision, sequence: payload.auctionHistorySequence, snapshotBefore: payload.snapshotBefore, takenAt: new Date(payload.committedAtRedisTimestampMS ?? 0), }, }, upsert: true, }, }; } export function parseAuctionHistoryMutation( mutation: unknown, ): ParsedAuctionHistoryMutation { if (!mutation || typeof mutation !== 'object') return { type: 'append' }; const record = mutation as Record; if (record.type === 'deleteAfterSequenceAndAppendUndo') { const targetHistorySequence = Number(record.targetHistorySequence); if (!Number.isFinite(targetHistorySequence) || targetHistorySequence < 0) { throw new Error('Invalid undo history mutation target sequence.'); } return { targetHistorySequence, type: 'deleteAfterSequenceAndAppendUndo', }; } if (record.type === 'reset') { const resetMode = record.resetMode === 'preserve-audit' ? 'preserve-audit' : 'clear-history'; return { resetMode, type: 'reset' }; } return { type: 'append' }; } function isDestructiveHistoryMutation( mutation: ParsedAuctionHistoryMutation, ): boolean { return ( mutation.type === 'deleteAfterSequenceAndAppendUndo' || (mutation.type === 'reset' && mutation.resetMode === 'clear-history') ); } async function readAuctionProjectionIdentityForMutation( AuctionModel: { findById?: (...args: never[]) => unknown; }, auctionId: unknown, session: mongoose.ClientSession | null | undefined, ): Promise { if (!AuctionModel.findById) return {}; const query = AuctionModel.findById(auctionId as never); const selected = query && typeof query === 'object' && 'select' in query ? (query as { select: (...args: never[]) => unknown }).select({ auctionLifecycleId: 1, revision: 1, } as never) : query; const scoped = selected && typeof selected === 'object' && 'session' in selected ? (selected as { session: (...args: never[]) => unknown }).session( session as never, ) : selected; const leaned = scoped && typeof scoped === 'object' && 'lean' in scoped ? (scoped as { lean: () => unknown }).lean() : scoped; const projection = leaned && typeof leaned === 'object' && 'exec' in leaned ? await (leaned as { exec: () => Promise }).exec() : await Promise.resolve(leaned); if (!projection || typeof projection !== 'object') return {}; const record = projection as Record; return { auctionLifecycleId: typeof record.auctionLifecycleId === 'string' ? record.auctionLifecycleId : undefined, revision: typeof record.revision === 'number' ? record.revision : Number(record.revision), }; } export function buildAuctionProjectionGuardedUpdate( payload: AuctionDurabilityPayload, ) { return { filter: { _id: payload.auctionId, auctionLifecycleId: payload.auctionLifecycleId, revision: { $lt: payload.revision }, }, update: { $set: payload.afterAuctionMongoProjection, }, }; } export function shouldApplyAuctionProjection( current: AuctionProjectionIdentity, payload: Pick, ): boolean { if (current.auctionLifecycleId !== payload.auctionLifecycleId) return false; return (current.revision ?? -1) < payload.revision; } export function shouldProcessAuctionDurabilityPayload( current: AuctionProjectionIdentity, payload: Pick, ): boolean { if (current.auctionLifecycleId !== payload.auctionLifecycleId) return false; return (current.revision ?? -1) <= payload.revision; } export function buildSideEffectApplicationUpsert( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ) { const identity = buildSideEffectApplicationIdentityFromPayload( payload, mutation, ); return { filter: identity, update: { $setOnInsert: { ...identity, auctionLifecycleId: payload.auctionLifecycleId, matchId: payload.matchId, attempts: 0, payload: mutation, payloadChecksum: hashStableJson(mutation), state: 'pending', }, }, }; } function parseMatchSideEffectMutation( value: Record | undefined, ): ParsedMatchSideEffectMutation | null { if (value?.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) { return { type: SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE }; } if (value?.type === RESET_MATCH_SIDE_EFFECT_TYPE) { return { type: RESET_MATCH_SIDE_EFFECT_TYPE }; } if (value?.type === UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE) { return { type: UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE }; } return null; } function buildSideEffectApplicationIdentityFromPayload( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ) { return { auctionId: payload.auctionId, auctionStateTransitionId: payload.auctionStateTransitionId, sideEffectType: mutation.type, }; } function hashStableJson(value: unknown): string { return createHash('sha256') .update( JSON.stringify( value, Object.keys(value as Record).sort(), ), ) .digest('hex'); } function hasMatchedDocument(result: unknown): boolean { if (!result || typeof result !== 'object') return false; const record = result as Record; const matchedCount = Number(record.matchedCount ?? record.modifiedCount ?? 0); return matchedCount > 0; } function buildMatchTeamsFromProjection( projection: Record, match: { participants: Array<{ preferredPositions?: Array<{ position?: string }>; userId: { toString(): string }; }>; }, ) { const auctionTeams = Array.isArray(projection.auctionTeams) ? (projection.auctionTeams as Array>) : []; const participantByUserId = new Map( match.participants.map((participant) => [ participant.userId.toString(), participant, ]), ); return auctionTeams.map((team) => { const teamIndex = Number(team.teamIndex); const captainUserId = team.captainUserId as MatchTeamMember['userId']; const captain = participantByUserId.get(captainUserId.toString()); const rawMembers = Array.isArray(team.members) ? (team.members as Array>) : []; const members: MatchTeamMember[] = [ { assignedPosition: (captain?.preferredPositions?.[0]?.position as | Position | undefined) ?? 'MID', cost: 0, userId: captainUserId, }, ...rawMembers .filter( (member) => member.userId?.toString() !== captainUserId.toString(), ) .map((member) => ({ assignedPosition: (member.assignedPosition ?? 'MID') as Position, cost: Number(member.memberAcquisitionBidPointAmount ?? 0), userId: member.userId as MatchTeamMember['userId'], })), ]; return { captainUserId, members, name: `${teamIndex}팀`, teamIndex, }; }); } function parseRedisHash(reply: unknown): Record { if (!reply) return {}; if (Array.isArray(reply)) { const result: Record = {}; for (let index = 0; index < reply.length; index += 2) { result[String(reply[index])] = String(reply[index + 1]); } return result; } if (typeof reply === 'object') { return Object.fromEntries( Object.entries(reply as Record).map(([key, value]) => [ key, String(value), ]), ); } return {}; } function formatErrorMessageForRedis(error: unknown): string { const message = error instanceof Error ? error.message : String(error); return message.slice(0, AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH); } function calculateDurabilityRetryDelayMS(attempts: number): number { if (!Number.isFinite(attempts) || attempts <= 1) { return AUCTION_DURABILITY_RETRY_DELAY_MS; } return Math.min( AUCTION_DURABILITY_MAX_RETRY_DELAY_MS, AUCTION_DURABILITY_RETRY_DELAY_MS * 2 ** (attempts - 1), ); } async function execMaybe( queryOrPromise: Promise | { exec(): Promise }, ) { if ( queryOrPromise && typeof (queryOrPromise as { exec?: unknown }).exec === 'function' ) { return (queryOrPromise as { exec(): Promise }).exec(); } return queryOrPromise; }