import { Injectable, Logger, OnModuleDestroy, OnModuleInit, Optional, ConflictException, } from '@nestjs/common'; import { createHash, randomUUID } from 'crypto'; import type mongoose from 'mongoose'; import { ConnectionService } from '../global/database/connection.service'; import type { AuctionMatchSideEffectStatus, MatchTeamMember, Position, } from '../global/database/schema'; import { TournamentCodeService } from '../tournament-code/tournament-code.service'; import { TournamentResultService } from '../tournament-result/tournament-result.service'; import { AuctionEventBusService } from './auction-event-bus.service'; import { AuctionService } from './auction.service'; import { AuctionBidArchiveService } from './auction-bid-archive.service'; import { AuctionLifecycleLock } from './auction-lifecycle-lock'; import { buildAuctionProjectionUnset } from './auction-projection'; import { AuctionSideEffectService, buildMatchSideEffectDelta, buildSideEffectLeaseAcquireUpdate, type MatchSideEffectDelta, } from './auction-side-effect.service'; import { AuctionRedisService } from './redis/auction-redis.service'; import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service'; import { AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, AUCTION_DURABILITY_RETRY_QUEUE_KEY, buildAuctionRedisKeys, } from './redis/auction-redis.keys'; // [AUCTION-DEBUG] 경매 "중복 멤버" 조사용 임시 계측. 조사 종료 후 제거 대상. import { recordNewRelicCustomEvent } from '../global/observability/new-relic'; const AUCTION_DURABILITY_TICK_INTERVAL_MS = 1000; const AUCTION_DURABILITY_BATCH_LIMIT = 20; const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000; const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000; const AUCTION_DURABILITY_MAX_ATTEMPTS = 10; const AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS = 120; const AUCTION_DURABILITY_MAX_PENDING_DURATION_MS = 10 * 60 * 1000; const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000; const AUCTION_DURABILITY_LEADER_LOCK_KEY = 'auction:auctionDurabilityScheduler:leader'; const AUCTION_DURABILITY_LEADER_LOCK_TTL_MS = 2500; const AUCTION_REDIS_COMPLETED_RETAIN_SECONDS = 7 * 24 * 60 * 60; const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000; const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete'; const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset'; const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects'; class SideEffectPendingError extends Error {} export interface AuctionDurabilityPayload { actorUserId?: unknown; afterAuctionMongoProjection: Record; auctionActionType: string; auctionHistorySequence: number; auctionId: unknown; auctionLifecycleId: string; auctionStateTransitionId: string; committedAtRedisTimestampMS?: number; historyMutation?: Record; matchId: unknown; matchSideEffectMutation: Record; shouldBroadcastState?: boolean; broadcastStateRevision?: number; revision: number; snapshotBefore: Record; } export interface AuctionProjectionIdentity { auctionLifecycleId?: string; revision?: number; matchSideEffectTransitionId?: string; } type ParsedAuctionHistoryMutation = | { type: 'append' } | { targetHistorySequence: number; type: 'deleteAfterSequenceAndAppendUndo' } | { resetMode: 'clear-history' | 'preserve-audit'; type: 'reset' }; type ParsedMatchSideEffectMutation = | { type: typeof SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE } | { type: typeof RESET_MATCH_SIDE_EFFECT_TYPE } | { type: typeof UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE }; interface CleanupSideEffectMatchDocument { competitionId?: unknown; gameSchedule?: unknown[]; participants: Array<{ excluded?: boolean }>; result?: unknown; resultPollState?: unknown; save(options?: { session: mongoose.ClientSession }): Promise; status: string; teamBuiltAt?: Date; teams?: unknown[]; tournamentCodeIds?: unknown[]; } @Injectable() export class AuctionDurabilityScheduler implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(AuctionDurabilityScheduler.name); private tickTimer?: NodeJS.Timeout; private tickInProgress = false; private deadLetterScanCursor = '0'; constructor( private readonly connectionService: ConnectionService, private readonly auctionRedisService: AuctionRedisService, @Optional() private readonly tournamentCodeService?: TournamentCodeService, @Optional() private readonly tournamentResultService?: TournamentResultService, @Optional() private readonly auctionSideEffectService?: AuctionSideEffectService, @Optional() private readonly auctionEventBusService?: AuctionEventBusService, @Optional() private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService, @Optional() private readonly auctionService?: AuctionService, @Optional() private readonly auctionBidArchiveService?: AuctionBidArchiveService, ) {} public onModuleInit(): void { this.tickTimer = setInterval( () => void this.safeTick(), AUCTION_DURABILITY_TICK_INTERVAL_MS, ); this.tickTimer.unref?.(); } public onModuleDestroy(): void { if (this.tickTimer) clearInterval(this.tickTimer); } public async readDueDurabilityPayloadKeys( currentRedisTimestampMS: number, batchLimit: number, ): Promise { const reply = await this.auctionRedisService .raw() .sendCommand([ 'ZRANGEBYSCORE', AUCTION_DURABILITY_RETRY_QUEUE_KEY, '-inf', String(currentRedisTimestampMS), 'LIMIT', '0', String(batchLimit), ]); return Array.isArray(reply) ? reply.map(String) : []; } public async processDueDurabilityPayloads(batchLimit: number): Promise<{ applied: number; failed: number; skipped: number; }> { const currentRedisTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const payloadKeys = await this.readDueDurabilityPayloadKeys( currentRedisTimestampMS, batchLimit, ); let applied = 0; let failed = 0; let skipped = 0; for (const payloadKey of payloadKeys) { try { const payload = await this.readDurabilityPayload(payloadKey); if (!payload) { skipped += 1; await this.clearDurabilityPayload(payloadKey); continue; } await this.applyDurabilityPayload(payload); applied += 1; // [AUCTION-DEBUG] 제거 대상. 영속 지연(lag): Redis 커밋 → Mongo 영속까지 걸린 시간 + 배치 백로그. // 사용자가 본 "되돌리기/이력이 2~3 뒤처짐"을 수치화한다(지연이 클수록 undo // 패널이 stale 시퀀스를 보여 엉뚱한 스냅샷 복원 가능성↑). recordNewRelicCustomEvent('AuctionServerDurability', { matchId: typeof payload.matchId === 'string' ? payload.matchId : '', auctionStateTransitionId: payload.auctionStateTransitionId, auctionActionType: payload.auctionActionType, revision: payload.revision, auctionHistorySequence: payload.auctionHistorySequence, persistLatencyMS: typeof payload.committedAtRedisTimestampMS === 'number' ? currentRedisTimestampMS - payload.committedAtRedisTimestampMS : -1, batchBacklog: payloadKeys.length, }); await this.clearDurabilityPayload(payloadKey); } catch (error) { if ( error instanceof SideEffectPendingError || error instanceof ConflictException ) { const disposition = await this.deferDurabilityPayload( payloadKey, currentRedisTimestampMS, error, ); if (disposition === 'dead-lettered') { failed += 1; this.logger.warn( `Auction durability payload exceeded its pending limit: ${payloadKey}`, ); } else skipped += 1; continue; } failed += 1; const failureDisposition = await this.recordDurabilityPayloadFailure( payloadKey, currentRedisTimestampMS, error, ); this.logger.warn( `Auction durability payload failed for ${payloadKey} (${failureDisposition}): ${ error instanceof Error ? error.message : error }`, ); } } return { applied, failed, skipped }; } public async readDurabilityPayload( payloadKey: string, ): Promise { const reply = await this.auctionRedisService .raw() .sendCommand(['HGETALL', payloadKey]); const hash = parseRedisHash(reply); const payloadJson = hash.payloadJson; if (!payloadJson) return null; const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload; const committedDeadline = Number(hash.committedBidRoundDeadlineTimestampMS); if (Number.isFinite(committedDeadline) && committedDeadline > 0) { payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS = committedDeadline; payload.afterAuctionMongoProjection.currentDeadline = new Date( committedDeadline, ).toISOString(); } const committedAtRedisTimestampMS = Number( hash.committedAtRedisTimestampMS, ); if (Number.isFinite(committedAtRedisTimestampMS)) { payload.committedAtRedisTimestampMS = committedAtRedisTimestampMS; } payload.shouldBroadcastState = hash.shouldBroadcastState === '1'; const broadcastStateRevision = Number(hash.broadcastStateRevision); if (Number.isFinite(broadcastStateRevision)) { payload.broadcastStateRevision = broadcastStateRevision; } return payload; } public async applyDurabilityPayload( payload: AuctionDurabilityPayload, ): Promise { const AuctionHistoryModel = await this.connectionService.getModel('auctionHistory'); const AuctionModel = await this.connectionService.getModel('auction'); const AuctionSideEffectApplicationModel = await this.connectionService.getModel('auctionSideEffectApplication'); const session = await this.connectionService.createSession(); const sideEffectMutation = parseMatchSideEffectMutation( payload.matchSideEffectMutation, ); let shouldUpdateProjection = true; let shouldApplySideEffects = true; let shouldApplyHistoryMutation = true; try { session?.startTransaction(); if (typeof AuctionModel.findById === 'function') { const currentProjection = await readAuctionProjectionIdentityForMutation( AuctionModel, payload.auctionId, session, ); if ( !shouldProcessAuctionDurabilityPayload(currentProjection, payload) ) { if ( currentProjection.auctionLifecycleId !== payload.auctionLifecycleId ) { if (session) await session.commitTransaction(); return; } shouldApplyHistoryMutation = !(await this.isAppendHistoryInvalidatedByLaterDestructiveMutation({ AuctionHistoryModel, payload, session, })); // History operations have their own sequence bounds. A newer ordinary // projection must not discard an earlier undo's history cleanup. shouldUpdateProjection = false; shouldApplySideEffects = Boolean(this.auctionRedisStateReaderService) || currentProjection.matchSideEffectTransitionId === payload.auctionStateTransitionId; } } if (shouldApplyHistoryMutation) await this.applyAuctionHistoryMutation({ AuctionHistoryModel, AuctionModel, payload, session, }); const projectionOperation = buildAuctionProjectionGuardedUpdate(payload); if (shouldUpdateProjection) await execMaybe( AuctionModel.updateOne( projectionOperation.filter as never, projectionOperation.update as never, { session } as never, ), ); if (sideEffectMutation && shouldApplySideEffects) { const sideEffectOperation = buildSideEffectApplicationUpsert( payload, sideEffectMutation, ); await execMaybe( AuctionSideEffectApplicationModel.updateOne( sideEffectOperation.filter as never, sideEffectOperation.update as never, { session, upsert: true, } as never, ), ); } if (session) await session.commitTransaction(); } catch (error) { if (session) await session.abortTransaction(); throw error; } finally { if (session) await session.endSession(); } await this.broadcastLatestRedisAuctionStateIfRequested(payload); if (sideEffectMutation && shouldApplySideEffects) { await this.applyMatchSideEffect(payload, sideEffectMutation); } } /** * 노드 크래시 복구용 백업 broadcast(1차 전달 아님). * * 정상 경로에서는 요청을 처리한 게이트웨이(수동 전환) 또는 * `AuctionDeadlineScheduler`(자동 만료)가 commit 직후 동기로 `state`를 방송한다. * 이 메서드는 "방송을 하기로 한 인스턴스가 broadcast 전에 죽은" 경우를 위해 * outbox를 처리하면서 최신 state를 한 번 더 방송하는 안전망이다. * 이 호출은 위 try 블록의 Mongo 트랜잭션이 성공해야(early return/throw 없이) * 도달하므로 1차 전달로 의존해서는 안 된다. 중복 방송은 클라 revision 가드로 무해. */ private async broadcastLatestRedisAuctionStateIfRequested( payload: AuctionDurabilityPayload, ): Promise { if (!payload.shouldBroadcastState) return; if (!this.auctionRedisStateReaderService || !this.auctionEventBusService) { return; } const matchId = String(payload.matchId); const state = await this.auctionRedisStateReaderService.readAuctionStateView(matchId); if (!state) return; const stateRevision = Number(state.revision); if ( Number.isFinite(payload.broadcastStateRevision) && Number.isFinite(stateRevision) && stateRevision < Number(payload.broadcastStateRevision) ) { return; } this.auctionEventBusService.broadcastState(matchId, state); } private async applyMatchSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { return new AuctionLifecycleLock(this.auctionRedisService).run( String(payload.matchId), async () => { if (this.auctionRedisStateReaderService) { const state = await this.auctionRedisStateReaderService.readAuctionStateView( String(payload.matchId), ); if (!state) throw new SideEffectPendingError( 'Auction realtime state is unavailable.', ); if ( state.auctionLifecycleId !== payload.auctionLifecycleId || (state.matchSideEffectTransitionId && state.matchSideEffectTransitionId !== payload.auctionStateTransitionId) || (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE && state.phase !== 'complete') ) return; // An ordinary later projection may have reached Mongo before this // side-effect-bearing transition. Install its pending generation only // while Redis still names it and Mongo has not advanced beyond it. const AuctionModel = await this.connectionService.getModel('auction'); const pendingRevision = Number( payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0, ); await execMaybe( AuctionModel.updateOne( { _id: payload.auctionId, auctionLifecycleId: payload.auctionLifecycleId, $or: [ { matchSideEffectRevision: { $lt: pendingRevision } }, { matchSideEffectRevision: { $exists: false } }, ], } as never, { $set: { matchSideEffectTransitionId: payload.auctionStateTransitionId, matchSideEffectRevision: pendingRevision, matchSideEffectStatus: 'pending', }, $unset: { matchSideEffectError: '' }, } as never, ), ); } await this.applyMatchSideEffectWhileLocked(payload, mutation); }, ); } private async applyMatchSideEffectWhileLocked( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) { await this.applySyncMatchTeamsSideEffect(payload, mutation); return; } if (mutation.type === RESET_MATCH_SIDE_EFFECT_TYPE) { await this.applyResetMatchSideEffect(payload, mutation); return; } if (mutation.type === UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE) { await this.applyUndoCompletedMatchSideEffect(payload, mutation); } } private async applySyncMatchTeamsSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { if (!this.tournamentCodeService || !this.tournamentResultService) { throw new Error('Auction side-effect services are not available.'); } const AuctionModel = await this.connectionService.getModel('auction'); const AuctionSideEffectApplicationModel = await this.connectionService.getModel('auctionSideEffectApplication'); const MatchModel = await this.connectionService.getModel('match'); const currentRedisTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const ownerToken = randomUUID(); const identity = buildSideEffectApplicationIdentityFromPayload( payload, mutation, ); const leaseResult = await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, $or: [ { state: 'pending' }, { state: 'failed' }, { state: 'processing', processingLeaseExpiresAtTimestampMS: { $lte: currentRedisTimestampMS, }, }, ], } as never, buildSideEffectLeaseAcquireUpdate({ currentRedisTimestampMS, leaseDurationMS: AUCTION_SIDE_EFFECT_LEASE_DURATION_MS, ownerToken, }) as never, ), ); if (!hasMatchedDocument(leaseResult)) { const existing = await AuctionSideEffectApplicationModel.findOne( identity as never, ) .lean() .exec(); if (existing?.state === 'completed') { await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: currentRedisTimestampMS, payload, status: 'completed', }); await this.expireCompletedAuctionRedisRuntime(payload); return; } throw new SideEffectPendingError( 'Auction side effect is still owned by another worker.', ); } const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus( { AuctionModel, currentServerTimestampMS: currentRedisTimestampMS, expectedMatchSideEffectRevision: Number( payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0, ), payload, status: 'processing', }, ); if (!processingDelta) { throw new SideEffectPendingError( 'Auction side-effect generation is not ready.', ); } const expectedMatchSideEffectRevision = processingDelta.matchSideEffectRevision; try { const match = (await execMaybe( MatchModel.findById(payload.matchId) as never, )) as { competitionId?: unknown; participants: Array<{ preferredPositions?: Array<{ position?: Position }>; userId: { toString(): string }; }>; save(): Promise; status: string; teamBuiltAt?: Date; teams: ReturnType; } | null; if (!match) throw new Error(`Match not found: ${String(payload.matchId)}`); if (match.competitionId) { await this.tournamentResultService.scheduleInitialPoll(match as never); } else { const matchTeams = buildMatchTeamsFromProjection( payload.afterAuctionMongoProjection, match, ); // [AUCTION-DEBUG] 제거 대상. 최종 팀에 같은 유저가 2번 들어갔는지 무조건 검사 // (출처가 markSold든 undo 복원이든 projection이든 여기서 결정적으로 잡힌다). const allMemberUserIds = matchTeams.flatMap((team) => team.members.map((member) => member.userId.toString()), ); const distinctMemberCount = new Set(allMemberUserIds).size; recordNewRelicCustomEvent('AuctionServerTeamBuild', { matchId: typeof payload.matchId === 'string' ? payload.matchId : '', auctionStateTransitionId: payload.auctionStateTransitionId, teamCount: matchTeams.length, totalMemberCount: allMemberUserIds.length, distinctMemberCount, hasDuplicateMember: allMemberUserIds.length !== distinctMemberCount, duplicateUserId: allMemberUserIds.find( (id, idx) => allMemberUserIds.indexOf(id) !== idx, ) ?? null, }); if (matchTeams.length === 0) { throw new Error( `Auction complete side effect produced no match teams for match ${String(payload.matchId)}.`, ); } const finalizedAt = match.teamBuiltAt ?? new Date(); match.teams = matchTeams; match.status = 'team-built'; match.teamBuiltAt = undefined; await match.save(); const preparedMatch = (await this.tournamentCodeService.reserveForMatch(match as never)) ?? match; preparedMatch.teamBuiltAt = finalizedAt; await preparedMatch.save(); await this.tournamentResultService.scheduleInitialPoll( preparedMatch as never, ); } const completedAtTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, processingOwnerToken: ownerToken, state: 'processing', } as never, { $set: { completedAtTimestampMS, state: 'completed', }, $unset: { lastError: '', processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: completedAtTimestampMS, errorMessage: undefined, expectedMatchSideEffectRevision, payload, status: 'completed', }); await this.expireCompletedAuctionRedisRuntime(payload); } catch (error) { const failureTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, processingOwnerToken: ownerToken, state: 'processing', } as never, { $set: { lastError: error instanceof Error ? error.message : String(error), state: 'failed', }, $unset: { processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: failureTimestampMS, errorMessage: error instanceof Error ? error.message : String(error), expectedMatchSideEffectRevision, payload, status: 'failed', }); throw error; } } private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: { AuctionHistoryModel: { findOne?: (...args: never[]) => unknown; }; payload: AuctionDurabilityPayload; session: mongoose.ClientSession | null | undefined; }): Promise { if (!params.AuctionHistoryModel.findOne) return true; // 늦게 도착한 append는 history gap을 메울 수 있지만, // 이후 undo/reset이 의도적으로 삭제한 sequence를 되살리면 안 된다. const query = params.AuctionHistoryModel.findOne({ auctionId: params.payload.auctionId, sequence: { $gt: params.payload.auctionHistorySequence }, $or: [ { 'historyMutation.targetHistorySequence': { $lt: params.payload.auctionHistorySequence, }, 'historyMutation.type': 'deleteAfterSequenceAndAppendUndo', }, { 'historyMutation.resetMode': { $ne: 'preserve-audit' }, 'historyMutation.type': 'reset', }, ], } as never); const scoped = query && typeof query === 'object' && 'session' in query ? (query as { session: (...args: never[]) => unknown }).session( params.session as never, ) : query; const leaned = scoped && typeof scoped === 'object' && 'lean' in scoped ? (scoped as { lean: () => unknown }).lean() : scoped; const result = leaned && typeof leaned === 'object' && 'exec' in leaned ? await (leaned as { exec: () => Promise }).exec() : await Promise.resolve(leaned); return Boolean(result); } private async applyResetMatchSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { await this.applyMatchCleanupSideEffect({ mutation, payload, updateMatch: (match) => { match.status = 'recruiting'; match.teams = []; match.tournamentCodeIds = []; match.gameSchedule = []; match.participants.forEach((participant) => { participant.excluded = false; }); match.teamBuiltAt = undefined; match.result = undefined; match.resultPollState = undefined; }, }); } private async applyUndoCompletedMatchSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ): Promise { await this.applyMatchCleanupSideEffect({ mutation, payload, updateMatch: (match) => { match.status = 'team-built'; match.teams = []; match.teamBuiltAt = undefined; match.tournamentCodeIds = []; match.gameSchedule = []; match.result = undefined; match.resultPollState = undefined; }, }); } private async applyMatchCleanupSideEffect(params: { mutation: ParsedMatchSideEffectMutation; payload: AuctionDurabilityPayload; updateMatch: (match: CleanupSideEffectMatchDocument) => void; }): Promise { if (!this.tournamentCodeService || !this.tournamentResultService) { throw new Error('Auction side-effect services are not available.'); } const AuctionModel = await this.connectionService.getModel('auction'); const AuctionSideEffectApplicationModel = await this.connectionService.getModel('auctionSideEffectApplication'); const MatchModel = await this.connectionService.getModel('match'); const currentRedisTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const ownerToken = randomUUID(); const identity = buildSideEffectApplicationIdentityFromPayload( params.payload, params.mutation, ); const leaseResult = await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, $or: [ { state: 'pending' }, { state: 'failed' }, { state: 'processing', processingLeaseExpiresAtTimestampMS: { $lte: currentRedisTimestampMS, }, }, ], } as never, buildSideEffectLeaseAcquireUpdate({ currentRedisTimestampMS, leaseDurationMS: AUCTION_SIDE_EFFECT_LEASE_DURATION_MS, ownerToken, }) as never, ), ); if (!hasMatchedDocument(leaseResult)) { const existing = await AuctionSideEffectApplicationModel.findOne( identity as never, ) .lean() .exec(); if (existing?.state === 'completed') { await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: currentRedisTimestampMS, payload: params.payload, status: 'completed', }); return; } throw new SideEffectPendingError( 'Auction side effect is still owned by another worker.', ); } const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus( { AuctionModel, currentServerTimestampMS: currentRedisTimestampMS, expectedMatchSideEffectRevision: Number( params.payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0, ), payload: params.payload, status: 'processing', }, ); if (!processingDelta) { throw new SideEffectPendingError( 'Auction side-effect generation is not ready.', ); } const expectedMatchSideEffectRevision = processingDelta.matchSideEffectRevision; try { const session = await this.connectionService.createSession(); if (!session) throw new Error('Failed to start Match cleanup transaction.'); try { session.startTransaction(); const match = (await MatchModel.findById( params.payload.matchId, ).session(session)) as CleanupSideEffectMatchDocument | null; if (!match) throw new Error(`Match not found: ${String(params.payload.matchId)}`); if (match.competitionId) throw new Error( 'Cannot reset or undo Match teams after the immutable Competition has been created.', ); await this.tournamentCodeService.releaseFromMatch( match as never, session, ); await this.tournamentResultService.cancelPolling( match as never, session, ); params.updateMatch(match); await match.save({ session }); await session.commitTransaction(); } catch (error) { await session.abortTransaction(); throw error; } finally { await session.endSession(); } const completedAtTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, processingOwnerToken: ownerToken, state: 'processing', } as never, { $set: { completedAtTimestampMS, state: 'completed', }, $unset: { lastError: '', processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: completedAtTimestampMS, errorMessage: undefined, expectedMatchSideEffectRevision, payload: params.payload, status: 'completed', }); } catch (error) { const failureTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); const errorMessage = error instanceof Error ? error.message : String(error); await execMaybe( AuctionSideEffectApplicationModel.updateOne( { ...identity, processingOwnerToken: ownerToken, state: 'processing', } as never, { $set: { lastError: errorMessage, state: 'failed', }, $unset: { processingLeaseExpiresAtTimestampMS: '', processingOwnerToken: '', processingStartedAtTimestampMS: '', }, } as never, ), ); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel, currentServerTimestampMS: failureTimestampMS, errorMessage, expectedMatchSideEffectRevision, payload: params.payload, status: 'failed', }); throw error; } } private async applyAuctionHistoryMutation(params: { AuctionHistoryModel: { deleteMany?: (...args: never[]) => unknown; updateOne: (...args: never[]) => unknown; }; AuctionModel: { findById?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; session?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; select?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; session?: (...args: never[]) => unknown; }; }; select?: (...args: never[]) => { lean?: () => { exec?: () => Promise }; session?: (...args: never[]) => unknown; }; }; }; payload: AuctionDurabilityPayload; session: mongoose.ClientSession | null | undefined; }): Promise { const mutation = parseAuctionHistoryMutation( params.payload.historyMutation, ); if (isDestructiveHistoryMutation(mutation)) { if (mutation.type === 'deleteAfterSequenceAndAppendUndo') { if (!params.AuctionHistoryModel.deleteMany) { throw new Error('Auction history deleteMany is not available.'); } await execMaybe( params.AuctionHistoryModel.deleteMany( { auctionId: params.payload.auctionId, sequence: { $gt: mutation.targetHistorySequence, $lt: params.payload.auctionHistorySequence, }, } as never, { session: params.session } as never, ) as Promise | { exec(): Promise }, ); } if (mutation.type === 'reset' && mutation.resetMode === 'clear-history') { if (!params.AuctionHistoryModel.deleteMany) { throw new Error('Auction history deleteMany is not available.'); } await execMaybe( params.AuctionHistoryModel.deleteMany( { auctionId: params.payload.auctionId, sequence: { $lt: params.payload.auctionHistorySequence }, } as never, { session: params.session } as never, ) as Promise | { exec(): Promise }, ); } } const historyOperation = buildAuctionHistoryUpsert( params.payload, ).updateOne; await execMaybe( params.AuctionHistoryModel.updateOne( historyOperation.filter as never, historyOperation.update as never, { session: params.session, upsert: true, } as never, ) as Promise | { exec(): Promise }, ); } private async persistAndBroadcastMatchSideEffectStatus(params: { AuctionModel: { findOneAndUpdate( filter: never, update: never, options: never, ): Promise | { exec(): Promise }; }; currentServerTimestampMS: number; errorMessage?: string; expectedMatchSideEffectRevision?: number; payload: AuctionDurabilityPayload; status: AuctionMatchSideEffectStatus; onlyUnfinished?: boolean; }): Promise { const updatedAuction = (await execMaybe( params.AuctionModel.findOneAndUpdate( { _id: params.payload.auctionId, auctionLifecycleId: params.payload.auctionLifecycleId, ...(params.onlyUnfinished ? { matchSideEffectStatus: { $ne: 'completed' } } : {}), ...(params.status === 'processing' ? { $or: [ { matchSideEffectTransitionId: params.payload.auctionStateTransitionId, }, { matchSideEffectTransitionId: { $exists: false }, matchSideEffectRevision: params.expectedMatchSideEffectRevision, }, ], } : { matchSideEffectTransitionId: params.payload.auctionStateTransitionId, ...(params.expectedMatchSideEffectRevision === undefined ? {} : { matchSideEffectRevision: params.expectedMatchSideEffectRevision, }), }), } as never, { $inc: { matchSideEffectRevision: 1 }, $set: { ...(params.errorMessage ? { matchSideEffectError: params.errorMessage } : {}), matchSideEffectStatus: params.status, matchSideEffectTransitionId: params.payload.auctionStateTransitionId, }, ...(params.errorMessage ? {} : { $unset: { matchSideEffectError: '' } }), } as never, { projection: { auctionLifecycleId: 1, matchSideEffectRevision: 1, matchSideEffectStatus: 1, }, returnDocument: 'after', } as never, ), )) as { auctionLifecycleId?: string; matchSideEffectRevision?: number; matchSideEffectStatus?: AuctionMatchSideEffectStatus; } | null; if (!updatedAuction?.auctionLifecycleId) return null; const delta = buildMatchSideEffectDelta({ auctionLifecycleId: updatedAuction.auctionLifecycleId, auctionStateTransitionId: params.payload.auctionStateTransitionId, currentServerTimestampMS: params.currentServerTimestampMS, matchSideEffectRevision: Number( updatedAuction.matchSideEffectRevision ?? 0, ), matchSideEffectStatus: updatedAuction.matchSideEffectStatus ?? params.status, }); const mirrored = await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({ auctionLifecycleId: delta.auctionLifecycleId, auctionStateTransitionId: params.payload.auctionStateTransitionId, matchId: String(params.payload.matchId), matchSideEffectRevision: delta.matchSideEffectRevision, matchSideEffectStatus: delta.matchSideEffectStatus, }); if (mirrored !== false) { this.auctionEventBusService?.broadcastMatchSideEffect( String(params.payload.matchId), delta, ); } return delta; } public async reconcileDeadLetterStatuses(limit = 20): Promise { const client = this.auctionRedisService.raw(); const now = await this.auctionRedisService.getCurrentRedisTimestampMS(); // Also reconcile entries dead-lettered before status notifications existed. const scanned = await client.sendCommand([ 'ZSCAN', AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, this.deadLetterScanCursor, 'COUNT', String(limit), ]); if (Array.isArray(scanned) && Array.isArray(scanned[1])) { this.deadLetterScanCursor = String(scanned[0]); for (let index = 0; index < scanned[1].length; index += 2) { const key = String(scanned[1][index]); if ( !(await client.sendCommand([ 'HGET', key, 'deadLetterStatusReconciledAtTimestampMS', ])) ) { await client.sendCommand([ 'ZADD', AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, 'NX', String(now), key, ]); } } } const reply = await client.sendCommand([ 'ZRANGEBYSCORE', AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, '-inf', String(now), 'LIMIT', '0', String(limit), ]); if (!Array.isArray(reply)) return; for (const key of reply.map(String)) { try { const payload = await this.readDurabilityPayload(key); if ( payload && parseMatchSideEffectMutation(payload.matchSideEffectMutation) ) { const state = await this.auctionRedisStateReaderService?.readAuctionStateView( String(payload.matchId), ); // A terminal notification for a previous generation must not change // a new completion/undo, even when Mongo is behind Redis. const superseded = state && (state.auctionLifecycleId !== payload.auctionLifecycleId || state.matchSideEffectStatus === 'completed' || (state.matchSideEffectTransitionId && state.matchSideEffectTransitionId !== payload.auctionStateTransitionId)); if (!superseded) { const AuctionModel = await this.connectionService.getModel('auction'); if ( state?.matchSideEffectTransitionId === payload.auctionStateTransitionId ) { const revision = Number( payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0, ); await AuctionModel.updateOne( { _id: payload.auctionId, auctionLifecycleId: payload.auctionLifecycleId, $or: [ { matchSideEffectRevision: { $lt: revision } }, { matchSideEffectRevision: { $exists: false } }, ], } as never, { $set: { matchSideEffectTransitionId: payload.auctionStateTransitionId, matchSideEffectRevision: revision, matchSideEffectStatus: 'pending', }, }, ); // Heal a same-generation revision mismatch before sending the // terminal status; otherwise Redis could ignore it as older. await AuctionModel.updateOne( { _id: payload.auctionId, auctionLifecycleId: payload.auctionLifecycleId, matchSideEffectTransitionId: payload.auctionStateTransitionId, matchSideEffectStatus: { $ne: 'completed' }, } as never, { $max: { matchSideEffectRevision: Number( state.matchSideEffectRevision ?? revision, ), }, }, ); } const error = await client.sendCommand(['HGET', key, 'lastError']); await this.persistAndBroadcastMatchSideEffectStatus({ AuctionModel: AuctionModel as never, payload, currentServerTimestampMS: now, status: 'failed', onlyUnfinished: true, errorMessage: typeof error === 'string' ? error : 'Auction side effect exhausted its retry limit.', }); } } await client.sendCommand([ 'EVAL', "if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 1 then redis.call('HSET', KEYS[1], 'deadLetterStatusReconciledAtTimestampMS', ARGV[1]) end redis.call('ZREM', KEYS[2], KEYS[1]); return 1", '2', key, AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, String(now), ]); } catch (error) { // Mongo or Redis mirroring may be unavailable at dead-letter time. Keep // the notification durable until the failure can be shown to viewers. await client.sendCommand([ 'ZADD', AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, String(now + 5000), key, ]); this.logger.warn( `Auction terminal status will retry for ${key}: ${String(error)}`, ); } } } private async safeTick(): Promise { if (this.tickInProgress) return; this.tickInProgress = true; const leaderLockToken = randomUUID(); let leaderLockAcquired = false; try { leaderLockAcquired = await this.auctionRedisService.acquireTokenLock( AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken, AUCTION_DURABILITY_LEADER_LOCK_TTL_MS, ); if (!leaderLockAcquired) return; // A failed maintenance task must not starve the other durable queues. for (const work of [ () => this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT), () => this.reconcileDeadLetterStatuses(AUCTION_DURABILITY_BATCH_LIMIT), () => this.auctionBidArchiveService?.persistPendingBids(), () => this.auctionService?.retryPendingCancellations( AUCTION_DURABILITY_BATCH_LIMIT, ), ]) { try { await work(); } catch (error) { this.logger.warn(`Auction maintenance task failed: ${String(error)}`); } } } catch (error) { this.logger.warn( `Auction durability tick failed: ${ error instanceof Error ? error.message : error }`, ); } finally { if (leaderLockAcquired) { await this.auctionRedisService .releaseTokenLock(AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken) .catch((error: unknown) => { this.logger.warn( `Auction durability lock release failed: ${String(error)}`, ); }); } this.tickInProgress = false; } } private async clearDurabilityPayload(payloadKey: string): Promise { await this.auctionRedisService .raw() .sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]); await this.auctionRedisService.raw().sendCommand(['DEL', payloadKey]); } private async requeueDurabilityPayload( payloadKey: string, retryAtRedisTimestampMS: number, ): Promise { await this.auctionRedisService .raw() .sendCommand([ 'ZADD', AUCTION_DURABILITY_RETRY_QUEUE_KEY, String(retryAtRedisTimestampMS), payloadKey, ]); } private async deferDurabilityPayload( payloadKey: string, timestampMS: number, error: unknown, ): Promise<'gone' | 'requeued' | 'dead-lettered'> { // Keep coordination waits separate from failed work, but bound them across // restarts too. Updating the payload and queue atomically avoids orphan jobs. const result = await this.auctionRedisService .raw() .sendCommand([ 'EVAL', [ "if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end", "local count = redis.call('HINCRBY', KEYS[1], 'pendingAttempts', 1)", "redis.call('HSETNX', KEYS[1], 'pendingSinceTimestampMS', ARGV[1])", "local since = tonumber(redis.call('HGET', KEYS[1], 'pendingSinceTimestampMS'))", "redis.call('HSET', KEYS[1], 'lastPendingReason', ARGV[5], 'lastPendingAtTimestampMS', ARGV[1])", 'if count >= tonumber(ARGV[3]) or tonumber(ARGV[1]) - since >= tonumber(ARGV[4]) then', " redis.call('ZREM', KEYS[2], KEYS[1])", " redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1], 'lastError', ARGV[5])", " redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1])", " redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1])", ' return 2', 'end', "redis.call('ZADD', KEYS[2], tonumber(ARGV[1]) + tonumber(ARGV[2]), KEYS[1])", 'return 1', ].join('\n'), '4', payloadKey, AUCTION_DURABILITY_RETRY_QUEUE_KEY, AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, String(timestampMS), String(AUCTION_DURABILITY_RETRY_DELAY_MS), String(AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS), String(AUCTION_DURABILITY_MAX_PENDING_DURATION_MS), formatErrorMessageForRedis(error), ]); return Number(result) === 2 ? 'dead-lettered' : Number(result) === 1 ? 'requeued' : 'gone'; } private async recordDurabilityPayloadFailure( payloadKey: string, currentRedisTimestampMS: number, error: unknown, ): Promise<'dead-lettered' | 'requeued'> { const errorMessage = formatErrorMessageForRedis(error); const attemptReply = await this.auctionRedisService .raw() .sendCommand(['HINCRBY', payloadKey, 'attempts', '1']); const attempts = Number(attemptReply); await this.auctionRedisService .raw() .sendCommand([ 'HSET', payloadKey, 'lastError', errorMessage, 'lastFailedAtRedisTimestampMS', String(currentRedisTimestampMS), ]); if ( Number.isFinite(attempts) && attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS ) { await this.auctionRedisService .raw() .sendCommand([ 'EVAL', "if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1]); redis.call('ZREM', KEYS[2], KEYS[1]); redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1]); redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1]); return 1", '4', payloadKey, AUCTION_DURABILITY_RETRY_QUEUE_KEY, AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, String(currentRedisTimestampMS), ]); return 'dead-lettered'; } await this.requeueDurabilityPayload( payloadKey, currentRedisTimestampMS + calculateDurabilityRetryDelayMS(attempts), ); return 'requeued'; } private async expireCompletedAuctionRedisRuntime( payload: AuctionDurabilityPayload, ): Promise { const matchId = String(payload.matchId); const keys = buildAuctionRedisKeys(matchId); await this.auctionRedisService .raw() .sendCommand([ 'EVAL', [ 'local expectedAuctionLifecycleId = ARGV[2]', "local currentAuctionLifecycleId = redis.call('HGET', KEYS[1], 'auctionLifecycleId')", 'if currentAuctionLifecycleId ~= expectedAuctionLifecycleId then', ' return 0', 'end', "redis.call('ZREM', KEYS[2], ARGV[1])", 'for index = 1, #KEYS do', ' if index ~= 2 then', " redis.call('EXPIRE', KEYS[index], ARGV[3])", ' end', 'end', 'return 1', ].join('\n'), '12', keys.liveStateKey, keys.deadlineQueueKey, keys.structureStateKey, keys.appliedTransitionIdsKey, keys.bidHistoryKey, keys.bidRequestResultsKey, keys.bidRequestResultOrderKey, keys.actionRequestResultsKey, keys.actionRequestResultOrderKey, keys.rehydrateLockKey, keys.stateTransitionLockKey, keys.rejectedBidRequestResultOrderKey, matchId, payload.auctionLifecycleId, String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS), ]); } } export function buildAuctionHistoryUpsert(payload: AuctionDurabilityPayload) { return { updateOne: { filter: { auctionId: payload.auctionId, auctionStateTransitionId: payload.auctionStateTransitionId, }, update: { $setOnInsert: { action: payload.auctionActionType, actorUserId: payload.actorUserId, auctionId: payload.auctionId, auctionLifecycleId: payload.auctionLifecycleId, auctionStateTransitionId: payload.auctionStateTransitionId, historyMutation: payload.historyMutation, matchId: payload.matchId, matchSideEffectMutation: payload.matchSideEffectMutation, revision: payload.revision, sequence: payload.auctionHistorySequence, snapshotBefore: payload.snapshotBefore, takenAt: new Date(payload.committedAtRedisTimestampMS ?? 0), }, }, upsert: true, }, }; } export function parseAuctionHistoryMutation( mutation: unknown, ): ParsedAuctionHistoryMutation { if (!mutation || typeof mutation !== 'object') return { type: 'append' }; const record = mutation as Record; if (record.type === 'deleteAfterSequenceAndAppendUndo') { const targetHistorySequence = Number(record.targetHistorySequence); if (!Number.isFinite(targetHistorySequence) || targetHistorySequence < 0) { throw new Error('Invalid undo history mutation target sequence.'); } return { targetHistorySequence, type: 'deleteAfterSequenceAndAppendUndo', }; } if (record.type === 'reset') { const resetMode = record.resetMode === 'preserve-audit' ? 'preserve-audit' : 'clear-history'; return { resetMode, type: 'reset' }; } return { type: 'append' }; } function isDestructiveHistoryMutation( mutation: ParsedAuctionHistoryMutation, ): boolean { return ( mutation.type === 'deleteAfterSequenceAndAppendUndo' || (mutation.type === 'reset' && mutation.resetMode === 'clear-history') ); } async function readAuctionProjectionIdentityForMutation( AuctionModel: { findById?: (...args: never[]) => unknown; }, auctionId: unknown, session: mongoose.ClientSession | null | undefined, ): Promise { if (!AuctionModel.findById) return {}; const query = AuctionModel.findById(auctionId as never); const selected = query && typeof query === 'object' && 'select' in query ? (query as { select: (...args: never[]) => unknown }).select({ auctionLifecycleId: 1, revision: 1, matchSideEffectTransitionId: 1, } as never) : query; const scoped = selected && typeof selected === 'object' && 'session' in selected ? (selected as { session: (...args: never[]) => unknown }).session( session as never, ) : selected; const leaned = scoped && typeof scoped === 'object' && 'lean' in scoped ? (scoped as { lean: () => unknown }).lean() : scoped; const projection = leaned && typeof leaned === 'object' && 'exec' in leaned ? await (leaned as { exec: () => Promise }).exec() : await Promise.resolve(leaned); if (!projection || typeof projection !== 'object') return {}; const record = projection as Record; return { matchSideEffectTransitionId: typeof record.matchSideEffectTransitionId === 'string' ? record.matchSideEffectTransitionId : undefined, auctionLifecycleId: typeof record.auctionLifecycleId === 'string' ? record.auctionLifecycleId : undefined, revision: typeof record.revision === 'number' ? record.revision : Number(record.revision), }; } export function buildAuctionProjectionGuardedUpdate( payload: AuctionDurabilityPayload, ) { return { filter: { _id: payload.auctionId, auctionLifecycleId: payload.auctionLifecycleId, revision: { $lt: payload.revision }, }, update: { $set: payload.afterAuctionMongoProjection, $unset: buildAuctionProjectionUnset(payload.afterAuctionMongoProjection), }, }; } export function shouldApplyAuctionProjection( current: AuctionProjectionIdentity, payload: Pick, ): boolean { if (current.auctionLifecycleId !== payload.auctionLifecycleId) return false; return (current.revision ?? -1) < payload.revision; } export function shouldProcessAuctionDurabilityPayload( current: AuctionProjectionIdentity, payload: Pick, ): boolean { if (current.auctionLifecycleId !== payload.auctionLifecycleId) return false; return (current.revision ?? -1) <= payload.revision; } export function buildSideEffectApplicationUpsert( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ) { const identity = buildSideEffectApplicationIdentityFromPayload( payload, mutation, ); return { filter: identity, update: { $setOnInsert: { ...identity, auctionLifecycleId: payload.auctionLifecycleId, matchId: payload.matchId, attempts: 0, payload: mutation, payloadChecksum: hashStableJson(mutation), state: 'pending', }, }, }; } function parseMatchSideEffectMutation( value: Record | undefined, ): ParsedMatchSideEffectMutation | null { if (value?.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) { return { type: SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE }; } if (value?.type === RESET_MATCH_SIDE_EFFECT_TYPE) { return { type: RESET_MATCH_SIDE_EFFECT_TYPE }; } if (value?.type === UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE) { return { type: UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE }; } return null; } function buildSideEffectApplicationIdentityFromPayload( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, ) { return { auctionId: payload.auctionId, auctionStateTransitionId: payload.auctionStateTransitionId, sideEffectType: mutation.type, }; } function hashStableJson(value: unknown): string { return createHash('sha256') .update( JSON.stringify( value, Object.keys(value as Record).sort(), ), ) .digest('hex'); } function hasMatchedDocument(result: unknown): boolean { if (!result || typeof result !== 'object') return false; const record = result as Record; const matchedCount = Number(record.matchedCount ?? record.modifiedCount ?? 0); return matchedCount > 0; } function buildMatchTeamsFromProjection( projection: Record, match: { participants: Array<{ preferredPositions?: Array<{ position?: string }>; userId: { toString(): string }; }>; }, ) { const auctionTeams = Array.isArray(projection.auctionTeams) ? (projection.auctionTeams as Array>) : []; const participantByUserId = new Map( match.participants.map((participant) => [ participant.userId.toString(), participant, ]), ); return auctionTeams.map((team) => { const teamIndex = Number(team.teamIndex); const captainUserId = team.captainUserId as MatchTeamMember['userId']; const captain = participantByUserId.get(captainUserId.toString()); const rawMembers = Array.isArray(team.members) ? (team.members as Array>) : []; const members: MatchTeamMember[] = [ { assignedPosition: (captain?.preferredPositions?.[0]?.position as | Position | undefined) ?? 'MID', cost: 0, userId: captainUserId, }, ...rawMembers .filter( (member) => member.userId?.toString() !== captainUserId.toString(), ) .map((member) => ({ assignedPosition: (member.assignedPosition ?? 'MID') as Position, cost: Number(member.memberAcquisitionBidPointAmount ?? 0), userId: member.userId as MatchTeamMember['userId'], })), ]; return { captainUserId, members, name: `${teamIndex}팀`, teamIndex, }; }); } function parseRedisHash(reply: unknown): Record { if (!reply) return {}; if (Array.isArray(reply)) { const result: Record = {}; for (let index = 0; index < reply.length; index += 2) { result[String(reply[index])] = String(reply[index + 1]); } return result; } if (typeof reply === 'object') { return Object.fromEntries( Object.entries(reply as Record).map(([key, value]) => [ key, String(value), ]), ); } return {}; } function formatErrorMessageForRedis(error: unknown): string { const message = error instanceof Error ? error.message : String(error); return message.slice(0, AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH); } function calculateDurabilityRetryDelayMS(attempts: number): number { if (!Number.isFinite(attempts) || attempts <= 1) { return AUCTION_DURABILITY_RETRY_DELAY_MS; } return Math.min( AUCTION_DURABILITY_MAX_RETRY_DELAY_MS, AUCTION_DURABILITY_RETRY_DELAY_MS * 2 ** (attempts - 1), ); } async function execMaybe( queryOrPromise: Promise | { exec(): Promise }, ) { if ( queryOrPromise && typeof (queryOrPromise as { exec?: unknown }).exec === 'function' ) { return (queryOrPromise as { exec(): Promise }).exec(); } return queryOrPromise; }