import os from 'node:os'; import mongoose from 'mongoose'; import { AuctionService } from '../src/modules/auction/auction.service'; import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler'; import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service'; import { migrateAuctionCancellationIndex } from '../src/modules/global/database/migrations/20261109-auction-cancellation-index'; import { AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY } from '../src/modules/auction/redis/auction-redis.keys'; import { TournamentCodeService } from '../src/modules/tournament-code/tournament-code.service'; import { TournamentResultService } from '../src/modules/tournament-result/tournament-result.service'; import { MatchSchema, AuctionSchema, AuctionHistoryEntrySchema, AuctionSideEffectApplicationSchema, TournamentCodeSchema, AuctionBidRecordSchema, } from '../src/modules/global/database/schema'; let connection: mongoose.Connection; let models: ReturnType; const redis = { acquireTokenLock: async () => true, releaseTokenLock: async () => true, getCurrentRedisTimestampMS: async () => Date.now(), raw: () => ({ sendCommand: async (command: string[]) => command[0] === 'SCAN' ? ['0', []] : 1, }), }; function createModels(connection: mongoose.Connection) { return { match: connection.model('match', MatchSchema), auction: connection.model('auction', AuctionSchema), auctionHistory: connection.model( 'auctionHistory', AuctionHistoryEntrySchema, ), auctionSideEffectApplication: connection.model( 'auctionSideEffectApplication', AuctionSideEffectApplicationSchema, ), tournamentCode: connection.model('tournamentCode', TournamentCodeSchema), auctionBidRecord: connection.model( 'auctionBidRecord', AuctionBidRecordSchema, ), }; } beforeAll(async () => { if ( process.env.MONGO_TEST_ISOLATED !== '1' || !process.env.MONGO_TEST_URL?.startsWith('mongodb://127.0.0.1:') ) throw new Error('Use test:mongodb with an isolated local replica set.'); connection = await mongoose .createConnection(process.env.MONGO_TEST_URL, { autoIndex: false, autoCreate: false, // NODE-7832: the driver's default OS adapter breaks the Jest handshake. runtimeAdapters: { os }, }) .asPromise(); models = createModels(connection); await Promise.all( Object.values(models).map((model) => model.createCollection()), ); }); beforeEach(async () => { await Promise.all(Object.values(models).map((model) => model.deleteMany())); }); afterEach(() => jest.restoreAllMocks()); afterAll(async () => { await connection?.close(); }); async function fixture(reserved: boolean) { const matchId = new mongoose.Types.ObjectId(), communityId = new mongoose.Types.ObjectId(), auctionId = new mongoose.Types.ObjectId(), codeId = new mongoose.Types.ObjectId(); await models.match.create({ _id: matchId, communityId, title: 'Isolated auction test', scheduledAt: new Date(), status: 'team-built', teamBuiltAt: new Date(), teams: [], participants: [], tournamentCodeIds: reserved ? [codeId] : [], gameSchedule: [], }); await models.auction.collection.insertOne({ _id: auctionId, matchId, auctionEngineVersion: 'redis-v1', auctionLifecycleId: 'life', phase: 'complete', revision: 10, matchSideEffectRevision: 0, matchSideEffectStatus: 'none', }); if (reserved) await models.tournamentCode.collection.insertOne({ _id: codeId, code: 'ISOLATED-TEST', state: 'reserved', matchId, }); const database = { getModel: async (name: keyof typeof models) => models[name], createSession: () => connection.startSession(), }; const codes = new TournamentCodeService(database as never, redis as never); const results = new TournamentResultService( database as never, {} as never, {} as never, {} as never, {} as never, {} as never, {} as never, {} as never, ); const service = new AuctionService( database as never, {} as never, codes, results, redis as never, {} as never, {} as never, ); return { matchId, communityId, auctionId, codeId, service, database, codes, results, }; } it.each([false, true])( 'cancels the auction atomically with reserved codes=%s', async (reserved) => { const f = await fixture(reserved); await f.service.reset({ matchId: f.matchId, communityId: f.communityId, clientObservedAuctionLifecycleId: 'life', }); const match = await models.match.findById(f.matchId).lean(); expect(match).toMatchObject({ status: 'recruiting', teams: [], tournamentCodeIds: [], gameSchedule: [], }); expect(match?.teamBuiltAt).toBeUndefined(); expect(await models.auction.findById(f.auctionId)).toBeNull(); if (reserved) expect( (await models.tournamentCode.findById(f.codeId).lean())?.state, ).toBe('released'); }, ); it('rolls back code release and Match changes if cancellation later fails', async () => { const f = await fixture(true); jest.spyOn(models.auctionHistory, 'deleteMany').mockImplementation(() => { throw new Error('forced history failure'); }); await expect( f.service.reset({ matchId: f.matchId, communityId: f.communityId, clientObservedAuctionLifecycleId: 'life', }), ).rejects.toThrow('forced history failure'); expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe( 'reserved', ); expect(await models.match.findById(f.matchId).lean()).toMatchObject({ status: 'team-built', __v: 0, tournamentCodeIds: [f.codeId], }); expect(await models.auction.findById(f.auctionId)).not.toBeNull(); }); it('retries cancellation cleanup from Mongo after Redis fails, without another cancellation request', async () => { const f = await fixture(true); const cleanup = jest .spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle') .mockRejectedValueOnce(new Error('Redis offline')) .mockResolvedValue(undefined); await f.service.reset({ matchId: f.matchId, communityId: f.communityId, clientObservedAuctionLifecycleId: 'life', }); expect(cleanup).not.toHaveBeenCalled(); await f.service.retryPendingCancellations(); expect(await models.match.findById(f.matchId).lean()).toMatchObject({ auctionCancellationCleanupPending: true, auctionCancellationCleanupAttempts: 1, }); await models.match.updateOne( { _id: f.matchId }, { $set: { auctionCancellationCleanupRetryAt: new Date(0) } }, ); await f.service.retryPendingCancellations(); expect(cleanup).toHaveBeenCalledTimes(2); expect( (await models.match.findById(f.matchId).lean()) ?.auctionCancellationCleanupPending, ).toBe(false); }); it('keeps all 1005 Mongo bid records and idempotently retries when Redis acknowledgement fails', async () => { const f = await fixture(false); const events = Array.from({ length: 1005 }, (_, i) => ({ _id: `${f.matchId.toString()}:life:${String(i + 1).padStart(16, '0')}`, matchId: f.matchId.toString(), auctionLifecycleId: 'life', revision: i + 1, bidRequestId: `bid:${i}`, bidRoundId: 'round', bidderTeamIndex: 1, bidPointAmount: i + 1, bidderUserId: new mongoose.Types.ObjectId().toString(), playerUserId: new mongoose.Types.ObjectId().toString(), bidPlacedAtTimestampMS: Date.now(), })); let failAck = true; const sendCommand = jest.fn(async (command: string[]) => { if (command[0] === 'ZRANGE') return events.map((e) => e._id); if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e)); if (failAck) throw new Error('Redis acknowledgement failed'); return 1; }); const archive = new AuctionBidArchiveService( f.database as never, { raw: () => ({ sendCommand }) } as never, ); await expect(archive.persistPendingBids(2000)).rejects.toThrow( 'acknowledgement failed', ); expect(await models.auctionBidRecord.countDocuments()).toBe(1005); failAck = false; await expect(archive.persistPendingBids(2000)).resolves.toBe(1005); expect(await models.auctionBidRecord.countDocuments()).toBe(1005); const page = await archive.list(f.matchId, 1000, 100); expect(page.items.map((item) => item.revision)).toEqual([ 1001, 1002, 1003, 1004, 1005, ]); expect(page.hasMore).toBe(false); }); it('migrates the cleanup index idempotently and examines only due pending matches', async () => { const f = await fixture(false); const documents = Array.from({ length: 10000 }, (_, i) => ({ _id: new mongoose.Types.ObjectId(), lastCancelledAuctionLifecycleId: `life-${i}`, ...(i < 5 ? { auctionCancellationCleanupPending: true, auctionCancellationCleanupRetryAt: new Date(0), } : i === 5 ? { auctionCancellationCleanupPending: true, auctionCancellationCleanupRetryAt: new Date(Date.now() + 3600000), } : {}), })); await models.match.collection.insertMany(documents); const find = jest.spyOn(models.match, 'find'); const cleanup = jest .spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle') .mockResolvedValue(undefined); await f.service.retryPendingCancellations(); expect(cleanup).toHaveBeenCalledTimes(5); const query = find.mock.calls[0][0] as unknown as Record; // Restore only the five due jobs so explain measures the actual worker query. await models.match.updateMany( { _id: { $in: documents.slice(0, 5).map((d) => d._id) } }, { $set: { auctionCancellationCleanupPending: true, auctionCancellationCleanupRetryAt: new Date(0), }, }, ); const explain = () => models.match.collection .find(query) .sort({ auctionCancellationCleanupRetryAt: 1 }) .limit(20) .explain('executionStats'); expect( (await explain()).executionStats.totalDocsExamined, ).toBeGreaterThanOrEqual(10000); expect(await migrateAuctionCancellationIndex(connection.db!)).toMatchObject({ dryRun: true, alreadyPresent: false, }); expect( (await models.match.collection.listIndexes().toArray()).some( (i) => i.name === 'auction_cancellation_cleanup_due', ), ).toBe(false); await migrateAuctionCancellationIndex(connection.db!, { dryRun: false }); expect( await migrateAuctionCancellationIndex(connection.db!, { dryRun: false }), ).toMatchObject({ alreadyPresent: true }); const plan = await explain(); expect(plan.executionStats.nReturned).toBe(5); expect(plan.executionStats.totalDocsExamined).toBe(5); expect(JSON.stringify(plan.queryPlanner.winningPlan)).toContain('IXSCAN'); }); function archiveEvent(matchId: mongoose.Types.ObjectId, revision: number) { return { _id: `${matchId.toString()}:life:${String(revision).padStart(16, '0')}`, matchId: matchId.toString(), auctionLifecycleId: 'life', revision, bidRequestId: `bid-${revision}`, bidRoundId: 'round', bidderUserId: new mongoose.Types.ObjectId().toString(), playerUserId: new mongoose.Types.ObjectId().toString(), bidderTeamIndex: 1, bidPointAmount: revision, bidPlacedAtTimestampMS: Date.now(), }; } it.each(['', undefined, 'round'])( 'persists and lists bids with optional round ID %s without duplicating retries', async (bidRoundId) => { const f = await fixture(false); const event = { ...archiveEvent(f.matchId, 1), bidRoundId }; const sendCommand = jest.fn(async (command: string[]) => { if (command[0] === 'ZRANGE') return [event._id]; if (command[0] === 'HMGET') return [JSON.stringify(event)]; return 1; }); const archive = new AuctionBidArchiveService( f.database as never, { raw: () => ({ sendCommand }) } as never, ); await expect(archive.persistPendingBids()).resolves.toBe(1); await expect(archive.persistPendingBids()).resolves.toBe(1); expect(await models.auctionBidRecord.countDocuments()).toBe(1); const page = await archive.list(f.matchId); expect(page.items).toHaveLength(1); expect(String(page.items[0].playerUserId)).toBe(event.playerUserId); if (bidRoundId) expect(page.items[0].bidRoundId).toBe(bidRoundId); else expect(page.items[0]).not.toHaveProperty('bidRoundId'); expect( sendCommand.mock.calls .filter(([command]) => command[0] === 'EVAL') .every(([command]) => command[2] === '2'), ).toBe(true); }, ); it.each([ 'json', 'revision', 'object-id', 'required-field', 'identity', 'missing-payload', ])( 'quarantines an invalid %s without losing the valid Mongo bid', async (kind) => { const f = await fixture(false); const bad = archiveEvent(f.matchId, 1), good = archiveEvent(f.matchId, 2); const invalid: Record = { ...bad }; if (kind === 'revision') invalid.revision = -1; if (kind === 'object-id') invalid.bidderUserId = 'not-an-object-id'; if (kind === 'required-field') delete invalid.bidderUserId; if (kind === 'identity') invalid.matchId = new mongoose.Types.ObjectId().toString(); const raw = kind === 'json' ? '{broken' : kind === 'missing-payload' ? null : JSON.stringify(invalid); const sendCommand = jest.fn(async (command: string[]) => { if (command[0] === 'ZRANGE') return [bad._id, good._id]; if (command[0] === 'HMGET') return [raw, JSON.stringify(good)]; return 1; }); const archive = new AuctionBidArchiveService( f.database as never, { raw: () => ({ sendCommand }) } as never, ); await expect(archive.persistPendingBids()).resolves.toBe(1); expect(await models.auctionBidRecord.countDocuments()).toBe(1); expect( await models.auctionBidRecord.findById(good._id).lean(), ).toMatchObject({ revision: 2 }); expect(await models.auctionBidRecord.findById(bad._id)).toBeNull(); const quarantine = sendCommand.mock.calls .map((c) => c[0]) .find((c) => c[0] === 'EVAL' && c[2] === '4')!; expect(JSON.parse(quarantine[11])).toMatchObject({ id: bad._id, rawPayload: raw, error: expect.any(String), }); const ack = sendCommand.mock.calls .map((c) => c[0]) .find((c) => c[0] === 'EVAL' && c[2] === '2')!; expect(ack.slice(5)).toEqual([good._id, JSON.stringify(good)]); }, ); it.each(['partial-write-error', 'partial-acknowledgement'])( 'retains the batch after %s and retries Mongo upserts without duplicates', async (failure) => { const f = await fixture(false); const events = [archiveEvent(f.matchId, 1), archiveEvent(f.matchId, 2)]; const sendCommand = jest.fn(async (command: string[]) => { if (command[0] === 'ZRANGE') return events.map((e) => e._id); if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e)); return 1; }); const archive = new AuctionBidArchiveService( f.database as never, { raw: () => ({ sendCommand }) } as never, ); const write = models.auctionBidRecord.bulkWrite.bind( models.auctionBidRecord, ); jest .spyOn(models.auctionBidRecord, 'bulkWrite') .mockImplementationOnce(async (operations, options) => { const result = await write(operations.slice(0, 1), options); if (failure === 'partial-write-error') throw new Error('Lost Mongo acknowledgement'); return result; }); await expect(archive.persistPendingBids()).rejects.toThrow(); expect(await models.auctionBidRecord.countDocuments()).toBe(1); expect( sendCommand.mock.calls.filter(([c]) => c[0] === 'EVAL'), ).toHaveLength(0); await expect(archive.persistPendingBids()).resolves.toBe(2); expect(await models.auctionBidRecord.countDocuments()).toBe(2); }, ); it.each(['Mongo', 'Redis mirroring'])( 'reconciles a terminal status after %s recovers', async (outage) => { const f = await fixture(false); const payload = { auctionId: f.auctionId.toString(), matchId: f.matchId.toString(), auctionLifecycleId: 'life', auctionStateTransitionId: 'complete-10', matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' }, afterAuctionMongoProjection: { matchSideEffectRevision: 1 }, }; await models.auction.updateOne( { _id: f.auctionId }, { $set: { matchSideEffectTransitionId: 'complete-10', matchSideEffectRevision: 1, matchSideEffectStatus: 'pending', }, }, ); let unavailable = outage === 'Mongo'; const sendCommand = jest.fn(async (command: string[]) => { if (command[0] === 'ZRANGEBYSCORE') return ['dead-letter']; if (command[0] === 'HGETALL') return { payloadJson: JSON.stringify(payload) }; if (command[0] === 'HGET') return 'retry limit exceeded'; return 1; }); const mirror = jest.fn().mockResolvedValue(true), broadcast = jest.fn(); if (outage === 'Redis mirroring') mirror.mockRejectedValueOnce(new Error('Redis unavailable')); const scheduler = new AuctionDurabilityScheduler( { getModel: async (name: string) => { if (unavailable) throw new Error('Mongo offline'); return models[name]; }, } as never, { raw: () => ({ sendCommand }), getCurrentRedisTimestampMS: async () => 1000, } as never, undefined, undefined, { mirrorMatchSideEffectStatus: mirror } as never, { broadcastMatchSideEffect: broadcast } as never, ); await scheduler.reconcileDeadLetterStatuses(); expect(sendCommand).toHaveBeenCalledWith([ 'ZADD', AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, '6000', 'dead-letter', ]); unavailable = false; await scheduler.reconcileDeadLetterStatuses(); expect( (await models.auction.findById(f.auctionId).lean()) ?.matchSideEffectStatus, ).toBe('failed'); expect(mirror).toHaveBeenCalledWith( expect.objectContaining({ auctionStateTransitionId: 'complete-10', matchSideEffectStatus: 'failed', }), ); expect(broadcast).toHaveBeenCalledTimes(1); // Replaying the old notification must not overwrite a new generation. await models.auction.updateOne( { _id: f.auctionId }, { $set: { matchSideEffectTransitionId: 'new', matchSideEffectStatus: 'processing', matchSideEffectRevision: 20, }, }, ); await scheduler.reconcileDeadLetterStatuses(); expect( (await models.auction.findById(f.auctionId).lean()) ?.matchSideEffectStatus, ).toBe('processing'); expect(broadcast).toHaveBeenCalledTimes(1); }, ); it('cleans up reserved codes and Match metadata after undoing completion', async () => { const f = await fixture(true); const scheduler = new AuctionDurabilityScheduler( f.database as never, redis as never, f.codes, f.results, undefined, undefined, { readAuctionStateView: async () => ({ auctionLifecycleId: 'life', phase: 'bidding', matchSideEffectTransitionId: 'undo', }), } as never, ); await scheduler.applyDurabilityPayload({ auctionId: f.auctionId, matchId: f.matchId, auctionLifecycleId: 'life', revision: 11, auctionHistorySequence: 11, auctionActionType: 'undo', auctionStateTransitionId: 'undo', snapshotBefore: {}, historyMutation: { type: 'append' }, matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' }, afterAuctionMongoProjection: { phase: 'bidding', revision: 11, matchSideEffectTransitionId: 'undo', matchSideEffectRevision: 1, matchSideEffectStatus: 'pending', }, }); expect(await models.match.findById(f.matchId).lean()).toMatchObject({ status: 'team-built', tournamentCodeIds: [], gameSchedule: [], teams: [], }); expect( (await models.match.findById(f.matchId).lean())?.teamBuiltAt, ).toBeUndefined(); expect( (await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus, ).toBe('completed'); }); it('reads an uncommitted Match in the provided session when cancelling polling by id', async () => { const f = await fixture(false); const session = await connection.startSession(); try { session.startTransaction(); const [created] = await models.match.create( [ { communityId: f.communityId, title: 'Uncommitted', scheduledAt: new Date(), }, ], { session }, ); expect(await models.match.findById(created._id)).toBeNull(); await f.results.cancelPolling(created._id, session); expect( await models.match.findById(created._id).session(session), ).not.toBeNull(); await session.abortTransaction(); } finally { await session.endSession(); } }); it('rolls back failed Undo cleanup and successfully retries with fresh Match versions', async () => { const f = await fixture(true); const scheduler = new AuctionDurabilityScheduler( f.database as never, redis as never, f.codes, f.results, undefined, undefined, { readAuctionStateView: async () => ({ auctionLifecycleId: 'life', phase: 'bidding', matchSideEffectTransitionId: 'undo-retry', }), } as never, ); const payload = { auctionId: f.auctionId, matchId: f.matchId, auctionLifecycleId: 'life', revision: 11, auctionHistorySequence: 11, auctionActionType: 'undo', auctionStateTransitionId: 'undo-retry', snapshotBefore: {}, historyMutation: { type: 'append' }, matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' }, afterAuctionMongoProjection: { phase: 'bidding', revision: 11, matchSideEffectTransitionId: 'undo-retry', matchSideEffectRevision: 1, matchSideEffectStatus: 'pending', }, }; jest .spyOn(f.results, 'cancelPolling') .mockRejectedValueOnce(new Error('temporary poll cleanup failure')); await expect(scheduler.applyDurabilityPayload(payload)).rejects.toThrow( 'temporary poll cleanup failure', ); expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe( 'reserved', ); expect(await models.match.findById(f.matchId).lean()).toMatchObject({ __v: 0, tournamentCodeIds: [f.codeId], }); await scheduler.applyDurabilityPayload(payload); expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe( 'released', ); expect( (await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus, ).toBe('completed'); }); it('answers a retransmitted cancellation of the same lifecycle instead of 404', async () => { const f = await fixture(true); const request = { matchId: f.matchId, communityId: f.communityId, clientObservedAuctionLifecycleId: 'life', }; await f.service.reset(request); await expect(f.service.reset(request)).resolves.toBeUndefined(); expect( (await models.match.findById(f.matchId).lean()) ?.lastCancelledAuctionLifecycleId, ).toBe('life'); await expect( f.service.reset({ ...request, clientObservedAuctionLifecycleId: 'other' }), ).rejects.toThrow('Auction not found.'); }); it.each([ 'new-lifecycle', 'new-transition', 'completed', 'revision-ahead', 'mirror-superseded', ])('fences terminal notifications against %s', async (scenario) => { const f = await fixture(false); await models.auction.updateOne( { _id: f.auctionId }, { $set: { matchSideEffectTransitionId: 'target', matchSideEffectRevision: 1, matchSideEffectStatus: 'pending', }, }, ); const payload = { auctionId: f.auctionId.toString(), matchId: f.matchId.toString(), auctionLifecycleId: 'life', auctionStateTransitionId: 'target', matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' }, afterAuctionMongoProjection: { matchSideEffectRevision: 1 }, }; const state = { auctionLifecycleId: scenario === 'new-lifecycle' ? 'new-life' : 'life', matchSideEffectTransitionId: scenario === 'new-transition' ? 'new' : 'target', matchSideEffectStatus: scenario === 'completed' ? 'completed' : 'pending', matchSideEffectRevision: 40, }; const sendCommand = jest.fn(async (command: string[]) => { if (command[0] === 'ZRANGEBYSCORE') return ['terminal']; if (command[0] === 'HGETALL') return { payloadJson: JSON.stringify(payload) }; if (command[0] === 'HGET') return 'exhausted'; return 1; }); const broadcast = jest.fn(); const mirror = jest.fn().mockResolvedValue(scenario !== 'mirror-superseded'); const scheduler = new AuctionDurabilityScheduler( f.database as never, { raw: () => ({ sendCommand }), getCurrentRedisTimestampMS: async () => 1000, } as never, undefined, undefined, { mirrorMatchSideEffectStatus: mirror } as never, { broadcastMatchSideEffect: broadcast } as never, { readAuctionStateView: async () => state } as never, ); await scheduler.reconcileDeadLetterStatuses(); const changed = ['revision-ahead', 'mirror-superseded'].includes(scenario); expect( (await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus, ).toBe(changed ? 'failed' : 'pending'); if (changed) expect(mirror).toHaveBeenCalledWith( expect.objectContaining({ matchSideEffectRevision: 41 }), ); expect(broadcast).toHaveBeenCalledTimes( scenario === 'revision-ahead' ? 1 : 0, ); });