import { createClient } from 'redis'; import mongoose from 'mongoose'; import { AuctionService } from '../src/modules/auction/auction.service'; import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service'; import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service'; import { AuctionRedisStateReaderService } from '../src/modules/auction/redis/auction-redis-state-reader.service'; import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys'; import { AuctionLifecycleLock, AuctionLifecycleBusyError, } from '../src/modules/auction/auction-lifecycle-lock'; import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler'; import { AuctionSideEffectService } from '../src/modules/auction/auction-side-effect.service'; import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service'; import { AUCTION_DURABILITY_RETRY_QUEUE_KEY, AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, AUCTION_BID_ARCHIVE_QUEUE_KEY, AUCTION_BID_ARCHIVE_PAYLOADS_KEY, AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, } from '../src/modules/auction/redis/auction-redis.keys'; const client = createClient({ url: process.env.REDIS_TEST_URL, socket: { reconnectStrategy: false }, }); const redis = new AuctionRedisService(client); const commands = new AuctionRedisCommandService(redis); const reader = new AuctionRedisStateReaderService(redis); beforeAll(async () => { if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) { throw new Error('Use test:redis to start an isolated local Redis server.'); } client.on('error', () => {}); await client.connect(); }); beforeEach(async () => { await client.flushDb(); }); afterAll(async () => { if (client.isOpen) await client.quit(); }); it.each(['count', 'elapsed'])( 'dead-letters permanently pending side effects by %s while retaining the payload', async (limit) => { const key = 'review:pending'; const now = Date.now(); await client.hSet(key, { payloadJson: '{}', pendingAttempts: limit === 'count' ? '119' : '1', pendingSinceTimestampMS: String(limit === 'elapsed' ? now - 600000 : now), }); await client.zAdd(AUCTION_DURABILITY_RETRY_QUEUE_KEY, { score: now, value: key, }); const scheduler = new AuctionDurabilityScheduler({} as never, redis); const internal = scheduler as unknown as { deferDurabilityPayload( key: string, time: number, error: unknown, ): Promise; }; expect( await internal.deferDurabilityPayload( key, now, new Error('state unavailable'), ), ).toBe('dead-lettered'); expect( await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key), ).toBeNull(); expect( await client.zScore(AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, key), ).toBe(now); expect(await client.hGet(key, 'payloadJson')).toBe('{}'); expect(await client.hGet(key, 'attempts')).toBeNull(); expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe( now, ); }, ); it('defers a temporary coordination wait without consuming failed-work attempts or recreating removed payloads', async () => { const key = 'review:pending'; const now = Date.now(); const scheduler = new AuctionDurabilityScheduler({} as never, redis); const internal = scheduler as unknown as { deferDurabilityPayload( key: string, time: number, error: unknown, ): Promise; }; expect( await internal.deferDurabilityPayload(key, now, new Error('busy')), ).toBe('gone'); expect(await client.exists(key)).toBe(0); await client.hSet(key, 'payloadJson', '{}'); expect( await internal.deferDurabilityPayload(key, now, new Error('busy')), ).toBe('requeued'); expect(await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key)).toBe( now + 5000, ); expect(await client.hGet(key, 'attempts')).toBeNull(); }); async function fixture() { const matchId = new mongoose.Types.ObjectId(); const id = matchId.toString(); const lifecycle = `life:${id}`; const playerA = new mongoose.Types.ObjectId().toString(); const playerB = new mongoose.Types.ObjectId().toString(); const captain = new mongoose.Types.ObjectId().toString(); const initialRound = `initial:${id}`; const structure = { auctionPlayerOrder: [{ userId: playerA }, { userId: playerB }], auctionTeams: [ { teamIndex: 1, captainUserId: captain, remainingBidPointAmount: 100000, members: [], }, ], currentAuctionPlayerOrderIndex: 0, currentAuctionRound: 1, passedAuctionPlayerQueue: [], maximumBidRoundDurationSeconds: 30, bidRoundExtensionSeconds: 5, initialTeamBidPointAmount: 100000, topTierAuctionPriorityThreshold: 'NONE', manualTopTierAuctionPriorityUserIds: [], }; await commands.hydrateState(id, { matchId: id, auctionLifecycleId: lifecycle, revision: 0, auctionHistorySequence: 1, phase: 'bidding', maximumBidRoundDurationSeconds: 30, structureStateJson: JSON.stringify(structure), liveStatePatch: { currentAuctionPlayerOrderIndex: 0, currentBidRoundPlayerUserId: playerA, currentBidRoundId: initialRound, maximumBidRoundDurationSeconds: 30, bidRoundExtensionSeconds: 5, 'team:1:captainUserId': captain, 'team:1:remainingBidPointAmount': 100000, 'team:1:assignedAuctionTeamMemberCount': 1, }, }); const auction = { _id: new mongoose.Types.ObjectId(), matchId, auctionEngineVersion: 'redis-v1', auctionLifecycleId: lifecycle, phase: 'bidding', revision: 1, }; const service = new AuctionService( { getModel: async () => ({ findOne: async () => auction }) } as never, {} as never, {} as never, {} as never, redis, commands, reader, ); const bid = ( requestId: string, amount: number, player = playerA, round = initialRound, ) => ({ matchId: id, bidderTeamIndex: 1, bidderUserId: captain, bidPointAmount: amount, bidRequestId: requestId, clientObservedAuctionLifecycleId: lifecycle, clientObservedCurrentBidRoundPlayerUserId: player, clientObservedBidRoundId: round, maximumAuctionTeamMemberCount: 5, minimumBidPointReservePerRemainingAuctionTeamMemberSlot: 10, maximumStoredAuctionBidHistoryCount: 1000, maximumStoredBidRequestResultCount: 500, minimumBidRequestResultReplayWindowMS: 300000, }); const undo = async (snapshot: Record) => { const internal = service as unknown as { undoRedisAuction(params: unknown): Promise; }; return internal.undoRedisAuction({ auction, snapshot, targetHistorySequence: 1, undoingCompletedAuction: false, }); }; return { id, matchId, lifecycle, playerA, playerB, initialRound, auction, service, bid, undo, keys: buildAuctionRedisKeys(id), }; } it('waits briefly for a busy recovery lock and returns the recovered state', async () => { const f = await fixture(); f.auction.revision = 10; const key = `auction:${f.id}:lifecycleMutationLock`; await redis.acquireTokenLock(key, 'other-reader', 30000); const released = new Promise((resolve, reject) => { setTimeout(() => { void redis.releaseTokenLock(key, 'other-reader').then(resolve, reject); }, 100); }); const [view] = await Promise.all([ f.service.getStateByMatchId(f.matchId), released, ]); expect(view).toMatchObject({ auctionLifecycleId: f.lifecycle, revision: 11 }); }); it('bounds recovery waits and reports a retriable state error without releasing another owner lock', async () => { const f = await fixture(); f.auction.revision = 10; const key = `auction:${f.id}:lifecycleMutationLock`; await redis.acquireTokenLock(key, 'slow-owner', 30000); await expect(f.service.getStateByMatchId(f.matchId)).rejects.toMatchObject({ response: { code: 'AUCTION_STATE_RECOVERING', retryAfterMS: 500 }, }); expect(await client.get(key)).toBe('slow-owner'); }); it('keeps accepting bids after 1000 history entries and preserves accepted request replay across players', async () => { const f = await fixture(); const first = await commands.placeBid( f.id, f.bid('accepted-before-next', 10), ); expect(first).toMatchObject({ ok: true }); await client.rPush( f.keys.bidHistoryKey, Array.from({ length: 999 }, () => JSON.stringify({ bidPointAmount: 10 })), ); await client.hSet(f.keys.liveStateKey, 'phase', 'sold'); await f.service.next(f.matchId); await f.service.openBidding(f.matchId); const state = (await reader.readAuctionStateView(f.id))!; const result = await commands.placeBid( f.id, f.bid('next-player', 10, f.playerB, String(state.currentBidRoundId)), ); expect(result).toMatchObject({ ok: true }); expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000); expect( await commands.placeBid(f.id, f.bid('accepted-before-next', 10)), ).toMatchObject({ ...first, isReplay: true }); }); it('500 rejected requests do not block a valid bid, and rejection retention stays bounded', async () => { const f = await fixture(); await client.hSet(f.keys.liveStateKey, 'currentHighestBidPointAmount', '10'); for (let i = 0; i < 501; i++) { expect( await commands.placeBid(f.id, f.bid(`too-low:${i}`, 5)), ).toMatchObject({ rejected: true, code: 'TOO_LOW' }); } expect(await client.lLen(f.keys.rejectedBidRequestResultOrderKey)).toBe(500); expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(0); expect(await commands.placeBid(f.id, f.bid('valid', 20))).toMatchObject({ ok: true, }); expect(await commands.placeBid(f.id, f.bid('valid', 30))).toMatchObject({ rejected: true, code: 'DUPLICATE_REQUEST_CONFLICT', }); }); it('still protects 500 accepted results within their replay window', async () => { const f = await fixture(); for (let i = 1; i <= 500; i++) expect( await commands.placeBid(f.id, f.bid(`accepted:${i}`, i)), ).toMatchObject({ ok: true }); expect(await commands.placeBid(f.id, f.bid('overflow', 501))).toMatchObject({ rejected: true, code: 'BID_REQUEST_RESULT_LIMIT', }); expect(await commands.placeBid(f.id, f.bid('accepted:1', 1))).toMatchObject({ ok: true, isReplay: true, }); }); it('migrates rejected results from the former shared quota without losing accepted replays', async () => { const f = await fixture(); await commands.placeBid(f.id, f.bid('accepted', 10)); await client.hDel(f.keys.bidRequestResultsKey, '_acceptedOnlyOrderV1'); for (let i = 0; i < 499; i++) { const id = `legacy-rejected:${i}`; await client.hSet(f.keys.bidRequestResultsKey, { [`${id}:status`]: 'rejected', [`${id}:createdAtTimestampMS`]: String(Date.now()), }); await client.rPush(f.keys.bidRequestResultOrderKey, id); } expect( await commands.placeBid(f.id, f.bid('next-accepted', 20)), ).toMatchObject({ ok: true }); expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(2); expect(await commands.placeBid(f.id, f.bid('accepted', 10))).toMatchObject({ ok: true, isReplay: true, }); }); it('encodes opaque round identities safely in bid history', async () => { const f = await fixture(); const round = 'round:with"quotes\\and-slashes'; await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', round); expect( await commands.placeBid(f.id, f.bid('quoted-round', 10, f.playerA, round)), ).toMatchObject({ ok: true, auctionBidHistoryEntry: { bidRoundId: round } }); expect((await reader.readAuctionStateView(f.id))?.auctionBidHistory).toEqual([ expect.objectContaining({ bidRoundId: round }), ]); }); it('rejects delayed pre-undo bids and omitted round identities while accepting the restored round', async () => { const f = await fixture(); const snapshot = (await reader.readAuctionStateView(f.id))!; await f.undo(snapshot); const restored = (await reader.readAuctionStateView(f.id))!; expect(restored.currentBidRoundId).not.toBe(f.initialRound); expect(await commands.placeBid(f.id, f.bid('old-round', 100))).toMatchObject({ code: 'STALE_BID_ROUND', }); expect( await commands.placeBid(f.id, { ...f.bid('missing-round', 100), clientObservedBidRoundId: undefined, }), ).toMatchObject({ code: 'BID_CLIENT_UPDATE_REQUIRED' }); expect( await commands.placeBid( f.id, f.bid('new-round', 100, f.playerA, String(restored.currentBidRoundId)), ), ).toMatchObject({ ok: true, auctionBidHistoryEntry: { playerUserId: f.playerA, bidRoundId: restored.currentBidRoundId, }, }); }); it('removes completion TTLs when resuming an auction', async () => { const f = await fixture(); const snapshot = (await reader.readAuctionStateView(f.id))!; await client.expire(f.keys.liveStateKey, 604800); await client.expire(f.keys.structureStateKey, 604800); await f.undo(snapshot); expect(await client.ttl(f.keys.liveStateKey)).toBe(-1); expect(await client.ttl(f.keys.structureStateKey)).toBe(-1); }); it('fences direct foreign-lifecycle hydration and repairs an orphan using current Mongo authority', async () => { const f = await fixture(); await client.hSet(f.keys.liveStateKey, 'revision', '150'); expect( await commands.hydrateState(f.id, { matchId: f.id, auctionLifecycleId: 'replacement', revision: 0, auctionHistorySequence: 1, phase: 'presenting', maximumBidRoundDurationSeconds: 30, structureStateJson: '{}', }), ).toMatchObject({ rejected: true, code: 'LIFECYCLE_CONFLICT' }); const current = { ...f.auction, auctionLifecycleId: 'replacement', phase: 'presenting', revision: 1, auctionTeams: [], auctionPlayerOrder: [], }; jest.spyOn(f.service, 'getByMatchId').mockResolvedValue(current as never); expect(await f.service.getStateByMatchId(f.matchId)).toMatchObject({ auctionLifecycleId: 'replacement', phase: 'presenting', }); expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe( 'replacement', ); }); it('does not let an old Mongo read overwrite the newly started lifecycle', async () => { const f = await fixture(); const stale = { ...f.auction, auctionLifecycleId: 'obsolete', revision: 500 }; jest .spyOn(f.service, 'getByMatchId') .mockResolvedValueOnce(stale as never) .mockResolvedValue(f.auction as never); await expect(f.service.getStateByMatchId(f.matchId)).rejects.toThrow( 'lifecycle has changed', ); expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe( f.lifecycle, ); }); it('serializes lifecycle mutations without deleting another owner lock', async () => { const lock = new AuctionLifecycleLock(redis); await lock.run('match', async () => { await expect(lock.run('match', async () => {})).rejects.toThrow( AuctionLifecycleBusyError, ); }); await expect(lock.run('match', async () => 'done')).resolves.toBe('done'); }); it('persists the exact Redis deadline when opening bidding and restoring a bidding snapshot', async () => { const f = await fixture(); await client.hSet(f.keys.liveStateKey, 'phase', 'presenting'); await client.hDel(f.keys.liveStateKey, 'currentBidRoundDeadlineTimestampMS'); await f.service.openBidding(f.matchId); const scheduler = new AuctionDurabilityScheduler({} as never, redis); const verify = async () => { const live = (await reader.readAuctionStateView(f.id))!; const queued = await client.zRange( AUCTION_DURABILITY_RETRY_QUEUE_KEY, 0, -1, ); const payloads = await Promise.all( queued.map((key) => scheduler.readDurabilityPayload(key)), ); const payload = payloads.find( (entry) => entry?.revision === live.revision, )!; expect( payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS, ).toBe(live.currentBidRoundDeadlineTimestampMS); expect( new Date( String(payload.afterAuctionMongoProjection.currentDeadline), ).getTime(), ).toBe(live.currentBidRoundDeadlineTimestampMS); expect( payload.afterAuctionMongoProjection.passedAuctionPlayerQueue, ).toEqual([]); return live; }; const snapshot = await verify(); await f.undo({ ...snapshot, currentBidRoundDeadlineTimestampMS: 1 }); await verify(); }); it('archives every accepted bid beyond the recent 1000, with no duplicate archive event for a replay', async () => { const f = await fixture(); for (let amount = 1; amount <= 1005; amount++) { expect( await commands.placeBid(f.id, { ...f.bid(`archive:${amount}`, amount), maximumStoredBidRequestResultCount: 2000, }), ).toMatchObject({ ok: true }); } expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000); expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005); expect( await commands.placeBid(f.id, { ...f.bid('archive:1', 1), maximumStoredBidRequestResultCount: 2000, }), ).toMatchObject({ ok: true, isReplay: true }); expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005); const [first] = await client.zRange(AUCTION_BID_ARCHIVE_QUEUE_KEY, 0, 0); expect( JSON.parse((await client.hGet(AUCTION_BID_ARCHIVE_PAYLOADS_KEY, first))!), ).toMatchObject({ bidPointAmount: 1, bidRequestId: 'archive:1', playerUserId: f.playerA, auctionLifecycleId: f.lifecycle, }); }); it.each(['missing', 'empty'])( 'archives an accepted pre-upgrade bid with a %s Redis round ID', async (kind) => { const f = await fixture(); if (kind === 'missing') await client.hDel(f.keys.liveStateKey, 'currentBidRoundId'); else await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', ''); expect( await commands.placeBid(f.id, { ...f.bid('legacy-round', 10), clientObservedBidRoundId: undefined, }), ).toMatchObject({ ok: true, auctionBidHistoryEntry: { bidRoundId: '', playerUserId: f.playerA }, }); const bulkWrite = jest.fn(async (operations: unknown[]) => ({ matchedCount: 0, upsertedCount: operations.length, })); const archive = new AuctionBidArchiveService( { getModel: async () => ({ bulkWrite }) } as never, redis, ); await expect(archive.persistPendingBids()).resolves.toBe(1); const write = bulkWrite.mock.calls[0][0][0] as { updateOne: { update: { $setOnInsert: Record } }; }; expect(write.updateOne.update.$setOnInsert).not.toHaveProperty( 'bidRoundId', ); expect(String(write.updateOne.update.$setOnInsert.playerUserId)).toBe( f.playerA, ); expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(0); }, ); it('does not mirror an old generation failure over a newer Redis side effect', async () => { const f = await fixture(); await client.hSet(f.keys.liveStateKey, { matchSideEffectTransitionId: 'new', matchSideEffectRevision: '2', matchSideEffectStatus: 'pending', }); const sideEffects = new AuctionSideEffectService(redis); expect( await sideEffects.mirrorMatchSideEffectStatus({ matchId: f.id, auctionLifecycleId: f.lifecycle, auctionStateTransitionId: 'old', matchSideEffectRevision: 100, matchSideEffectStatus: 'failed', }), ).toBe(false); expect(await client.hGet(f.keys.liveStateKey, 'matchSideEffectStatus')).toBe( 'pending', ); }); it('queues a terminal notification for ordinary failures and backfills old dead letters', async () => { const key = 'review:ordinary-failure'; await client.hSet(key, { payloadJson: '{}', attempts: '9' }); const scheduler = new AuctionDurabilityScheduler({} as never, redis); const internal = scheduler as unknown as { recordDurabilityPayloadFailure( key: string, now: number, error: unknown, ): Promise; }; expect( await internal.recordDurabilityPayloadFailure( key, 1000, new Error('failed before worker'), ), ).toBe('dead-lettered'); expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe( 1000, ); // Simulate an older deployment which did not create notification jobs. await client.zRem(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key); await scheduler.reconcileDeadLetterStatuses(); expect( await client.hGet(key, 'deadLetterStatusReconciledAtTimestampMS'), ).not.toBeNull(); expect( await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key), ).toBeNull(); });