fix(auction): harden lifecycle and durable bid storage

Serialize lifecycle changes and make cancellation, deadlines, and match side effects recoverable and idempotent.

Archive accepted bids durably, quarantine invalid payloads, and preserve compatibility with empty round IDs. Synchronize HTTP polling and socket state with scoped resync events.

Add cancellation cleanup index migration and regression coverage. Validated with 744 API tests, 65 Redis integration tests, and 28 Mongo integration tests.
This commit is contained in:
dahyeon committed 2026-11-09 23:24:11 +09:00
1 parent f2eec479d9
commit f649490cd2
55 files changed
+5374 -522

No files matched your search

@@ -0,0 +1,180 @@
import { createClient } from 'redis';
import mongoose from 'mongoose';
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
import {
AUCTION_BID_ARCHIVE_QUEUE_KEY as queue,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY as payloads,
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY as deadQueue,
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY as deadPayloads,
} from '../src/modules/auction/redis/auction-redis.keys';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new AuctionRedisService(client);
const matchId = new mongoose.Types.ObjectId().toString();
function event(revision: number) {
return {
_id: `${matchId}:life:${String(revision).padStart(16, '0')}`,
matchId,
auctionLifecycleId: 'life',
revision,
bidRequestId: `bid-${revision}`,
bidderUserId: new mongoose.Types.ObjectId().toString(),
bidderTeamIndex: 1,
bidPointAmount: revision,
bidPlacedAtTimestampMS: Date.now(),
};
}
function fixture() {
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
matchedCount: 0,
upsertedCount: operations.length,
}));
const service = new AuctionBidArchiveService(
{ getModel: async () => ({ bulkWrite }) } as never,
redis,
);
return { service, bulkWrite };
}
async function enqueue(id: string, raw: string | null) {
await client.zAdd(queue, { score: 1, value: id });
if (raw !== null) await client.hSet(payloads, id, raw);
}
beforeAll(async () => {
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL)
throw new Error('Use the isolated Redis test runner.');
client.on('error', () => {});
await client.connect();
});
beforeEach(async () => {
await client.flushDb();
});
afterEach(() => jest.restoreAllMocks());
afterAll(async () => {
if (client.isOpen) await client.quit();
});
it('isolates malformed, missing and invalid payloads while saving valid bids and retaining the originals', async () => {
const { service, bulkWrite } = fixture();
const good = event(10);
const invalid = [
['broken', '{broken'],
['empty', ''],
['null', 'null'],
['array', '[]'],
['missing', null],
[event(1)._id, JSON.stringify({ ...event(1), revision: -1 })],
[event(2)._id, JSON.stringify({ ...event(2), bidderUserId: 'bad' })],
[event(3)._id, JSON.stringify({ ...event(3), bidderUserId: undefined })],
[event(4)._id, JSON.stringify({ ...event(4), bidPointAmount: '4' })],
] as Array<[string, string | null]>;
for (const [id, raw] of invalid) await enqueue(id, raw);
await enqueue(good._id, JSON.stringify(good));
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(bulkWrite.mock.calls[0][0]).toHaveLength(1);
expect(await client.zCard(queue)).toBe(0);
expect(await client.hLen(payloads)).toBe(0);
expect(await client.zCard(deadQueue)).toBe(invalid.length);
for (const [id, raw] of invalid) {
expect(JSON.parse((await client.hGet(deadPayloads, id))!)).toMatchObject({
id,
rawPayload: raw,
error: expect.any(String),
quarantinedAtTimestampMS: expect.any(Number),
});
}
expect(await client.ttl(deadPayloads)).toBe(-1);
await expect(service.persistPendingBids()).resolves.toBe(0);
});
it('retains valid bids during a Mongo outage, after moving malformed entries out of the pending queue', async () => {
const { service, bulkWrite } = fixture();
const good = event(1);
await enqueue('broken', '{broken');
await enqueue(good._id, JSON.stringify(good));
bulkWrite.mockRejectedValueOnce(new Error('Mongo offline'));
await expect(service.persistPendingBids()).rejects.toThrow('Mongo offline');
expect(await client.zRange(queue, 0, -1)).toEqual([good._id]);
expect(await client.hGet(payloads, good._id)).toBe(JSON.stringify(good));
expect(await client.zCard(deadQueue)).toBe(1);
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(await client.zCard(queue)).toBe(0);
});
it('does not quarantine a payload repaired concurrently after HMGET', async () => {
const { service } = fixture();
const good = event(1);
await enqueue(good._id, '{broken');
const send = client.sendCommand.bind(client);
jest.spyOn(client, 'sendCommand').mockImplementation(async (command) => {
const result = await send(command);
if (command[0] === 'HMGET')
await client.hSet(payloads, good._id, JSON.stringify(good));
return result;
});
await expect(service.persistPendingBids()).resolves.toBe(0);
expect(await client.zCard(deadQueue)).toBe(0);
expect(await client.zRange(queue, 0, -1)).toEqual([good._id]);
await expect(service.persistPendingBids()).resolves.toBe(1);
});
it('preserves pending payloads if quarantine cannot be written', async () => {
const { service, bulkWrite } = fixture();
const good = event(1);
await enqueue('broken', '{broken');
await enqueue(good._id, JSON.stringify(good));
await client.set(deadPayloads, 'wrong-type');
await expect(service.persistPendingBids()).rejects.toThrow('WRONGTYPE');
expect(await client.zCard(queue)).toBe(2);
expect(await client.hGet(payloads, 'broken')).toBe('{broken');
expect(bulkWrite).not.toHaveBeenCalled();
await client.del(deadPayloads);
await expect(service.persistPendingBids()).resolves.toBe(1);
});
it('does not acknowledge a payload changed while Mongo was saving', async () => {
const { service, bulkWrite } = fixture();
const good = event(1);
await enqueue(good._id, JSON.stringify(good));
bulkWrite.mockImplementationOnce(async () => {
await client.hSet(payloads, good._id, '{replaced');
return { matchedCount: 0, upsertedCount: 1 };
});
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(await client.hGet(payloads, good._id)).toBe('{replaced');
expect(await client.zCard(queue)).toBe(1);
});
it.each([
['bidRoundId', null],
['bidRoundId', 1],
['bidRoundId', {}],
['bidRoundId', []],
['bidRoundId', ' '],
['playerUserId', ''],
])('continues quarantining invalid %s=%j', async (field, value) => {
const { service, bulkWrite } = fixture();
const bad = { ...event(1), [field]: value };
await enqueue(bad._id, JSON.stringify(bad));
await expect(service.persistPendingBids()).resolves.toBe(0);
expect(bulkWrite).not.toHaveBeenCalled();
expect(await client.zCard(deadQueue)).toBe(1);
});
it('archives a pre-upgrade bid with an empty round ID instead of quarantining it', async () => {
const { service, bulkWrite } = fixture();
const legacy = { ...event(1), bidRoundId: '' };
await enqueue(legacy._id, JSON.stringify(legacy));
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(await client.zCard(deadQueue)).toBe(0);
expect(await client.zCard(queue)).toBe(0);
const [[operations]] = bulkWrite.mock.calls as unknown as [
[Array<{ updateOne: { update: { $setOnInsert: object } } }>],
];
expect(operations[0].updateOne.update.$setOnInsert).not.toHaveProperty(
'bidRoundId',
);
});
@@ -0,0 +1,769 @@
import mongoose from 'mongoose';
import { AuctionService } from '../src/modules/auction/auction.service';
import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler';
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
import { migrateAuctionCancellationIndex } from '../src/modules/global/database/migrations/20261109-auction-cancellation-index';
import { AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY } from '../src/modules/auction/redis/auction-redis.keys';
import { TournamentCodeService } from '../src/modules/tournament-code/tournament-code.service';
import { TournamentResultService } from '../src/modules/tournament-result/tournament-result.service';
import {
MatchSchema,
AuctionSchema,
AuctionHistoryEntrySchema,
AuctionSideEffectApplicationSchema,
TournamentCodeSchema,
AuctionBidRecordSchema,
} from '../src/modules/global/database/schema';
let connection: mongoose.Connection;
let models: Record<string, mongoose.Model<any>>;
const redis = {
acquireTokenLock: async () => true,
releaseTokenLock: async () => true,
getCurrentRedisTimestampMS: async () => Date.now(),
raw: () => ({
sendCommand: async (command: string[]) =>
command[0] === 'SCAN' ? ['0', []] : 1,
}),
};
beforeAll(async () => {
if (
process.env.MONGO_TEST_ISOLATED !== '1' ||
!process.env.MONGO_TEST_URL?.startsWith('mongodb://127.0.0.1:')
)
throw new Error('Use test:mongodb with an isolated local replica set.');
connection = await mongoose
.createConnection(process.env.MONGO_TEST_URL, {
autoIndex: false,
autoCreate: false,
})
.asPromise();
models = {
match: connection.model('match', MatchSchema),
auction: connection.model('auction', AuctionSchema),
auctionHistory: connection.model(
'auctionHistory',
AuctionHistoryEntrySchema,
),
auctionSideEffectApplication: connection.model(
'auctionSideEffectApplication',
AuctionSideEffectApplicationSchema,
),
tournamentCode: connection.model('tournamentCode', TournamentCodeSchema),
auctionBidRecord: connection.model(
'auctionBidRecord',
AuctionBidRecordSchema,
),
};
await Promise.all(
Object.values(models).map((model) => model.createCollection()),
);
});
beforeEach(async () => {
await Promise.all(Object.values(models).map((model) => model.deleteMany({})));
});
afterEach(() => jest.restoreAllMocks());
afterAll(async () => {
await connection?.close();
});
async function fixture(reserved: boolean) {
const matchId = new mongoose.Types.ObjectId(),
communityId = new mongoose.Types.ObjectId(),
auctionId = new mongoose.Types.ObjectId(),
codeId = new mongoose.Types.ObjectId();
await models.match.create({
_id: matchId,
communityId,
title: 'Isolated auction test',
scheduledAt: new Date(),
status: 'team-built',
teamBuiltAt: new Date(),
teams: [],
participants: [],
tournamentCodeIds: reserved ? [codeId] : [],
gameSchedule: [],
__v: 0,
});
await models.auction.collection.insertOne({
_id: auctionId,
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: 'life',
phase: 'complete',
revision: 10,
matchSideEffectRevision: 0,
matchSideEffectStatus: 'none',
});
if (reserved)
await models.tournamentCode.collection.insertOne({
_id: codeId,
code: 'ISOLATED-TEST',
state: 'reserved',
matchId,
});
const database = {
getModel: async (name: string) => models[name],
createSession: () => connection.startSession(),
};
const codes = new TournamentCodeService(database as never, redis as never);
const results = new TournamentResultService(
database as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
);
const service = new AuctionService(
database as never,
{} as never,
codes,
results,
redis as never,
{} as never,
{} as never,
);
return {
matchId,
communityId,
auctionId,
codeId,
service,
database,
codes,
results,
};
}
it.each([false, true])(
'cancels the auction atomically with reserved codes=%s',
async (reserved) => {
const f = await fixture(reserved);
await f.service.reset({
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
});
const match = await models.match.findById(f.matchId).lean();
expect(match).toMatchObject({
status: 'recruiting',
teams: [],
tournamentCodeIds: [],
gameSchedule: [],
});
expect(match?.teamBuiltAt).toBeUndefined();
expect(await models.auction.findById(f.auctionId)).toBeNull();
if (reserved)
expect(
(await models.tournamentCode.findById(f.codeId).lean())?.state,
).toBe('released');
},
);
it('rolls back code release and Match changes if cancellation later fails', async () => {
const f = await fixture(true);
jest.spyOn(models.auctionHistory, 'deleteMany').mockImplementation(() => {
throw new Error('forced history failure');
});
await expect(
f.service.reset({
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
}),
).rejects.toThrow('forced history failure');
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
'reserved',
);
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
status: 'team-built',
__v: 0,
tournamentCodeIds: [f.codeId],
});
expect(await models.auction.findById(f.auctionId)).not.toBeNull();
});
it('retries cancellation cleanup from Mongo after Redis fails, without another cancellation request', async () => {
const f = await fixture(true);
const cleanup = jest
.spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle')
.mockRejectedValueOnce(new Error('Redis offline'))
.mockResolvedValue(undefined);
await f.service.reset({
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
});
expect(cleanup).not.toHaveBeenCalled();
await f.service.retryPendingCancellations();
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
auctionCancellationCleanupPending: true,
auctionCancellationCleanupAttempts: 1,
});
await models.match.updateOne(
{ _id: f.matchId },
{ $set: { auctionCancellationCleanupRetryAt: new Date(0) } },
);
await f.service.retryPendingCancellations();
expect(cleanup).toHaveBeenCalledTimes(2);
expect(
(await models.match.findById(f.matchId).lean())
?.auctionCancellationCleanupPending,
).toBe(false);
});
it('keeps all 1005 Mongo bid records and idempotently retries when Redis acknowledgement fails', async () => {
const f = await fixture(false);
const events = Array.from({ length: 1005 }, (_, i) => ({
_id: `${f.matchId.toString()}:life:${String(i + 1).padStart(16, '0')}`,
matchId: f.matchId.toString(),
auctionLifecycleId: 'life',
revision: i + 1,
bidRequestId: `bid:${i}`,
bidRoundId: 'round',
bidderTeamIndex: 1,
bidPointAmount: i + 1,
bidderUserId: new mongoose.Types.ObjectId().toString(),
playerUserId: new mongoose.Types.ObjectId().toString(),
bidPlacedAtTimestampMS: Date.now(),
}));
let failAck = true;
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return events.map((e) => e._id);
if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e));
if (failAck) throw new Error('Redis acknowledgement failed');
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
await expect(archive.persistPendingBids(2000)).rejects.toThrow(
'acknowledgement failed',
);
expect(await models.auctionBidRecord.countDocuments()).toBe(1005);
failAck = false;
await expect(archive.persistPendingBids(2000)).resolves.toBe(1005);
expect(await models.auctionBidRecord.countDocuments()).toBe(1005);
const page = await archive.list(f.matchId, 1000, 100);
expect(page.items.map((item) => item.revision)).toEqual([
1001, 1002, 1003, 1004, 1005,
]);
expect(page.hasMore).toBe(false);
});
it('migrates the cleanup index idempotently and examines only due pending matches', async () => {
const f = await fixture(false);
const documents = Array.from({ length: 10000 }, (_, i) => ({
_id: new mongoose.Types.ObjectId(),
lastCancelledAuctionLifecycleId: `life-${i}`,
...(i < 5
? {
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: new Date(0),
}
: i === 5
? {
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: new Date(Date.now() + 3600000),
}
: {}),
}));
await models.match.collection.insertMany(documents);
const find = jest.spyOn(models.match, 'find');
const cleanup = jest
.spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle')
.mockResolvedValue(undefined);
await f.service.retryPendingCancellations();
expect(cleanup).toHaveBeenCalledTimes(5);
const query = find.mock.calls[0][0] as unknown as Record<string, unknown>;
// Restore only the five due jobs so explain measures the actual worker query.
await models.match.updateMany(
{ _id: { $in: documents.slice(0, 5).map((d) => d._id) } },
{
$set: {
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: new Date(0),
},
},
);
const explain = () =>
models.match.collection
.find(query)
.sort({ auctionCancellationCleanupRetryAt: 1 })
.limit(20)
.explain('executionStats');
expect(
(await explain()).executionStats.totalDocsExamined,
).toBeGreaterThanOrEqual(10000);
expect(await migrateAuctionCancellationIndex(connection.db!)).toMatchObject({
dryRun: true,
alreadyPresent: false,
});
expect(
(await models.match.collection.listIndexes().toArray()).some(
(i) => i.name === 'auction_cancellation_cleanup_due',
),
).toBe(false);
await migrateAuctionCancellationIndex(connection.db!, { dryRun: false });
expect(
await migrateAuctionCancellationIndex(connection.db!, { dryRun: false }),
).toMatchObject({ alreadyPresent: true });
const plan = await explain();
expect(plan.executionStats.nReturned).toBe(5);
expect(plan.executionStats.totalDocsExamined).toBe(5);
expect(JSON.stringify(plan.queryPlanner.winningPlan)).toContain('IXSCAN');
});
function archiveEvent(matchId: mongoose.Types.ObjectId, revision: number) {
return {
_id: `${matchId.toString()}:life:${String(revision).padStart(16, '0')}`,
matchId: matchId.toString(),
auctionLifecycleId: 'life',
revision,
bidRequestId: `bid-${revision}`,
bidRoundId: 'round',
bidderUserId: new mongoose.Types.ObjectId().toString(),
playerUserId: new mongoose.Types.ObjectId().toString(),
bidderTeamIndex: 1,
bidPointAmount: revision,
bidPlacedAtTimestampMS: Date.now(),
};
}
it.each(['', undefined, 'round'])(
'persists and lists bids with optional round ID %s without duplicating retries',
async (bidRoundId) => {
const f = await fixture(false);
const event = { ...archiveEvent(f.matchId, 1), bidRoundId };
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return [event._id];
if (command[0] === 'HMGET') return [JSON.stringify(event)];
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
await expect(archive.persistPendingBids()).resolves.toBe(1);
await expect(archive.persistPendingBids()).resolves.toBe(1);
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
const page = await archive.list(f.matchId);
expect(page.items).toHaveLength(1);
expect(String(page.items[0].playerUserId)).toBe(event.playerUserId);
if (bidRoundId) expect(page.items[0].bidRoundId).toBe(bidRoundId);
else expect(page.items[0]).not.toHaveProperty('bidRoundId');
expect(
sendCommand.mock.calls
.filter(([command]) => command[0] === 'EVAL')
.every(([command]) => command[2] === '2'),
).toBe(true);
},
);
it.each([
'json',
'revision',
'object-id',
'required-field',
'identity',
'missing-payload',
])(
'quarantines an invalid %s without losing the valid Mongo bid',
async (kind) => {
const f = await fixture(false);
const bad = archiveEvent(f.matchId, 1),
good = archiveEvent(f.matchId, 2);
const invalid: Record<string, unknown> = { ...bad };
if (kind === 'revision') invalid.revision = -1;
if (kind === 'object-id') invalid.bidderUserId = 'not-an-object-id';
if (kind === 'required-field') delete invalid.bidderUserId;
if (kind === 'identity')
invalid.matchId = new mongoose.Types.ObjectId().toString();
const raw =
kind === 'json'
? '{broken'
: kind === 'missing-payload'
? null
: JSON.stringify(invalid);
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return [bad._id, good._id];
if (command[0] === 'HMGET') return [raw, JSON.stringify(good)];
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
await expect(archive.persistPendingBids()).resolves.toBe(1);
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
expect(
await models.auctionBidRecord.findById(good._id).lean(),
).toMatchObject({ revision: 2 });
expect(await models.auctionBidRecord.findById(bad._id)).toBeNull();
const quarantine = sendCommand.mock.calls
.map((c) => c[0])
.find((c) => c[0] === 'EVAL' && c[2] === '4')!;
expect(JSON.parse(quarantine[11])).toMatchObject({
id: bad._id,
rawPayload: raw,
error: expect.any(String),
});
const ack = sendCommand.mock.calls
.map((c) => c[0])
.find((c) => c[0] === 'EVAL' && c[2] === '2')!;
expect(ack.slice(5)).toEqual([good._id, JSON.stringify(good)]);
},
);
it.each(['partial-write-error', 'partial-acknowledgement'])(
'retains the batch after %s and retries Mongo upserts without duplicates',
async (failure) => {
const f = await fixture(false);
const events = [archiveEvent(f.matchId, 1), archiveEvent(f.matchId, 2)];
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return events.map((e) => e._id);
if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e));
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
const write = models.auctionBidRecord.bulkWrite.bind(
models.auctionBidRecord,
);
jest
.spyOn(models.auctionBidRecord, 'bulkWrite')
.mockImplementationOnce(async (operations, options) => {
const result = await write(operations.slice(0, 1), options);
if (failure === 'partial-write-error')
throw new Error('Lost Mongo acknowledgement');
return result;
});
await expect(archive.persistPendingBids()).rejects.toThrow();
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
expect(
sendCommand.mock.calls.filter(([c]) => c[0] === 'EVAL'),
).toHaveLength(0);
await expect(archive.persistPendingBids()).resolves.toBe(2);
expect(await models.auctionBidRecord.countDocuments()).toBe(2);
},
);
it.each(['Mongo', 'Redis mirroring'])(
'reconciles a terminal status after %s recovers',
async (outage) => {
const f = await fixture(false);
const payload = {
auctionId: f.auctionId.toString(),
matchId: f.matchId.toString(),
auctionLifecycleId: 'life',
auctionStateTransitionId: 'complete-10',
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
afterAuctionMongoProjection: { matchSideEffectRevision: 1 },
};
await models.auction.updateOne(
{ _id: f.auctionId },
{
$set: {
matchSideEffectTransitionId: 'complete-10',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
},
);
let unavailable = outage === 'Mongo';
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGEBYSCORE') return ['dead-letter'];
if (command[0] === 'HGETALL')
return { payloadJson: JSON.stringify(payload) };
if (command[0] === 'HGET') return 'retry limit exceeded';
return 1;
});
const mirror = jest.fn().mockResolvedValue(true),
broadcast = jest.fn();
if (outage === 'Redis mirroring')
mirror.mockRejectedValueOnce(new Error('Redis unavailable'));
const scheduler = new AuctionDurabilityScheduler(
{
getModel: async (name: string) => {
if (unavailable) throw new Error('Mongo offline');
return models[name];
},
} as never,
{
raw: () => ({ sendCommand }),
getCurrentRedisTimestampMS: async () => 1000,
} as never,
undefined,
undefined,
{ mirrorMatchSideEffectStatus: mirror } as never,
{ broadcastMatchSideEffect: broadcast } as never,
);
await scheduler.reconcileDeadLetterStatuses();
expect(sendCommand).toHaveBeenCalledWith([
'ZADD',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
'6000',
'dead-letter',
]);
unavailable = false;
await scheduler.reconcileDeadLetterStatuses();
expect(
(await models.auction.findById(f.auctionId).lean())
?.matchSideEffectStatus,
).toBe('failed');
expect(mirror).toHaveBeenCalledWith(
expect.objectContaining({
auctionStateTransitionId: 'complete-10',
matchSideEffectStatus: 'failed',
}),
);
expect(broadcast).toHaveBeenCalledTimes(1);
// Replaying the old notification must not overwrite a new generation.
await models.auction.updateOne(
{ _id: f.auctionId },
{
$set: {
matchSideEffectTransitionId: 'new',
matchSideEffectStatus: 'processing',
matchSideEffectRevision: 20,
},
},
);
await scheduler.reconcileDeadLetterStatuses();
expect(
(await models.auction.findById(f.auctionId).lean())
?.matchSideEffectStatus,
).toBe('processing');
expect(broadcast).toHaveBeenCalledTimes(1);
},
);
it('cleans up reserved codes and Match metadata after undoing completion', async () => {
const f = await fixture(true);
const scheduler = new AuctionDurabilityScheduler(
f.database as never,
redis as never,
f.codes,
f.results,
undefined,
undefined,
{
readAuctionStateView: async () => ({
auctionLifecycleId: 'life',
phase: 'bidding',
matchSideEffectTransitionId: 'undo',
}),
} as never,
);
await scheduler.applyDurabilityPayload({
auctionId: f.auctionId,
matchId: f.matchId,
auctionLifecycleId: 'life',
revision: 11,
auctionHistorySequence: 11,
auctionActionType: 'undo',
auctionStateTransitionId: 'undo',
snapshotBefore: {},
historyMutation: { type: 'append' },
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
afterAuctionMongoProjection: {
phase: 'bidding',
revision: 11,
matchSideEffectTransitionId: 'undo',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
});
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
status: 'team-built',
tournamentCodeIds: [],
gameSchedule: [],
teams: [],
});
expect(
(await models.match.findById(f.matchId).lean())?.teamBuiltAt,
).toBeUndefined();
expect(
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
).toBe('completed');
});
it('reads an uncommitted Match in the provided session when cancelling polling by id', async () => {
const f = await fixture(false);
const session = await connection.startSession();
try {
session.startTransaction();
const [created] = await models.match.create(
[
{
communityId: f.communityId,
title: 'Uncommitted',
scheduledAt: new Date(),
},
],
{ session },
);
expect(await models.match.findById(created._id)).toBeNull();
await f.results.cancelPolling(
created._id as mongoose.Types.ObjectId,
session,
);
expect(
await models.match.findById(created._id).session(session),
).not.toBeNull();
await session.abortTransaction();
} finally {
await session.endSession();
}
});
it('rolls back failed Undo cleanup and successfully retries with fresh Match versions', async () => {
const f = await fixture(true);
const scheduler = new AuctionDurabilityScheduler(
f.database as never,
redis as never,
f.codes,
f.results,
undefined,
undefined,
{
readAuctionStateView: async () => ({
auctionLifecycleId: 'life',
phase: 'bidding',
matchSideEffectTransitionId: 'undo-retry',
}),
} as never,
);
const payload = {
auctionId: f.auctionId,
matchId: f.matchId,
auctionLifecycleId: 'life',
revision: 11,
auctionHistorySequence: 11,
auctionActionType: 'undo',
auctionStateTransitionId: 'undo-retry',
snapshotBefore: {},
historyMutation: { type: 'append' },
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
afterAuctionMongoProjection: {
phase: 'bidding',
revision: 11,
matchSideEffectTransitionId: 'undo-retry',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
};
jest
.spyOn(f.results, 'cancelPolling')
.mockRejectedValueOnce(new Error('temporary poll cleanup failure'));
await expect(scheduler.applyDurabilityPayload(payload)).rejects.toThrow(
'temporary poll cleanup failure',
);
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
'reserved',
);
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
__v: 0,
tournamentCodeIds: [f.codeId],
});
await scheduler.applyDurabilityPayload(payload);
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
'released',
);
expect(
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
).toBe('completed');
});
it('answers a retransmitted cancellation of the same lifecycle instead of 404', async () => {
const f = await fixture(true);
const request = {
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
};
await f.service.reset(request);
await expect(f.service.reset(request)).resolves.toBeUndefined();
expect(
(await models.match.findById(f.matchId).lean())
?.lastCancelledAuctionLifecycleId,
).toBe('life');
await expect(
f.service.reset({ ...request, clientObservedAuctionLifecycleId: 'other' }),
).rejects.toThrow('Auction not found.');
});
it.each([
'new-lifecycle',
'new-transition',
'completed',
'revision-ahead',
'mirror-superseded',
])('fences terminal notifications against %s', async (scenario) => {
const f = await fixture(false);
await models.auction.updateOne(
{ _id: f.auctionId },
{
$set: {
matchSideEffectTransitionId: 'target',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
},
);
const payload = {
auctionId: f.auctionId.toString(),
matchId: f.matchId.toString(),
auctionLifecycleId: 'life',
auctionStateTransitionId: 'target',
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
afterAuctionMongoProjection: { matchSideEffectRevision: 1 },
};
const state = {
auctionLifecycleId: scenario === 'new-lifecycle' ? 'new-life' : 'life',
matchSideEffectTransitionId:
scenario === 'new-transition' ? 'new' : 'target',
matchSideEffectStatus: scenario === 'completed' ? 'completed' : 'pending',
matchSideEffectRevision: 40,
};
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGEBYSCORE') return ['terminal'];
if (command[0] === 'HGETALL')
return { payloadJson: JSON.stringify(payload) };
if (command[0] === 'HGET') return 'exhausted';
return 1;
});
const broadcast = jest.fn();
const mirror = jest.fn().mockResolvedValue(scenario !== 'mirror-superseded');
const scheduler = new AuctionDurabilityScheduler(
f.database as never,
{
raw: () => ({ sendCommand }),
getCurrentRedisTimestampMS: async () => 1000,
} as never,
undefined,
undefined,
{ mirrorMatchSideEffectStatus: mirror } as never,
{ broadcastMatchSideEffect: broadcast } as never,
{ readAuctionStateView: async () => state } as never,
);
await scheduler.reconcileDeadLetterStatuses();
const changed = ['revision-ahead', 'mirror-superseded'].includes(scenario);
expect(
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
).toBe(changed ? 'failed' : 'pending');
if (changed)
expect(mirror).toHaveBeenCalledWith(
expect.objectContaining({ matchSideEffectRevision: 41 }),
);
expect(broadcast).toHaveBeenCalledTimes(
scenario === 'revision-ahead' ? 1 : 0,
);
});
@@ -0,0 +1,598 @@
import { createClient } from 'redis';
import mongoose from 'mongoose';
import { AuctionService } from '../src/modules/auction/auction.service';
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service';
import { AuctionRedisStateReaderService } from '../src/modules/auction/redis/auction-redis-state-reader.service';
import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys';
import {
AuctionLifecycleLock,
AuctionLifecycleBusyError,
} from '../src/modules/auction/auction-lifecycle-lock';
import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler';
import { AuctionSideEffectService } from '../src/modules/auction/auction-side-effect.service';
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
import {
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
} from '../src/modules/auction/redis/auction-redis.keys';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new AuctionRedisService(client);
const commands = new AuctionRedisCommandService(redis);
const reader = new AuctionRedisStateReaderService(redis);
beforeAll(async () => {
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) {
throw new Error('Use test:redis to start an isolated local Redis server.');
}
client.on('error', () => {});
await client.connect();
});
beforeEach(async () => {
await client.flushDb();
});
afterAll(async () => {
if (client.isOpen) await client.quit();
});
it.each(['count', 'elapsed'])(
'dead-letters permanently pending side effects by %s while retaining the payload',
async (limit) => {
const key = 'review:pending';
const now = Date.now();
await client.hSet(key, {
payloadJson: '{}',
pendingAttempts: limit === 'count' ? '119' : '1',
pendingSinceTimestampMS: String(limit === 'elapsed' ? now - 600000 : now),
});
await client.zAdd(AUCTION_DURABILITY_RETRY_QUEUE_KEY, {
score: now,
value: key,
});
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
deferDurabilityPayload(
key: string,
time: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.deferDurabilityPayload(
key,
now,
new Error('state unavailable'),
),
).toBe('dead-lettered');
expect(
await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key),
).toBeNull();
expect(
await client.zScore(AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, key),
).toBe(now);
expect(await client.hGet(key, 'payloadJson')).toBe('{}');
expect(await client.hGet(key, 'attempts')).toBeNull();
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
now,
);
},
);
it('defers a temporary coordination wait without consuming failed-work attempts or recreating removed payloads', async () => {
const key = 'review:pending';
const now = Date.now();
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
deferDurabilityPayload(
key: string,
time: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.deferDurabilityPayload(key, now, new Error('busy')),
).toBe('gone');
expect(await client.exists(key)).toBe(0);
await client.hSet(key, 'payloadJson', '{}');
expect(
await internal.deferDurabilityPayload(key, now, new Error('busy')),
).toBe('requeued');
expect(await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key)).toBe(
now + 5000,
);
expect(await client.hGet(key, 'attempts')).toBeNull();
});
async function fixture() {
const matchId = new mongoose.Types.ObjectId();
const id = matchId.toString();
const lifecycle = `life:${id}`;
const playerA = new mongoose.Types.ObjectId().toString();
const playerB = new mongoose.Types.ObjectId().toString();
const captain = new mongoose.Types.ObjectId().toString();
const initialRound = `initial:${id}`;
const structure = {
auctionPlayerOrder: [{ userId: playerA }, { userId: playerB }],
auctionTeams: [
{
teamIndex: 1,
captainUserId: captain,
remainingBidPointAmount: 100000,
members: [],
},
],
currentAuctionPlayerOrderIndex: 0,
currentAuctionRound: 1,
passedAuctionPlayerQueue: [],
maximumBidRoundDurationSeconds: 30,
bidRoundExtensionSeconds: 5,
initialTeamBidPointAmount: 100000,
topTierAuctionPriorityThreshold: 'NONE',
manualTopTierAuctionPriorityUserIds: [],
};
await commands.hydrateState(id, {
matchId: id,
auctionLifecycleId: lifecycle,
revision: 0,
auctionHistorySequence: 1,
phase: 'bidding',
maximumBidRoundDurationSeconds: 30,
structureStateJson: JSON.stringify(structure),
liveStatePatch: {
currentAuctionPlayerOrderIndex: 0,
currentBidRoundPlayerUserId: playerA,
currentBidRoundId: initialRound,
maximumBidRoundDurationSeconds: 30,
bidRoundExtensionSeconds: 5,
'team:1:captainUserId': captain,
'team:1:remainingBidPointAmount': 100000,
'team:1:assignedAuctionTeamMemberCount': 1,
},
});
const auction = {
_id: new mongoose.Types.ObjectId(),
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: lifecycle,
phase: 'bidding',
revision: 1,
};
const service = new AuctionService(
{ getModel: async () => ({ findOne: async () => auction }) } as never,
{} as never,
{} as never,
{} as never,
redis,
commands,
reader,
);
const bid = (
requestId: string,
amount: number,
player = playerA,
round = initialRound,
) => ({
matchId: id,
bidderTeamIndex: 1,
bidderUserId: captain,
bidPointAmount: amount,
bidRequestId: requestId,
clientObservedAuctionLifecycleId: lifecycle,
clientObservedCurrentBidRoundPlayerUserId: player,
clientObservedBidRoundId: round,
maximumAuctionTeamMemberCount: 5,
minimumBidPointReservePerRemainingAuctionTeamMemberSlot: 10,
maximumStoredAuctionBidHistoryCount: 1000,
maximumStoredBidRequestResultCount: 500,
minimumBidRequestResultReplayWindowMS: 300000,
});
const undo = async (snapshot: Record<string, unknown>) => {
const internal = service as unknown as {
undoRedisAuction(params: unknown): Promise<unknown>;
};
return internal.undoRedisAuction({
auction,
snapshot,
targetHistorySequence: 1,
undoingCompletedAuction: false,
});
};
return {
id,
matchId,
lifecycle,
playerA,
playerB,
initialRound,
auction,
service,
bid,
undo,
keys: buildAuctionRedisKeys(id),
};
}
it('waits briefly for a busy recovery lock and returns the recovered state', async () => {
const f = await fixture();
f.auction.revision = 10;
const key = `auction:${f.id}:lifecycleMutationLock`;
await redis.acquireTokenLock(key, 'other-reader', 30000);
const released = new Promise((resolve, reject) => {
setTimeout(() => {
void redis.releaseTokenLock(key, 'other-reader').then(resolve, reject);
}, 100);
});
const [view] = await Promise.all([
f.service.getStateByMatchId(f.matchId),
released,
]);
expect(view).toMatchObject({ auctionLifecycleId: f.lifecycle, revision: 11 });
});
it('bounds recovery waits and reports a retriable state error without releasing another owner lock', async () => {
const f = await fixture();
f.auction.revision = 10;
const key = `auction:${f.id}:lifecycleMutationLock`;
await redis.acquireTokenLock(key, 'slow-owner', 30000);
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toMatchObject({
response: { code: 'AUCTION_STATE_RECOVERING', retryAfterMS: 500 },
});
expect(await client.get(key)).toBe('slow-owner');
});
it('keeps accepting bids after 1000 history entries and preserves accepted request replay across players', async () => {
const f = await fixture();
const first = await commands.placeBid(
f.id,
f.bid('accepted-before-next', 10),
);
expect(first).toMatchObject({ ok: true });
await client.rPush(
f.keys.bidHistoryKey,
Array.from({ length: 999 }, () => JSON.stringify({ bidPointAmount: 10 })),
);
await client.hSet(f.keys.liveStateKey, 'phase', 'sold');
await f.service.next(f.matchId);
await f.service.openBidding(f.matchId);
const state = (await reader.readAuctionStateView(f.id))!;
const result = await commands.placeBid(
f.id,
f.bid('next-player', 10, f.playerB, String(state.currentBidRoundId)),
);
expect(result).toMatchObject({ ok: true });
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
expect(
await commands.placeBid(f.id, f.bid('accepted-before-next', 10)),
).toMatchObject({ ...first, isReplay: true });
});
it('500 rejected requests do not block a valid bid, and rejection retention stays bounded', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'currentHighestBidPointAmount', '10');
for (let i = 0; i < 501; i++) {
expect(
await commands.placeBid(f.id, f.bid(`too-low:${i}`, 5)),
).toMatchObject({ rejected: true, code: 'TOO_LOW' });
}
expect(await client.lLen(f.keys.rejectedBidRequestResultOrderKey)).toBe(500);
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(0);
expect(await commands.placeBid(f.id, f.bid('valid', 20))).toMatchObject({
ok: true,
});
expect(await commands.placeBid(f.id, f.bid('valid', 30))).toMatchObject({
rejected: true,
code: 'DUPLICATE_REQUEST_CONFLICT',
});
});
it('still protects 500 accepted results within their replay window', async () => {
const f = await fixture();
for (let i = 1; i <= 500; i++)
expect(
await commands.placeBid(f.id, f.bid(`accepted:${i}`, i)),
).toMatchObject({ ok: true });
expect(await commands.placeBid(f.id, f.bid('overflow', 501))).toMatchObject({
rejected: true,
code: 'BID_REQUEST_RESULT_LIMIT',
});
expect(await commands.placeBid(f.id, f.bid('accepted:1', 1))).toMatchObject({
ok: true,
isReplay: true,
});
});
it('migrates rejected results from the former shared quota without losing accepted replays', async () => {
const f = await fixture();
await commands.placeBid(f.id, f.bid('accepted', 10));
await client.hDel(f.keys.bidRequestResultsKey, '_acceptedOnlyOrderV1');
for (let i = 0; i < 499; i++) {
const id = `legacy-rejected:${i}`;
await client.hSet(f.keys.bidRequestResultsKey, {
[`${id}:status`]: 'rejected',
[`${id}:createdAtTimestampMS`]: String(Date.now()),
});
await client.rPush(f.keys.bidRequestResultOrderKey, id);
}
expect(
await commands.placeBid(f.id, f.bid('next-accepted', 20)),
).toMatchObject({ ok: true });
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(2);
expect(await commands.placeBid(f.id, f.bid('accepted', 10))).toMatchObject({
ok: true,
isReplay: true,
});
});
it('encodes opaque round identities safely in bid history', async () => {
const f = await fixture();
const round = 'round:with"quotes\\and-slashes';
await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', round);
expect(
await commands.placeBid(f.id, f.bid('quoted-round', 10, f.playerA, round)),
).toMatchObject({ ok: true, auctionBidHistoryEntry: { bidRoundId: round } });
expect((await reader.readAuctionStateView(f.id))?.auctionBidHistory).toEqual([
expect.objectContaining({ bidRoundId: round }),
]);
});
it('rejects delayed pre-undo bids and omitted round identities while accepting the restored round', async () => {
const f = await fixture();
const snapshot = (await reader.readAuctionStateView(f.id))!;
await f.undo(snapshot);
const restored = (await reader.readAuctionStateView(f.id))!;
expect(restored.currentBidRoundId).not.toBe(f.initialRound);
expect(await commands.placeBid(f.id, f.bid('old-round', 100))).toMatchObject({
code: 'STALE_BID_ROUND',
});
expect(
await commands.placeBid(f.id, {
...f.bid('missing-round', 100),
clientObservedBidRoundId: undefined,
}),
).toMatchObject({ code: 'BID_CLIENT_UPDATE_REQUIRED' });
expect(
await commands.placeBid(
f.id,
f.bid('new-round', 100, f.playerA, String(restored.currentBidRoundId)),
),
).toMatchObject({
ok: true,
auctionBidHistoryEntry: {
playerUserId: f.playerA,
bidRoundId: restored.currentBidRoundId,
},
});
});
it('removes completion TTLs when resuming an auction', async () => {
const f = await fixture();
const snapshot = (await reader.readAuctionStateView(f.id))!;
await client.expire(f.keys.liveStateKey, 604800);
await client.expire(f.keys.structureStateKey, 604800);
await f.undo(snapshot);
expect(await client.ttl(f.keys.liveStateKey)).toBe(-1);
expect(await client.ttl(f.keys.structureStateKey)).toBe(-1);
});
it('fences direct foreign-lifecycle hydration and repairs an orphan using current Mongo authority', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'revision', '150');
expect(
await commands.hydrateState(f.id, {
matchId: f.id,
auctionLifecycleId: 'replacement',
revision: 0,
auctionHistorySequence: 1,
phase: 'presenting',
maximumBidRoundDurationSeconds: 30,
structureStateJson: '{}',
}),
).toMatchObject({ rejected: true, code: 'LIFECYCLE_CONFLICT' });
const current = {
...f.auction,
auctionLifecycleId: 'replacement',
phase: 'presenting',
revision: 1,
auctionTeams: [],
auctionPlayerOrder: [],
};
jest.spyOn(f.service, 'getByMatchId').mockResolvedValue(current as never);
expect(await f.service.getStateByMatchId(f.matchId)).toMatchObject({
auctionLifecycleId: 'replacement',
phase: 'presenting',
});
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
'replacement',
);
});
it('does not let an old Mongo read overwrite the newly started lifecycle', async () => {
const f = await fixture();
const stale = { ...f.auction, auctionLifecycleId: 'obsolete', revision: 500 };
jest
.spyOn(f.service, 'getByMatchId')
.mockResolvedValueOnce(stale as never)
.mockResolvedValue(f.auction as never);
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toThrow(
'lifecycle has changed',
);
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
f.lifecycle,
);
});
it('serializes lifecycle mutations without deleting another owner lock', async () => {
const lock = new AuctionLifecycleLock(redis);
await lock.run('match', async () => {
await expect(lock.run('match', async () => {})).rejects.toThrow(
AuctionLifecycleBusyError,
);
});
await expect(lock.run('match', async () => 'done')).resolves.toBe('done');
});
it('persists the exact Redis deadline when opening bidding and restoring a bidding snapshot', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'phase', 'presenting');
await client.hDel(f.keys.liveStateKey, 'currentBidRoundDeadlineTimestampMS');
await f.service.openBidding(f.matchId);
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const verify = async () => {
const live = (await reader.readAuctionStateView(f.id))!;
const queued = await client.zRange(
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
0,
-1,
);
const payloads = await Promise.all(
queued.map((key) => scheduler.readDurabilityPayload(key)),
);
const payload = payloads.find(
(entry) => entry?.revision === live.revision,
)!;
expect(
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS,
).toBe(live.currentBidRoundDeadlineTimestampMS);
expect(
new Date(
String(payload.afterAuctionMongoProjection.currentDeadline),
).getTime(),
).toBe(live.currentBidRoundDeadlineTimestampMS);
expect(
payload.afterAuctionMongoProjection.passedAuctionPlayerQueue,
).toEqual([]);
return live;
};
const snapshot = await verify();
await f.undo({ ...snapshot, currentBidRoundDeadlineTimestampMS: 1 });
await verify();
});
it('archives every accepted bid beyond the recent 1000, with no duplicate archive event for a replay', async () => {
const f = await fixture();
for (let amount = 1; amount <= 1005; amount++) {
expect(
await commands.placeBid(f.id, {
...f.bid(`archive:${amount}`, amount),
maximumStoredBidRequestResultCount: 2000,
}),
).toMatchObject({ ok: true });
}
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
expect(
await commands.placeBid(f.id, {
...f.bid('archive:1', 1),
maximumStoredBidRequestResultCount: 2000,
}),
).toMatchObject({ ok: true, isReplay: true });
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
const [first] = await client.zRange(AUCTION_BID_ARCHIVE_QUEUE_KEY, 0, 0);
expect(
JSON.parse((await client.hGet(AUCTION_BID_ARCHIVE_PAYLOADS_KEY, first))!),
).toMatchObject({
bidPointAmount: 1,
bidRequestId: 'archive:1',
playerUserId: f.playerA,
auctionLifecycleId: f.lifecycle,
});
});
it.each(['missing', 'empty'])(
'archives an accepted pre-upgrade bid with a %s Redis round ID',
async (kind) => {
const f = await fixture();
if (kind === 'missing')
await client.hDel(f.keys.liveStateKey, 'currentBidRoundId');
else await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', '');
expect(
await commands.placeBid(f.id, {
...f.bid('legacy-round', 10),
clientObservedBidRoundId: undefined,
}),
).toMatchObject({
ok: true,
auctionBidHistoryEntry: { bidRoundId: '', playerUserId: f.playerA },
});
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
matchedCount: 0,
upsertedCount: operations.length,
}));
const archive = new AuctionBidArchiveService(
{ getModel: async () => ({ bulkWrite }) } as never,
redis,
);
await expect(archive.persistPendingBids()).resolves.toBe(1);
const write = bulkWrite.mock.calls[0][0][0] as {
updateOne: { update: { $setOnInsert: Record<string, unknown> } };
};
expect(write.updateOne.update.$setOnInsert).not.toHaveProperty(
'bidRoundId',
);
expect(String(write.updateOne.update.$setOnInsert.playerUserId)).toBe(
f.playerA,
);
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(0);
},
);
it('does not mirror an old generation failure over a newer Redis side effect', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, {
matchSideEffectTransitionId: 'new',
matchSideEffectRevision: '2',
matchSideEffectStatus: 'pending',
});
const sideEffects = new AuctionSideEffectService(redis);
expect(
await sideEffects.mirrorMatchSideEffectStatus({
matchId: f.id,
auctionLifecycleId: f.lifecycle,
auctionStateTransitionId: 'old',
matchSideEffectRevision: 100,
matchSideEffectStatus: 'failed',
}),
).toBe(false);
expect(await client.hGet(f.keys.liveStateKey, 'matchSideEffectStatus')).toBe(
'pending',
);
});
it('queues a terminal notification for ordinary failures and backfills old dead letters', async () => {
const key = 'review:ordinary-failure';
await client.hSet(key, { payloadJson: '{}', attempts: '9' });
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
recordDurabilityPayloadFailure(
key: string,
now: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.recordDurabilityPayloadFailure(
key,
1000,
new Error('failed before worker'),
),
).toBe('dead-lettered');
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
1000,
);
// Simulate an older deployment which did not create notification jobs.
await client.zRem(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key);
await scheduler.reconcileDeadLetterStatuses();
expect(
await client.hGet(key, 'deadLetterStatusReconciledAtTimestampMS'),
).not.toBeNull();
expect(
await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key),
).toBeNull();
});
+7
View File
@@ -0,0 +1,7 @@
{
"rootDir": "..",
"testEnvironment": "node",
"testRegex": "test/.*\\.mongo-spec\\.ts$",
"transform": { "^.+\\.ts$": "ts-jest" },
"testTimeout": 15000
}
+134
View File
@@ -0,0 +1,134 @@
import { spawn } from 'node:child_process';
import { createRequire } from 'node:module';
import { mkdtemp, mkdir, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, relative, resolve } from 'node:path';
import { createServer } from 'node:net';
import { setTimeout as delay } from 'node:timers/promises';
import mongoose from 'mongoose';
const require = createRequire(import.meta.url);
const listener = createServer();
await new Promise((accept) => listener.listen(0, '127.0.0.1', accept));
const port = listener.address().port;
await new Promise((accept) => listener.close(accept));
const directory = await mkdtemp(join(tmpdir(), 'lolympus-mongo-test-'));
await mkdir(join(directory, 'data'));
const mongo = spawn(
process.env.MONGODB_TEST_SERVER || 'mongod',
[
'--bind_ip',
'127.0.0.1',
'--port',
String(port),
'--replSet',
'auction_review',
'--dbpath',
join(directory, 'data'),
'--logpath',
join(directory, 'mongod.log'),
'--wiredTigerCacheSizeGB',
'0.25',
],
{ windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] },
);
let startupError;
mongo.on('error', (error) => {
startupError = error;
});
const closed = new Promise((accept) => mongo.once('close', accept));
let output = '';
mongo.stdout.on('data', (chunk) => {
output += chunk;
});
mongo.stderr.on('data', (chunk) => {
output += chunk;
});
const uri = `mongodb://127.0.0.1:${port}/auction_review`;
const client = new mongoose.mongo.MongoClient(`${uri}?directConnection=true`, {
serverSelectionTimeoutMS: 500,
});
try {
let connected = false;
for (let attempt = 0; attempt < 60; attempt++) {
if (startupError) throw startupError;
if (mongo.exitCode !== null) throw new Error(`MongoDB exited: ${output}`);
try {
await client.connect();
connected = true;
break;
} catch {
await delay(100);
}
}
if (!connected) throw new Error(`MongoDB did not start: ${output}`);
await client
.db('admin')
.command({
replSetInitiate: {
_id: 'auction_review',
members: [{ _id: 0, host: `127.0.0.1:${port}` }],
},
});
let primary = false;
for (let attempt = 0; attempt < 100; attempt++) {
if ((await client.db('admin').command({ hello: 1 })).isWritablePrimary) {
primary = true;
break;
}
await delay(100);
}
if (!primary)
throw new Error('MongoDB replica set did not elect its primary.');
const build = await client.db('admin').command({ buildInfo: 1 });
console.log(
`Integration tests using isolated MongoDB ${build.version}, Mongoose ${mongoose.version}, loopback port ${port}`,
);
const jest = spawn(
process.execPath,
[
require.resolve('jest/bin/jest'),
'--config',
'test/jest-mongo.json',
'--runInBand',
...process.argv.slice(2),
],
{
cwd: resolve(import.meta.dirname, '..'),
windowsHide: true,
stdio: 'inherit',
env: {
...process.env,
NODE_ENV: 'test',
MONGO_TEST_ISOLATED: '1',
MONGO_TEST_URL: `${uri}?replicaSet=auction_review`,
},
},
);
process.exitCode = await new Promise((accept, reject) => {
jest.once('error', reject);
jest.once('exit', (code) => accept(code ?? 1));
});
} finally {
await client
.db('admin')
.command({ shutdown: 1, force: true })
.catch(() => {});
await client.close().catch(() => {});
if (mongo.pid && mongo.exitCode === null) mongo.kill();
await closed;
// Only remove the unique temporary directory created by this runner.
const target = relative(resolve(tmpdir()), resolve(directory));
if (
!target.startsWith('lolympus-mongo-test-') ||
target.includes('..') ||
/[/\\]/.test(target)
)
throw new Error('Unsafe MongoDB test directory.');
await rm(directory, {
recursive: true,
force: true,
maxRetries: 10,
retryDelay: 100,
});
}