Files
lolympus-tournament-support/apps/api/test/auction-correctness.integration-spec.ts
dahyeon f649490cd2 fix(auction): harden lifecycle and durable bid storage
Serialize lifecycle changes and make cancellation, deadlines, and match side effects recoverable and idempotent.

Archive accepted bids durably, quarantine invalid payloads, and preserve compatibility with empty round IDs. Synchronize HTTP polling and socket state with scoped resync events.

Add cancellation cleanup index migration and regression coverage. Validated with 744 API tests, 65 Redis integration tests, and 28 Mongo integration tests.
2026-11-09 23:24:11 +09:00

599 lines
20 KiB
TypeScript

import { createClient } from 'redis';
import mongoose from 'mongoose';
import { AuctionService } from '../src/modules/auction/auction.service';
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service';
import { AuctionRedisStateReaderService } from '../src/modules/auction/redis/auction-redis-state-reader.service';
import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys';
import {
AuctionLifecycleLock,
AuctionLifecycleBusyError,
} from '../src/modules/auction/auction-lifecycle-lock';
import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler';
import { AuctionSideEffectService } from '../src/modules/auction/auction-side-effect.service';
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
import {
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
} from '../src/modules/auction/redis/auction-redis.keys';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new AuctionRedisService(client);
const commands = new AuctionRedisCommandService(redis);
const reader = new AuctionRedisStateReaderService(redis);
beforeAll(async () => {
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) {
throw new Error('Use test:redis to start an isolated local Redis server.');
}
client.on('error', () => {});
await client.connect();
});
beforeEach(async () => {
await client.flushDb();
});
afterAll(async () => {
if (client.isOpen) await client.quit();
});
it.each(['count', 'elapsed'])(
'dead-letters permanently pending side effects by %s while retaining the payload',
async (limit) => {
const key = 'review:pending';
const now = Date.now();
await client.hSet(key, {
payloadJson: '{}',
pendingAttempts: limit === 'count' ? '119' : '1',
pendingSinceTimestampMS: String(limit === 'elapsed' ? now - 600000 : now),
});
await client.zAdd(AUCTION_DURABILITY_RETRY_QUEUE_KEY, {
score: now,
value: key,
});
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
deferDurabilityPayload(
key: string,
time: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.deferDurabilityPayload(
key,
now,
new Error('state unavailable'),
),
).toBe('dead-lettered');
expect(
await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key),
).toBeNull();
expect(
await client.zScore(AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, key),
).toBe(now);
expect(await client.hGet(key, 'payloadJson')).toBe('{}');
expect(await client.hGet(key, 'attempts')).toBeNull();
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
now,
);
},
);
it('defers a temporary coordination wait without consuming failed-work attempts or recreating removed payloads', async () => {
const key = 'review:pending';
const now = Date.now();
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
deferDurabilityPayload(
key: string,
time: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.deferDurabilityPayload(key, now, new Error('busy')),
).toBe('gone');
expect(await client.exists(key)).toBe(0);
await client.hSet(key, 'payloadJson', '{}');
expect(
await internal.deferDurabilityPayload(key, now, new Error('busy')),
).toBe('requeued');
expect(await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key)).toBe(
now + 5000,
);
expect(await client.hGet(key, 'attempts')).toBeNull();
});
async function fixture() {
const matchId = new mongoose.Types.ObjectId();
const id = matchId.toString();
const lifecycle = `life:${id}`;
const playerA = new mongoose.Types.ObjectId().toString();
const playerB = new mongoose.Types.ObjectId().toString();
const captain = new mongoose.Types.ObjectId().toString();
const initialRound = `initial:${id}`;
const structure = {
auctionPlayerOrder: [{ userId: playerA }, { userId: playerB }],
auctionTeams: [
{
teamIndex: 1,
captainUserId: captain,
remainingBidPointAmount: 100000,
members: [],
},
],
currentAuctionPlayerOrderIndex: 0,
currentAuctionRound: 1,
passedAuctionPlayerQueue: [],
maximumBidRoundDurationSeconds: 30,
bidRoundExtensionSeconds: 5,
initialTeamBidPointAmount: 100000,
topTierAuctionPriorityThreshold: 'NONE',
manualTopTierAuctionPriorityUserIds: [],
};
await commands.hydrateState(id, {
matchId: id,
auctionLifecycleId: lifecycle,
revision: 0,
auctionHistorySequence: 1,
phase: 'bidding',
maximumBidRoundDurationSeconds: 30,
structureStateJson: JSON.stringify(structure),
liveStatePatch: {
currentAuctionPlayerOrderIndex: 0,
currentBidRoundPlayerUserId: playerA,
currentBidRoundId: initialRound,
maximumBidRoundDurationSeconds: 30,
bidRoundExtensionSeconds: 5,
'team:1:captainUserId': captain,
'team:1:remainingBidPointAmount': 100000,
'team:1:assignedAuctionTeamMemberCount': 1,
},
});
const auction = {
_id: new mongoose.Types.ObjectId(),
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: lifecycle,
phase: 'bidding',
revision: 1,
};
const service = new AuctionService(
{ getModel: async () => ({ findOne: async () => auction }) } as never,
{} as never,
{} as never,
{} as never,
redis,
commands,
reader,
);
const bid = (
requestId: string,
amount: number,
player = playerA,
round = initialRound,
) => ({
matchId: id,
bidderTeamIndex: 1,
bidderUserId: captain,
bidPointAmount: amount,
bidRequestId: requestId,
clientObservedAuctionLifecycleId: lifecycle,
clientObservedCurrentBidRoundPlayerUserId: player,
clientObservedBidRoundId: round,
maximumAuctionTeamMemberCount: 5,
minimumBidPointReservePerRemainingAuctionTeamMemberSlot: 10,
maximumStoredAuctionBidHistoryCount: 1000,
maximumStoredBidRequestResultCount: 500,
minimumBidRequestResultReplayWindowMS: 300000,
});
const undo = async (snapshot: Record<string, unknown>) => {
const internal = service as unknown as {
undoRedisAuction(params: unknown): Promise<unknown>;
};
return internal.undoRedisAuction({
auction,
snapshot,
targetHistorySequence: 1,
undoingCompletedAuction: false,
});
};
return {
id,
matchId,
lifecycle,
playerA,
playerB,
initialRound,
auction,
service,
bid,
undo,
keys: buildAuctionRedisKeys(id),
};
}
it('waits briefly for a busy recovery lock and returns the recovered state', async () => {
const f = await fixture();
f.auction.revision = 10;
const key = `auction:${f.id}:lifecycleMutationLock`;
await redis.acquireTokenLock(key, 'other-reader', 30000);
const released = new Promise((resolve, reject) => {
setTimeout(() => {
void redis.releaseTokenLock(key, 'other-reader').then(resolve, reject);
}, 100);
});
const [view] = await Promise.all([
f.service.getStateByMatchId(f.matchId),
released,
]);
expect(view).toMatchObject({ auctionLifecycleId: f.lifecycle, revision: 11 });
});
it('bounds recovery waits and reports a retriable state error without releasing another owner lock', async () => {
const f = await fixture();
f.auction.revision = 10;
const key = `auction:${f.id}:lifecycleMutationLock`;
await redis.acquireTokenLock(key, 'slow-owner', 30000);
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toMatchObject({
response: { code: 'AUCTION_STATE_RECOVERING', retryAfterMS: 500 },
});
expect(await client.get(key)).toBe('slow-owner');
});
it('keeps accepting bids after 1000 history entries and preserves accepted request replay across players', async () => {
const f = await fixture();
const first = await commands.placeBid(
f.id,
f.bid('accepted-before-next', 10),
);
expect(first).toMatchObject({ ok: true });
await client.rPush(
f.keys.bidHistoryKey,
Array.from({ length: 999 }, () => JSON.stringify({ bidPointAmount: 10 })),
);
await client.hSet(f.keys.liveStateKey, 'phase', 'sold');
await f.service.next(f.matchId);
await f.service.openBidding(f.matchId);
const state = (await reader.readAuctionStateView(f.id))!;
const result = await commands.placeBid(
f.id,
f.bid('next-player', 10, f.playerB, String(state.currentBidRoundId)),
);
expect(result).toMatchObject({ ok: true });
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
expect(
await commands.placeBid(f.id, f.bid('accepted-before-next', 10)),
).toMatchObject({ ...first, isReplay: true });
});
it('500 rejected requests do not block a valid bid, and rejection retention stays bounded', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'currentHighestBidPointAmount', '10');
for (let i = 0; i < 501; i++) {
expect(
await commands.placeBid(f.id, f.bid(`too-low:${i}`, 5)),
).toMatchObject({ rejected: true, code: 'TOO_LOW' });
}
expect(await client.lLen(f.keys.rejectedBidRequestResultOrderKey)).toBe(500);
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(0);
expect(await commands.placeBid(f.id, f.bid('valid', 20))).toMatchObject({
ok: true,
});
expect(await commands.placeBid(f.id, f.bid('valid', 30))).toMatchObject({
rejected: true,
code: 'DUPLICATE_REQUEST_CONFLICT',
});
});
it('still protects 500 accepted results within their replay window', async () => {
const f = await fixture();
for (let i = 1; i <= 500; i++)
expect(
await commands.placeBid(f.id, f.bid(`accepted:${i}`, i)),
).toMatchObject({ ok: true });
expect(await commands.placeBid(f.id, f.bid('overflow', 501))).toMatchObject({
rejected: true,
code: 'BID_REQUEST_RESULT_LIMIT',
});
expect(await commands.placeBid(f.id, f.bid('accepted:1', 1))).toMatchObject({
ok: true,
isReplay: true,
});
});
it('migrates rejected results from the former shared quota without losing accepted replays', async () => {
const f = await fixture();
await commands.placeBid(f.id, f.bid('accepted', 10));
await client.hDel(f.keys.bidRequestResultsKey, '_acceptedOnlyOrderV1');
for (let i = 0; i < 499; i++) {
const id = `legacy-rejected:${i}`;
await client.hSet(f.keys.bidRequestResultsKey, {
[`${id}:status`]: 'rejected',
[`${id}:createdAtTimestampMS`]: String(Date.now()),
});
await client.rPush(f.keys.bidRequestResultOrderKey, id);
}
expect(
await commands.placeBid(f.id, f.bid('next-accepted', 20)),
).toMatchObject({ ok: true });
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(2);
expect(await commands.placeBid(f.id, f.bid('accepted', 10))).toMatchObject({
ok: true,
isReplay: true,
});
});
it('encodes opaque round identities safely in bid history', async () => {
const f = await fixture();
const round = 'round:with"quotes\\and-slashes';
await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', round);
expect(
await commands.placeBid(f.id, f.bid('quoted-round', 10, f.playerA, round)),
).toMatchObject({ ok: true, auctionBidHistoryEntry: { bidRoundId: round } });
expect((await reader.readAuctionStateView(f.id))?.auctionBidHistory).toEqual([
expect.objectContaining({ bidRoundId: round }),
]);
});
it('rejects delayed pre-undo bids and omitted round identities while accepting the restored round', async () => {
const f = await fixture();
const snapshot = (await reader.readAuctionStateView(f.id))!;
await f.undo(snapshot);
const restored = (await reader.readAuctionStateView(f.id))!;
expect(restored.currentBidRoundId).not.toBe(f.initialRound);
expect(await commands.placeBid(f.id, f.bid('old-round', 100))).toMatchObject({
code: 'STALE_BID_ROUND',
});
expect(
await commands.placeBid(f.id, {
...f.bid('missing-round', 100),
clientObservedBidRoundId: undefined,
}),
).toMatchObject({ code: 'BID_CLIENT_UPDATE_REQUIRED' });
expect(
await commands.placeBid(
f.id,
f.bid('new-round', 100, f.playerA, String(restored.currentBidRoundId)),
),
).toMatchObject({
ok: true,
auctionBidHistoryEntry: {
playerUserId: f.playerA,
bidRoundId: restored.currentBidRoundId,
},
});
});
it('removes completion TTLs when resuming an auction', async () => {
const f = await fixture();
const snapshot = (await reader.readAuctionStateView(f.id))!;
await client.expire(f.keys.liveStateKey, 604800);
await client.expire(f.keys.structureStateKey, 604800);
await f.undo(snapshot);
expect(await client.ttl(f.keys.liveStateKey)).toBe(-1);
expect(await client.ttl(f.keys.structureStateKey)).toBe(-1);
});
it('fences direct foreign-lifecycle hydration and repairs an orphan using current Mongo authority', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'revision', '150');
expect(
await commands.hydrateState(f.id, {
matchId: f.id,
auctionLifecycleId: 'replacement',
revision: 0,
auctionHistorySequence: 1,
phase: 'presenting',
maximumBidRoundDurationSeconds: 30,
structureStateJson: '{}',
}),
).toMatchObject({ rejected: true, code: 'LIFECYCLE_CONFLICT' });
const current = {
...f.auction,
auctionLifecycleId: 'replacement',
phase: 'presenting',
revision: 1,
auctionTeams: [],
auctionPlayerOrder: [],
};
jest.spyOn(f.service, 'getByMatchId').mockResolvedValue(current as never);
expect(await f.service.getStateByMatchId(f.matchId)).toMatchObject({
auctionLifecycleId: 'replacement',
phase: 'presenting',
});
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
'replacement',
);
});
it('does not let an old Mongo read overwrite the newly started lifecycle', async () => {
const f = await fixture();
const stale = { ...f.auction, auctionLifecycleId: 'obsolete', revision: 500 };
jest
.spyOn(f.service, 'getByMatchId')
.mockResolvedValueOnce(stale as never)
.mockResolvedValue(f.auction as never);
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toThrow(
'lifecycle has changed',
);
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
f.lifecycle,
);
});
it('serializes lifecycle mutations without deleting another owner lock', async () => {
const lock = new AuctionLifecycleLock(redis);
await lock.run('match', async () => {
await expect(lock.run('match', async () => {})).rejects.toThrow(
AuctionLifecycleBusyError,
);
});
await expect(lock.run('match', async () => 'done')).resolves.toBe('done');
});
it('persists the exact Redis deadline when opening bidding and restoring a bidding snapshot', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'phase', 'presenting');
await client.hDel(f.keys.liveStateKey, 'currentBidRoundDeadlineTimestampMS');
await f.service.openBidding(f.matchId);
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const verify = async () => {
const live = (await reader.readAuctionStateView(f.id))!;
const queued = await client.zRange(
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
0,
-1,
);
const payloads = await Promise.all(
queued.map((key) => scheduler.readDurabilityPayload(key)),
);
const payload = payloads.find(
(entry) => entry?.revision === live.revision,
)!;
expect(
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS,
).toBe(live.currentBidRoundDeadlineTimestampMS);
expect(
new Date(
String(payload.afterAuctionMongoProjection.currentDeadline),
).getTime(),
).toBe(live.currentBidRoundDeadlineTimestampMS);
expect(
payload.afterAuctionMongoProjection.passedAuctionPlayerQueue,
).toEqual([]);
return live;
};
const snapshot = await verify();
await f.undo({ ...snapshot, currentBidRoundDeadlineTimestampMS: 1 });
await verify();
});
it('archives every accepted bid beyond the recent 1000, with no duplicate archive event for a replay', async () => {
const f = await fixture();
for (let amount = 1; amount <= 1005; amount++) {
expect(
await commands.placeBid(f.id, {
...f.bid(`archive:${amount}`, amount),
maximumStoredBidRequestResultCount: 2000,
}),
).toMatchObject({ ok: true });
}
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
expect(
await commands.placeBid(f.id, {
...f.bid('archive:1', 1),
maximumStoredBidRequestResultCount: 2000,
}),
).toMatchObject({ ok: true, isReplay: true });
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
const [first] = await client.zRange(AUCTION_BID_ARCHIVE_QUEUE_KEY, 0, 0);
expect(
JSON.parse((await client.hGet(AUCTION_BID_ARCHIVE_PAYLOADS_KEY, first))!),
).toMatchObject({
bidPointAmount: 1,
bidRequestId: 'archive:1',
playerUserId: f.playerA,
auctionLifecycleId: f.lifecycle,
});
});
it.each(['missing', 'empty'])(
'archives an accepted pre-upgrade bid with a %s Redis round ID',
async (kind) => {
const f = await fixture();
if (kind === 'missing')
await client.hDel(f.keys.liveStateKey, 'currentBidRoundId');
else await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', '');
expect(
await commands.placeBid(f.id, {
...f.bid('legacy-round', 10),
clientObservedBidRoundId: undefined,
}),
).toMatchObject({
ok: true,
auctionBidHistoryEntry: { bidRoundId: '', playerUserId: f.playerA },
});
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
matchedCount: 0,
upsertedCount: operations.length,
}));
const archive = new AuctionBidArchiveService(
{ getModel: async () => ({ bulkWrite }) } as never,
redis,
);
await expect(archive.persistPendingBids()).resolves.toBe(1);
const write = bulkWrite.mock.calls[0][0][0] as {
updateOne: { update: { $setOnInsert: Record<string, unknown> } };
};
expect(write.updateOne.update.$setOnInsert).not.toHaveProperty(
'bidRoundId',
);
expect(String(write.updateOne.update.$setOnInsert.playerUserId)).toBe(
f.playerA,
);
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(0);
},
);
it('does not mirror an old generation failure over a newer Redis side effect', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, {
matchSideEffectTransitionId: 'new',
matchSideEffectRevision: '2',
matchSideEffectStatus: 'pending',
});
const sideEffects = new AuctionSideEffectService(redis);
expect(
await sideEffects.mirrorMatchSideEffectStatus({
matchId: f.id,
auctionLifecycleId: f.lifecycle,
auctionStateTransitionId: 'old',
matchSideEffectRevision: 100,
matchSideEffectStatus: 'failed',
}),
).toBe(false);
expect(await client.hGet(f.keys.liveStateKey, 'matchSideEffectStatus')).toBe(
'pending',
);
});
it('queues a terminal notification for ordinary failures and backfills old dead letters', async () => {
const key = 'review:ordinary-failure';
await client.hSet(key, { payloadJson: '{}', attempts: '9' });
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
recordDurabilityPayloadFailure(
key: string,
now: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.recordDurabilityPayloadFailure(
key,
1000,
new Error('failed before worker'),
),
).toBe('dead-lettered');
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
1000,
);
// Simulate an older deployment which did not create notification jobs.
await client.zRem(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key);
await scheduler.reconcileDeadLetterStatuses();
expect(
await client.hGet(key, 'deadLetterStatusReconciledAtTimestampMS'),
).not.toBeNull();
expect(
await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key),
).toBeNull();
});