fix(auction): harden lifecycle and durable bid storage

Serialize lifecycle changes and make cancellation, deadlines, and match side effects recoverable and idempotent.

Archive accepted bids durably, quarantine invalid payloads, and preserve compatibility with empty round IDs. Synchronize HTTP polling and socket state with scoped resync events.

Add cancellation cleanup index migration and regression coverage. Validated with 744 API tests, 65 Redis integration tests, and 28 Mongo integration tests.
This commit is contained in:
dahyeon committed 2026-11-09 23:24:11 +09:00
1 parent f2eec479d9
commit f649490cd2
55 files changed
+5374 -522

No files matched your search

@@ -7,7 +7,13 @@ function makeService(
) {
const sendCommand = jest
.fn()
.mockImplementation(() => Promise.resolve(commandReplies.shift()));
.mockImplementation((command: string[]) =>
Promise.resolve(
command[0] === 'EVAL'
? [currentServerTimestampMS, ...commandReplies.splice(0, 3)]
: commandReplies.shift(),
),
);
const auctionRedisService = {
getCurrentRedisTimestampMS: jest
.fn()
@@ -125,19 +131,14 @@ describe('AuctionRedisStateReaderService', () => {
});
expect(view).not.toHaveProperty('team:2:remainingBidPointAmount');
expect(view).not.toHaveProperty('bidRoundExpiryDecision');
expect(sendCommand).toHaveBeenNthCalledWith(1, [
'HGETALL',
expect(sendCommand).toHaveBeenCalledTimes(1);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining("redis.call('TIME')"),
'3',
'auction:match-1:liveAuctionState',
]);
expect(sendCommand).toHaveBeenNthCalledWith(2, [
'GET',
'auction:match-1:auctionStructureState',
]);
expect(sendCommand).toHaveBeenNthCalledWith(3, [
'LRANGE',
'auction:match-1:auctionBidHistory',
'0',
'-1',
]);
});
@@ -111,18 +111,11 @@ export class AuctionRedisStateReaderService {
liveReply,
structureStateJson,
bidEntries,
] = await Promise.all([
this.auctionRedisService.getCurrentRedisTimestampMS(),
this.auctionRedisService
.raw()
.sendCommand(['HGETALL', keys.liveStateKey]),
this.auctionRedisService
.raw()
.sendCommand(['GET', keys.structureStateKey]),
this.auctionRedisService
.raw()
.sendCommand(['LRANGE', keys.bidHistoryKey, '0', '-1']),
]);
] = await this.auctionRedisService
.raw()
.sendCommand<
unknown[]
>(['EVAL', ["local now = redis.call('TIME')", 'local timestamp = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)', "return {tostring(timestamp), redis.call('HGETALL', KEYS[1]), redis.call('GET', KEYS[2]), redis.call('LRANGE', KEYS[3], 0, -1)}"].join('\n'), '3', keys.liveStateKey, keys.structureStateKey, keys.bidHistoryKey]);
const liveState = parseHashReply(liveReply, {
includeInternalLiveStateFields:
@@ -144,7 +137,7 @@ export class AuctionRedisStateReaderService {
auctionBidHistoryEntries: Array.isArray(bidEntries)
? bidEntries.map(String)
: [],
currentServerTimestampMS,
currentServerTimestampMS: Number(currentServerTimestampMS),
liveState: liveState as Record<string, unknown> & {
auctionLifecycleId: string;
phase: string;
@@ -46,6 +46,7 @@ describe('auction apply transition redis script contract', () => {
'auction:match-1:bidRequestResults',
'auction:match-1:bidRequestResultOrder',
'auction:bidRoundDeadlineQueue',
'auction:match-1:rejectedBidRequestResultOrder',
]);
expect(invocation.values.slice(0, 19)).toEqual([
'match-1',
@@ -79,6 +79,7 @@ export function buildAuctionApplyTransitionScriptInvocation(
keys.bidRequestResultsKey,
keys.bidRequestResultOrderKey,
keys.deadlineQueueKey,
keys.rejectedBidRequestResultOrderKey,
],
script: AUCTION_APPLY_TRANSITION_SCRIPT,
values: [
@@ -128,6 +129,7 @@ local auctionBidHistoryKey = KEYS[8]
local bidRequestResultsKey = KEYS[9]
local bidRequestResultOrderKey = KEYS[10]
local bidRoundDeadlineQueueKey = KEYS[11]
local rejectedBidRequestResultOrderKey = KEYS[12]
local matchId = ARGV[1]
local auctionLifecycleId = ARGV[2]
@@ -270,16 +272,27 @@ for i = 1, liveStatePatchPairCount do
end
redis.call('HSET', unpack(hsetArgs))
local committedBidRoundDeadlineTimestampMS = ''
if deadlineMode == 'set-relative' then
local currentBidRoundDeadlineTimestampMS = currentRedisTimestampMS + (deadlineDurationSeconds * 1000)
committedBidRoundDeadlineTimestampMS = tostring(currentBidRoundDeadlineTimestampMS)
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS', tostring(currentBidRoundDeadlineTimestampMS))
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundId', auctionStateTransitionId)
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
elseif deadlineMode == 'remove' then
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
end
-- A resumed auction must not disappear when its former completion TTL elapses.
if redis.call('HGET', liveAuctionStateKey, 'phase') ~= 'complete' then
for _, index in ipairs({1, 2, 3, 6, 7, 8, 9, 10, 12}) do
redis.call('PERSIST', KEYS[index])
end
end
if shouldResetBidRuntime == '1' then
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey)
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey, rejectedBidRequestResultOrderKey)
end
redis.call('SADD', appliedAuctionStateTransitionIdsKey, transitionDedupeMember)
@@ -288,6 +301,8 @@ redis.call(
durabilityPayloadKey,
'payloadJson',
durabilityPayloadJson,
'committedBidRoundDeadlineTimestampMS',
committedBidRoundDeadlineTimestampMS,
'preparedAtRedisTimestampMS',
tostring(currentRedisTimestampMS),
'committedAtRedisTimestampMS',
@@ -30,6 +30,9 @@ describe('auction bid redis script contract', () => {
'auction:match-1:bidRequestResults',
'auction:match-1:bidRequestResultOrder',
'auction:bidRoundDeadlineQueue',
'auction:match-1:rejectedBidRequestResultOrder',
'auction:bidArchiveQueue',
'auction:bidArchivePayloads',
]);
expect(invocation.values).toEqual([
'match-1',
@@ -45,6 +48,7 @@ describe('auction bid redis script contract', () => {
'100',
'60000',
'200',
'',
]);
});
@@ -1,4 +1,8 @@
import type { AuctionRedisKeys } from './auction-redis.keys';
import {
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
} from './auction-redis.keys';
import { defineAuctionRedisScript } from './auction-redis.scripts';
export interface AuctionBidScriptInput {
@@ -8,6 +12,7 @@ export interface AuctionBidScriptInput {
bidderUserId: string;
clientObservedAuctionLifecycleId: string;
clientObservedCurrentBidRoundPlayerUserId: string;
clientObservedBidRoundId?: string;
maximumAuctionTeamMemberCount: number;
maximumStoredAuctionBidHistoryCount: number;
maximumStoredBidRequestResultCount: number;
@@ -40,15 +45,19 @@ export function buildBidRequestPayloadFingerprint(params: {
bidPointAmount: number;
clientObservedAuctionLifecycleId: string;
clientObservedCurrentBidRoundPlayerUserId: string;
clientObservedBidRoundId?: string;
matchId: string;
}): string {
return [
const parts = [
params.matchId,
params.clientObservedAuctionLifecycleId,
params.clientObservedCurrentBidRoundPlayerUserId,
params.bidderTeamIndex,
params.bidPointAmount,
].join('|');
];
if (params.clientObservedBidRoundId)
parts.push(params.clientObservedBidRoundId);
return parts.join('|');
}
export function buildAuctionBidScriptInvocation(
@@ -62,6 +71,9 @@ export function buildAuctionBidScriptInvocation(
keys.bidRequestResultsKey,
keys.bidRequestResultOrderKey,
keys.deadlineQueueKey,
keys.rejectedBidRequestResultOrderKey,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
],
script: AUCTION_BID_SCRIPT,
values: [
@@ -78,6 +90,7 @@ export function buildAuctionBidScriptInvocation(
String(input.maximumStoredBidRequestResultCount),
String(input.minimumBidRequestResultReplayWindowMS),
String(input.maximumStoredAuctionBidHistoryCount),
input.clientObservedBidRoundId ?? '',
],
};
}
@@ -90,6 +103,9 @@ local auctionBidHistoryKey = KEYS[2]
local bidRequestResultsKey = KEYS[3]
local bidRequestResultOrderKey = KEYS[4]
local bidRoundDeadlineQueueKey = KEYS[5]
local rejectedBidRequestResultOrderKey = KEYS[6]
local bidArchiveQueueKey = KEYS[7]
local bidArchivePayloadsKey = KEYS[8]
local matchId = ARGV[1]
local bidderTeamIndex = ARGV[2]
@@ -104,6 +120,7 @@ local bidRequestPayloadFingerprint = ARGV[10]
local maximumStoredBidRequestResultCount = tonumber(ARGV[11])
local minimumBidRequestResultReplayWindowMS = tonumber(ARGV[12])
local maximumStoredAuctionBidHistoryCount = tonumber(ARGV[13])
local clientObservedBidRoundId = ARGV[14] or ''
local redisTime = redis.call('TIME')
local currentRedisTimestampMS = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)
@@ -168,12 +185,23 @@ if existingFingerprint then
return replayExistingRequest()
end
trimOldBidRequestResults()
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
-- Upgrade the old shared queue once, preserving every accepted replay record.
if redis.call('HSETNX', bidRequestResultsKey, '_acceptedOnlyOrderV1', '1') == 1 then
for _, oldRequestId in ipairs(redis.call('LRANGE', bidRequestResultOrderKey, 0, -1)) do
if redis.call('HGET', bidRequestResultsKey, oldRequestId .. ':status') == 'rejected' then
redis.call('LREM', bidRequestResultOrderKey, 0, oldRequestId)
hdelBidRequest(oldRequestId)
end
end
end
local function storeRejectedRequest(rejectCode)
-- Rejections have their own bounded retention. They must never consume the
-- protected replay capacity of accepted bids.
while redis.call('LLEN', rejectedBidRequestResultOrderKey) >= maximumStoredBidRequestResultCount do
local oldest = redis.call('LPOP', rejectedBidRequestResultOrderKey)
if oldest then hdelBidRequest(oldest) end
end
local rejectedCurrentBidRoundDeadlineTimestampMS = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or ''
redis.call(
'HSET',
@@ -191,7 +219,7 @@ local function storeRejectedRequest(rejectCode)
requestField('createdAtTimestampMS'),
tostring(currentRedisTimestampMS)
)
redis.call('RPUSH', bidRequestResultOrderKey, bidRequestId)
redis.call('RPUSH', rejectedBidRequestResultOrderKey, bidRequestId)
return {'REJECTED', rejectCode, bidRequestId, tostring(currentRedisTimestampMS), rejectedCurrentBidRoundDeadlineTimestampMS}
end
@@ -202,9 +230,16 @@ end
local auctionLifecycleId = redis.call('HGET', liveAuctionStateKey, 'auctionLifecycleId')
local currentBidRoundPlayerUserId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundPlayerUserId')
local currentBidRoundId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundId') or ''
if currentBidRoundId ~= '' and clientObservedBidRoundId == '' then
return storeRejectedRequest('BID_CLIENT_UPDATE_REQUIRED')
end
if auctionLifecycleId ~= clientObservedAuctionLifecycleId or currentBidRoundPlayerUserId ~= clientObservedCurrentBidRoundPlayerUserId then
return storeRejectedRequest('STALE_BID_ROUND')
end
if currentBidRoundId ~= clientObservedBidRoundId then
return storeRejectedRequest('STALE_BID_ROUND')
end
local currentBidRoundDeadlineTimestampMS = tonumber(redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or '0')
if currentBidRoundDeadlineTimestampMS <= currentRedisTimestampMS then
@@ -235,8 +270,9 @@ if bidPointAmount + requiredReserveAfterBid > remainingBidPointAmount then
return storeRejectedRequest('INSUFFICIENT')
end
if redis.call('LLEN', auctionBidHistoryKey) >= maximumStoredAuctionBidHistoryCount then
return storeRejectedRequest('BID_HISTORY_LIMIT')
trimOldBidRequestResults()
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
end
local maximumBidRoundDurationSeconds = tonumber(redis.call('HGET', liveAuctionStateKey, 'maximumBidRoundDurationSeconds') or '0')
@@ -254,14 +290,36 @@ end
local nextBidRoundDeadlineTimestampMS = currentRedisTimestampMS + nextRemainingBidRoundTimeMS
local revision = redis.call('HINCRBY', liveAuctionStateKey, 'revision', 1)
local auctionBidHistoryEntryJson =
'{"bidderTeamIndex":' .. bidderTeamIndex ..
',"bidPointAmount":' .. tostring(bidPointAmount) ..
',"bidderUserId":"' .. bidderUserId ..
'","bidPlacedAtTimestampMS":' .. tostring(currentRedisTimestampMS) ..
'}'
local auctionBidHistoryEntryJson = cjson.encode({
bidRequestId = bidRequestId,
revision = revision,
bidderTeamIndex = tonumber(bidderTeamIndex),
bidPointAmount = bidPointAmount,
bidderUserId = bidderUserId,
playerUserId = currentBidRoundPlayerUserId,
bidRoundId = currentBidRoundId,
bidPlacedAtTimestampMS = currentRedisTimestampMS
})
-- Accepted replay returns above this point. Persist every accepted event before
-- trimming the recent view; the durable archive worker removes only saved events.
local archiveId = matchId .. ':' .. auctionLifecycleId .. ':' .. string.format('%016d', revision)
redis.call('HSET', bidArchivePayloadsKey, archiveId, cjson.encode({
_id = archiveId,
matchId = matchId,
auctionLifecycleId = auctionLifecycleId,
revision = revision,
bidRequestId = bidRequestId,
bidderTeamIndex = tonumber(bidderTeamIndex),
bidPointAmount = bidPointAmount,
bidderUserId = bidderUserId,
playerUserId = currentBidRoundPlayerUserId,
bidRoundId = currentBidRoundId,
bidPlacedAtTimestampMS = currentRedisTimestampMS
}))
redis.call('ZADD', bidArchiveQueueKey, currentRedisTimestampMS, archiveId)
redis.call('RPUSH', auctionBidHistoryKey, auctionBidHistoryEntryJson)
redis.call('LTRIM', auctionBidHistoryKey, -maximumStoredAuctionBidHistoryCount, -1)
redis.call(
'HSET',
liveAuctionStateKey,
@@ -4,6 +4,14 @@ export const AUCTION_DURABILITY_RETRY_QUEUE_KEY =
'auction:auctionDurabilityRetryQueue';
export const AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY =
'auction:auctionDurabilityDeadLetterQueue';
export const AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY =
'auction:deadLetterStatusQueue';
export const AUCTION_BID_ARCHIVE_QUEUE_KEY = 'auction:bidArchiveQueue';
export const AUCTION_BID_ARCHIVE_PAYLOADS_KEY = 'auction:bidArchivePayloads';
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY =
'auction:bidArchiveDeadLetterQueue';
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY =
'auction:bidArchiveDeadLetterPayloads';
export interface AuctionRedisKeys {
actionRequestResultOrderKey: string;
@@ -12,6 +20,7 @@ export interface AuctionRedisKeys {
bidHistoryKey: string;
bidRequestResultOrderKey: string;
bidRequestResultsKey: string;
rejectedBidRequestResultOrderKey: string;
deadlineQueueKey: string;
durabilityDeadLetterQueueKey: string;
durabilityPayloadKey: (auctionStateTransitionId: string) => string;
@@ -33,6 +42,7 @@ export function buildAuctionRedisKeys(matchId: string): AuctionRedisKeys {
bidHistoryKey: `${prefix}:auctionBidHistory`,
bidRequestResultOrderKey: `${prefix}:bidRequestResultOrder`,
bidRequestResultsKey: `${prefix}:bidRequestResults`,
rejectedBidRequestResultOrderKey: `${prefix}:rejectedBidRequestResultOrder`,
deadlineQueueKey: AUCTION_BID_ROUND_DEADLINE_QUEUE_KEY,
durabilityDeadLetterQueueKey: AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
durabilityPayloadKey: (auctionStateTransitionId: string) =>
@@ -24,7 +24,7 @@ describe('auction redis state assembly', () => {
mongo: { auctionLifecycleId: 'new-life', revision: 2 },
},
{
expected: false,
expected: true,
live: { auctionLifecycleId: 'redis-life', revision: 3 },
mongo: { auctionLifecycleId: 'mongo-life', revision: 2 },
},
@@ -62,13 +62,11 @@ export function shouldHydrateRedisAuctionState(
params: ShouldHydrateRedisAuctionStateParams,
): boolean {
if (!params.live) return true;
if (params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId)
return true;
const redisRevision = params.live.revision ?? -1;
const mongoRevision = params.mongo.revision;
if (mongoRevision > redisRevision) return true;
return (
mongoRevision === redisRevision &&
params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId
);
return mongoRevision > redisRevision;
}
export function assembleAuctionViewFromRedisState(
@@ -140,13 +138,17 @@ if not liveStatePatchPairCount or #ARGV ~= 8 + (liveStatePatchPairCount * 2) the
return {'REJECTED', 'INVALID_PATCH_ARITY', tostring(currentRedisTimestampMS)}
end
-- A lifecycle replacement requires Mongo authority and an explicit cleanup
-- under the lifecycle lock. Revision numbers from different runs are unrelated.
if redisAuctionLifecycleId and redisAuctionLifecycleId ~= mongoAuctionLifecycleId then
return {'REJECTED', 'LIFECYCLE_CONFLICT', tostring(currentRedisTimestampMS)}
end
local shouldHydrate = false
if redisRevision < 0 then
shouldHydrate = true
elseif mongoRevision > redisRevision then
shouldHydrate = true
elseif mongoRevision == redisRevision and mongoAuctionLifecycleId ~= redisAuctionLifecycleId then
shouldHydrate = true
end
if not shouldHydrate then
@@ -184,6 +186,7 @@ if phase == 'bidding' then
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
else
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
end
return {'OK', tostring(hydratedRevision), tostring(currentRedisTimestampMS)}