fix(auction): harden lifecycle and durable bid storage

Serialize lifecycle changes and make cancellation, deadlines, and match side effects recoverable and idempotent.

Archive accepted bids durably, quarantine invalid payloads, and preserve compatibility with empty round IDs. Synchronize HTTP polling and socket state with scoped resync events.

Add cancellation cleanup index migration and regression coverage. Validated with 744 API tests, 65 Redis integration tests, and 28 Mongo integration tests.
This commit is contained in:
dahyeon committed 2026-11-09 23:24:11 +09:00
1 parent f2eec479d9
commit f649490cd2
55 files changed
+5374 -522

No files matched your search

@@ -4,6 +4,7 @@ import {
OnModuleDestroy,
OnModuleInit,
Optional,
ConflictException,
} from '@nestjs/common';
import { createHash, randomUUID } from 'crypto';
import type mongoose from 'mongoose';
@@ -16,6 +17,10 @@ import type {
import { TournamentCodeService } from '../tournament-code/tournament-code.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { AuctionEventBusService } from './auction-event-bus.service';
import { AuctionService } from './auction.service';
import { AuctionBidArchiveService } from './auction-bid-archive.service';
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
import { buildAuctionProjectionUnset } from './auction-projection';
import {
AuctionSideEffectService,
buildMatchSideEffectDelta,
@@ -26,6 +31,7 @@ import { AuctionRedisService } from './redis/auction-redis.service';
import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service';
import {
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
buildAuctionRedisKeys,
} from './redis/auction-redis.keys';
@@ -36,6 +42,8 @@ const AUCTION_DURABILITY_BATCH_LIMIT = 20;
const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000;
const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000;
const AUCTION_DURABILITY_MAX_ATTEMPTS = 10;
const AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS = 120;
const AUCTION_DURABILITY_MAX_PENDING_DURATION_MS = 10 * 60 * 1000;
const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000;
const AUCTION_DURABILITY_LEADER_LOCK_KEY =
'auction:auctionDurabilityScheduler:leader';
@@ -45,6 +53,7 @@ const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000;
const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete';
const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset';
const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects';
class SideEffectPendingError extends Error {}
export interface AuctionDurabilityPayload {
actorUserId?: unknown;
@@ -67,6 +76,7 @@ export interface AuctionDurabilityPayload {
export interface AuctionProjectionIdentity {
auctionLifecycleId?: string;
revision?: number;
matchSideEffectTransitionId?: string;
}
type ParsedAuctionHistoryMutation =
@@ -85,7 +95,7 @@ interface CleanupSideEffectMatchDocument {
participants: Array<{ excluded?: boolean }>;
result?: unknown;
resultPollState?: unknown;
save(): Promise<unknown>;
save(options?: { session: mongoose.ClientSession }): Promise<unknown>;
status: string;
teamBuiltAt?: Date;
teams?: unknown[];
@@ -99,6 +109,7 @@ export class AuctionDurabilityScheduler
private readonly logger = new Logger(AuctionDurabilityScheduler.name);
private tickTimer?: NodeJS.Timeout;
private tickInProgress = false;
private deadLetterScanCursor = '0';
constructor(
private readonly connectionService: ConnectionService,
@@ -113,6 +124,9 @@ export class AuctionDurabilityScheduler
private readonly auctionEventBusService?: AuctionEventBusService,
@Optional()
private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService,
@Optional() private readonly auctionService?: AuctionService,
@Optional()
private readonly auctionBidArchiveService?: AuctionBidArchiveService,
) {}
public onModuleInit(): void {
@@ -187,6 +201,23 @@ export class AuctionDurabilityScheduler
});
await this.clearDurabilityPayload(payloadKey);
} catch (error) {
if (
error instanceof SideEffectPendingError ||
error instanceof ConflictException
) {
const disposition = await this.deferDurabilityPayload(
payloadKey,
currentRedisTimestampMS,
error,
);
if (disposition === 'dead-lettered') {
failed += 1;
this.logger.warn(
`Auction durability payload exceeded its pending limit: ${payloadKey}`,
);
} else skipped += 1;
continue;
}
failed += 1;
const failureDisposition = await this.recordDurabilityPayloadFailure(
payloadKey,
@@ -214,6 +245,14 @@ export class AuctionDurabilityScheduler
const payloadJson = hash.payloadJson;
if (!payloadJson) return null;
const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload;
const committedDeadline = Number(hash.committedBidRoundDeadlineTimestampMS);
if (Number.isFinite(committedDeadline) && committedDeadline > 0) {
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS =
committedDeadline;
payload.afterAuctionMongoProjection.currentDeadline = new Date(
committedDeadline,
).toISOString();
}
const committedAtRedisTimestampMS = Number(
hash.committedAtRedisTimestampMS,
);
@@ -240,12 +279,12 @@ export class AuctionDurabilityScheduler
const sideEffectMutation = parseMatchSideEffectMutation(
payload.matchSideEffectMutation,
);
let shouldUpdateProjection = true;
let shouldApplySideEffects = true;
let shouldApplyHistoryMutation = true;
try {
session?.startTransaction();
const historyMutation = parseAuctionHistoryMutation(
payload.historyMutation,
);
if (typeof AuctionModel.findById === 'function') {
const currentProjection =
await readAuctionProjectionIdentityForMutation(
@@ -256,37 +295,44 @@ export class AuctionDurabilityScheduler
if (
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
) {
// 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다.
// 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다.
if (
historyMutation.type === 'append' &&
currentProjection.auctionLifecycleId === payload.auctionLifecycleId
currentProjection.auctionLifecycleId !== payload.auctionLifecycleId
) {
await this.applyStaleAppendHistoryRecovery({
if (session) await session.commitTransaction();
return;
}
shouldApplyHistoryMutation =
!(await this.isAppendHistoryInvalidatedByLaterDestructiveMutation({
AuctionHistoryModel,
payload,
session,
});
}
if (session) await session.commitTransaction();
return;
}));
// History operations have their own sequence bounds. A newer ordinary
// projection must not discard an earlier undo's history cleanup.
shouldUpdateProjection = false;
shouldApplySideEffects =
Boolean(this.auctionRedisStateReaderService) ||
currentProjection.matchSideEffectTransitionId ===
payload.auctionStateTransitionId;
}
}
await this.applyAuctionHistoryMutation({
AuctionHistoryModel,
AuctionModel,
payload,
session,
});
if (shouldApplyHistoryMutation)
await this.applyAuctionHistoryMutation({
AuctionHistoryModel,
AuctionModel,
payload,
session,
});
const projectionOperation = buildAuctionProjectionGuardedUpdate(payload);
await execMaybe(
AuctionModel.updateOne(
projectionOperation.filter as never,
projectionOperation.update as never,
{ session } as never,
),
);
if (sideEffectMutation) {
if (shouldUpdateProjection)
await execMaybe(
AuctionModel.updateOne(
projectionOperation.filter as never,
projectionOperation.update as never,
{ session } as never,
),
);
if (sideEffectMutation && shouldApplySideEffects) {
const sideEffectOperation = buildSideEffectApplicationUpsert(
payload,
sideEffectMutation,
@@ -312,7 +358,7 @@ export class AuctionDurabilityScheduler
await this.broadcastLatestRedisAuctionStateIfRequested(payload);
if (sideEffectMutation) {
if (sideEffectMutation && shouldApplySideEffects) {
await this.applyMatchSideEffect(payload, sideEffectMutation);
}
}
@@ -352,6 +398,64 @@ export class AuctionDurabilityScheduler
private async applyMatchSideEffect(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
return new AuctionLifecycleLock(this.auctionRedisService).run(
String(payload.matchId),
async () => {
if (this.auctionRedisStateReaderService) {
const state =
await this.auctionRedisStateReaderService.readAuctionStateView(
String(payload.matchId),
);
if (!state)
throw new SideEffectPendingError(
'Auction realtime state is unavailable.',
);
if (
state.auctionLifecycleId !== payload.auctionLifecycleId ||
(state.matchSideEffectTransitionId &&
state.matchSideEffectTransitionId !==
payload.auctionStateTransitionId) ||
(mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE &&
state.phase !== 'complete')
)
return;
// An ordinary later projection may have reached Mongo before this
// side-effect-bearing transition. Install its pending generation only
// while Redis still names it and Mongo has not advanced beyond it.
const AuctionModel = await this.connectionService.getModel('auction');
const pendingRevision = Number(
payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
);
await execMaybe(
AuctionModel.updateOne(
{
_id: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
$or: [
{ matchSideEffectRevision: { $lt: pendingRevision } },
{ matchSideEffectRevision: { $exists: false } },
],
} as never,
{
$set: {
matchSideEffectTransitionId: payload.auctionStateTransitionId,
matchSideEffectRevision: pendingRevision,
matchSideEffectStatus: 'pending',
},
$unset: { matchSideEffectError: '' },
} as never,
),
);
}
await this.applyMatchSideEffectWhileLocked(payload, mutation);
},
);
}
private async applyMatchSideEffectWhileLocked(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) {
await this.applySyncMatchTeamsSideEffect(payload, mutation);
@@ -407,7 +511,26 @@ export class AuctionDurabilityScheduler
}) as never,
),
);
if (!hasMatchedDocument(leaseResult)) return;
if (!hasMatchedDocument(leaseResult)) {
const existing = await AuctionSideEffectApplicationModel.findOne(
identity as never,
)
.lean()
.exec();
if (existing?.state === 'completed') {
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: currentRedisTimestampMS,
payload,
status: 'completed',
});
await this.expireCompletedAuctionRedisRuntime(payload);
return;
}
throw new SideEffectPendingError(
'Auction side effect is still owned by another worker.',
);
}
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
{
@@ -420,7 +543,11 @@ export class AuctionDurabilityScheduler
status: 'processing',
},
);
if (!processingDelta) return;
if (!processingDelta) {
throw new SideEffectPendingError(
'Auction side-effect generation is not ready.',
);
}
const expectedMatchSideEffectRevision =
processingDelta.matchSideEffectRevision;
@@ -489,7 +616,11 @@ export class AuctionDurabilityScheduler
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
completedAtTimestampMS,
@@ -518,7 +649,11 @@ export class AuctionDurabilityScheduler
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
lastError: error instanceof Error ? error.message : String(error),
@@ -544,33 +679,6 @@ export class AuctionDurabilityScheduler
}
}
private async applyStaleAppendHistoryRecovery(params: {
AuctionHistoryModel: {
findOne?: (...args: never[]) => unknown;
updateOne: (...args: never[]) => unknown;
};
payload: AuctionDurabilityPayload;
session: mongoose.ClientSession | null | undefined;
}): Promise<void> {
const invalidated =
await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params);
if (invalidated) return;
const historyOperation = buildAuctionHistoryUpsert(
params.payload,
).updateOne;
await execMaybe(
params.AuctionHistoryModel.updateOne(
historyOperation.filter as never,
historyOperation.update as never,
{
session: params.session,
upsert: true,
} as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
);
}
private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: {
AuctionHistoryModel: {
findOne?: (...args: never[]) => unknown;
@@ -622,9 +730,7 @@ export class AuctionDurabilityScheduler
await this.applyMatchCleanupSideEffect({
mutation,
payload,
updateMatch: async (match) => {
await this.tournamentCodeService?.releaseFromMatch(match as never);
await this.tournamentResultService?.cancelPolling(match as never);
updateMatch: (match) => {
match.status = 'recruiting';
match.teams = [];
match.tournamentCodeIds = [];
@@ -635,7 +741,6 @@ export class AuctionDurabilityScheduler
match.teamBuiltAt = undefined;
match.result = undefined;
match.resultPollState = undefined;
await match.save();
},
});
}
@@ -647,13 +752,14 @@ export class AuctionDurabilityScheduler
await this.applyMatchCleanupSideEffect({
mutation,
payload,
updateMatch: async (match) => {
await this.tournamentCodeService?.releaseFromMatch(match as never);
await this.tournamentResultService?.cancelPolling(match as never);
updateMatch: (match) => {
match.status = 'team-built';
match.teams = [];
match.teamBuiltAt = undefined;
match.tournamentCodeIds = [];
match.gameSchedule = [];
match.result = undefined;
match.resultPollState = undefined;
await match.save();
},
});
}
@@ -661,7 +767,7 @@ export class AuctionDurabilityScheduler
private async applyMatchCleanupSideEffect(params: {
mutation: ParsedMatchSideEffectMutation;
payload: AuctionDurabilityPayload;
updateMatch: (match: CleanupSideEffectMatchDocument) => Promise<void>;
updateMatch: (match: CleanupSideEffectMatchDocument) => void;
}): Promise<void> {
if (!this.tournamentCodeService || !this.tournamentResultService) {
throw new Error('Auction side-effect services are not available.');
@@ -700,7 +806,25 @@ export class AuctionDurabilityScheduler
}) as never,
),
);
if (!hasMatchedDocument(leaseResult)) return;
if (!hasMatchedDocument(leaseResult)) {
const existing = await AuctionSideEffectApplicationModel.findOne(
identity as never,
)
.lean()
.exec();
if (existing?.state === 'completed') {
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: currentRedisTimestampMS,
payload: params.payload,
status: 'completed',
});
return;
}
throw new SideEffectPendingError(
'Auction side effect is still owned by another worker.',
);
}
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
{
@@ -714,29 +838,56 @@ export class AuctionDurabilityScheduler
status: 'processing',
},
);
if (!processingDelta) return;
if (!processingDelta) {
throw new SideEffectPendingError(
'Auction side-effect generation is not ready.',
);
}
const expectedMatchSideEffectRevision =
processingDelta.matchSideEffectRevision;
try {
const match = (await execMaybe(
MatchModel.findById(params.payload.matchId) as never,
)) as CleanupSideEffectMatchDocument | null;
if (!match) {
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
}
if (match.competitionId) {
throw new Error(
'Cannot reset or undo Match teams after the immutable Competition has been created.',
const session = await this.connectionService.createSession();
if (!session)
throw new Error('Failed to start Match cleanup transaction.');
try {
session.startTransaction();
const match = (await MatchModel.findById(
params.payload.matchId,
).session(session)) as CleanupSideEffectMatchDocument | null;
if (!match)
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
if (match.competitionId)
throw new Error(
'Cannot reset or undo Match teams after the immutable Competition has been created.',
);
await this.tournamentCodeService.releaseFromMatch(
match as never,
session,
);
await this.tournamentResultService.cancelPolling(
match as never,
session,
);
params.updateMatch(match);
await match.save({ session });
await session.commitTransaction();
} catch (error) {
await session.abortTransaction();
throw error;
} finally {
await session.endSession();
}
await params.updateMatch(match);
const completedAtTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
completedAtTimestampMS,
@@ -766,7 +917,11 @@ export class AuctionDurabilityScheduler
error instanceof Error ? error.message : String(error);
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
lastError: errorMessage,
@@ -828,7 +983,10 @@ export class AuctionDurabilityScheduler
params.AuctionHistoryModel.deleteMany(
{
auctionId: params.payload.auctionId,
sequence: { $gt: mutation.targetHistorySequence },
sequence: {
$gt: mutation.targetHistorySequence,
$lt: params.payload.auctionHistorySequence,
},
} as never,
{ session: params.session } as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
@@ -842,6 +1000,7 @@ export class AuctionDurabilityScheduler
params.AuctionHistoryModel.deleteMany(
{
auctionId: params.payload.auctionId,
sequence: { $lt: params.payload.auctionHistorySequence },
} as never,
{ session: params.session } as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
@@ -874,16 +1033,43 @@ export class AuctionDurabilityScheduler
};
currentServerTimestampMS: number;
errorMessage?: string;
expectedMatchSideEffectRevision: number;
expectedMatchSideEffectRevision?: number;
payload: AuctionDurabilityPayload;
status: AuctionMatchSideEffectStatus;
onlyUnfinished?: boolean;
}): Promise<MatchSideEffectDelta | null> {
const updatedAuction = (await execMaybe(
params.AuctionModel.findOneAndUpdate(
{
_id: params.payload.auctionId,
auctionLifecycleId: params.payload.auctionLifecycleId,
matchSideEffectRevision: params.expectedMatchSideEffectRevision,
...(params.onlyUnfinished
? { matchSideEffectStatus: { $ne: 'completed' } }
: {}),
...(params.status === 'processing'
? {
$or: [
{
matchSideEffectTransitionId:
params.payload.auctionStateTransitionId,
},
{
matchSideEffectTransitionId: { $exists: false },
matchSideEffectRevision:
params.expectedMatchSideEffectRevision,
},
],
}
: {
matchSideEffectTransitionId:
params.payload.auctionStateTransitionId,
...(params.expectedMatchSideEffectRevision === undefined
? {}
: {
matchSideEffectRevision:
params.expectedMatchSideEffectRevision,
}),
}),
} as never,
{
$inc: { matchSideEffectRevision: 1 },
@@ -892,6 +1078,8 @@ export class AuctionDurabilityScheduler
? { matchSideEffectError: params.errorMessage }
: {}),
matchSideEffectStatus: params.status,
matchSideEffectTransitionId:
params.payload.auctionStateTransitionId,
},
...(params.errorMessage
? {}
@@ -915,6 +1103,7 @@ export class AuctionDurabilityScheduler
const delta = buildMatchSideEffectDelta({
auctionLifecycleId: updatedAuction.auctionLifecycleId,
auctionStateTransitionId: params.payload.auctionStateTransitionId,
currentServerTimestampMS: params.currentServerTimestampMS,
matchSideEffectRevision: Number(
updatedAuction.matchSideEffectRevision ?? 0,
@@ -923,19 +1112,170 @@ export class AuctionDurabilityScheduler
updatedAuction.matchSideEffectStatus ?? params.status,
});
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
auctionLifecycleId: delta.auctionLifecycleId,
matchId: String(params.payload.matchId),
matchSideEffectRevision: delta.matchSideEffectRevision,
matchSideEffectStatus: delta.matchSideEffectStatus,
});
this.auctionEventBusService?.broadcastMatchSideEffect(
String(params.payload.matchId),
delta,
);
const mirrored =
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
auctionLifecycleId: delta.auctionLifecycleId,
auctionStateTransitionId: params.payload.auctionStateTransitionId,
matchId: String(params.payload.matchId),
matchSideEffectRevision: delta.matchSideEffectRevision,
matchSideEffectStatus: delta.matchSideEffectStatus,
});
if (mirrored !== false) {
this.auctionEventBusService?.broadcastMatchSideEffect(
String(params.payload.matchId),
delta,
);
}
return delta;
}
public async reconcileDeadLetterStatuses(limit = 20): Promise<void> {
const client = this.auctionRedisService.raw();
const now = await this.auctionRedisService.getCurrentRedisTimestampMS();
// Also reconcile entries dead-lettered before status notifications existed.
const scanned = await client.sendCommand([
'ZSCAN',
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
this.deadLetterScanCursor,
'COUNT',
String(limit),
]);
if (Array.isArray(scanned) && Array.isArray(scanned[1])) {
this.deadLetterScanCursor = String(scanned[0]);
for (let index = 0; index < scanned[1].length; index += 2) {
const key = String(scanned[1][index]);
if (
!(await client.sendCommand([
'HGET',
key,
'deadLetterStatusReconciledAtTimestampMS',
]))
) {
await client.sendCommand([
'ZADD',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
'NX',
String(now),
key,
]);
}
}
}
const reply = await client.sendCommand([
'ZRANGEBYSCORE',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
'-inf',
String(now),
'LIMIT',
'0',
String(limit),
]);
if (!Array.isArray(reply)) return;
for (const key of reply.map(String)) {
try {
const payload = await this.readDurabilityPayload(key);
if (
payload &&
parseMatchSideEffectMutation(payload.matchSideEffectMutation)
) {
const state =
await this.auctionRedisStateReaderService?.readAuctionStateView(
String(payload.matchId),
);
// A terminal notification for a previous generation must not change
// a new completion/undo, even when Mongo is behind Redis.
const superseded =
state &&
(state.auctionLifecycleId !== payload.auctionLifecycleId ||
state.matchSideEffectStatus === 'completed' ||
(state.matchSideEffectTransitionId &&
state.matchSideEffectTransitionId !==
payload.auctionStateTransitionId));
if (!superseded) {
const AuctionModel =
await this.connectionService.getModel('auction');
if (
state?.matchSideEffectTransitionId ===
payload.auctionStateTransitionId
) {
const revision = Number(
payload.afterAuctionMongoProjection.matchSideEffectRevision ??
0,
);
await AuctionModel.updateOne(
{
_id: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
$or: [
{ matchSideEffectRevision: { $lt: revision } },
{ matchSideEffectRevision: { $exists: false } },
],
} as never,
{
$set: {
matchSideEffectTransitionId:
payload.auctionStateTransitionId,
matchSideEffectRevision: revision,
matchSideEffectStatus: 'pending',
},
},
);
// Heal a same-generation revision mismatch before sending the
// terminal status; otherwise Redis could ignore it as older.
await AuctionModel.updateOne(
{
_id: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
matchSideEffectTransitionId: payload.auctionStateTransitionId,
matchSideEffectStatus: { $ne: 'completed' },
} as never,
{
$max: {
matchSideEffectRevision: Number(
state.matchSideEffectRevision ?? revision,
),
},
},
);
}
const error = await client.sendCommand(['HGET', key, 'lastError']);
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel: AuctionModel as never,
payload,
currentServerTimestampMS: now,
status: 'failed',
onlyUnfinished: true,
errorMessage:
typeof error === 'string'
? error
: 'Auction side effect exhausted its retry limit.',
});
}
}
await client.sendCommand([
'EVAL',
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 1 then redis.call('HSET', KEYS[1], 'deadLetterStatusReconciledAtTimestampMS', ARGV[1]) end redis.call('ZREM', KEYS[2], KEYS[1]); return 1",
'2',
key,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(now),
]);
} catch (error) {
// Mongo or Redis mirroring may be unavailable at dead-letter time. Keep
// the notification durable until the failure can be shown to viewers.
await client.sendCommand([
'ZADD',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(now + 5000),
key,
]);
this.logger.warn(
`Auction terminal status will retry for ${key}: ${String(error)}`,
);
}
}
}
private async safeTick(): Promise<void> {
if (this.tickInProgress) return;
this.tickInProgress = true;
@@ -948,7 +1288,22 @@ export class AuctionDurabilityScheduler
AUCTION_DURABILITY_LEADER_LOCK_TTL_MS,
);
if (!leaderLockAcquired) return;
await this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT);
// A failed maintenance task must not starve the other durable queues.
for (const work of [
() => this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT),
() => this.reconcileDeadLetterStatuses(AUCTION_DURABILITY_BATCH_LIMIT),
() => this.auctionBidArchiveService?.persistPendingBids(),
() =>
this.auctionService?.retryPendingCancellations(
AUCTION_DURABILITY_BATCH_LIMIT,
),
]) {
try {
await work();
} catch (error) {
this.logger.warn(`Auction maintenance task failed: ${String(error)}`);
}
}
} catch (error) {
this.logger.warn(
`Auction durability tick failed: ${
@@ -957,10 +1312,13 @@ export class AuctionDurabilityScheduler
);
} finally {
if (leaderLockAcquired) {
await this.auctionRedisService.releaseTokenLock(
AUCTION_DURABILITY_LEADER_LOCK_KEY,
leaderLockToken,
);
await this.auctionRedisService
.releaseTokenLock(AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken)
.catch((error: unknown) => {
this.logger.warn(
`Auction durability lock release failed: ${String(error)}`,
);
});
}
this.tickInProgress = false;
}
@@ -987,6 +1345,51 @@ export class AuctionDurabilityScheduler
]);
}
private async deferDurabilityPayload(
payloadKey: string,
timestampMS: number,
error: unknown,
): Promise<'gone' | 'requeued' | 'dead-lettered'> {
// Keep coordination waits separate from failed work, but bound them across
// restarts too. Updating the payload and queue atomically avoids orphan jobs.
const result = await this.auctionRedisService
.raw()
.sendCommand([
'EVAL',
[
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end",
"local count = redis.call('HINCRBY', KEYS[1], 'pendingAttempts', 1)",
"redis.call('HSETNX', KEYS[1], 'pendingSinceTimestampMS', ARGV[1])",
"local since = tonumber(redis.call('HGET', KEYS[1], 'pendingSinceTimestampMS'))",
"redis.call('HSET', KEYS[1], 'lastPendingReason', ARGV[5], 'lastPendingAtTimestampMS', ARGV[1])",
'if count >= tonumber(ARGV[3]) or tonumber(ARGV[1]) - since >= tonumber(ARGV[4]) then',
" redis.call('ZREM', KEYS[2], KEYS[1])",
" redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1], 'lastError', ARGV[5])",
" redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1])",
" redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1])",
' return 2',
'end',
"redis.call('ZADD', KEYS[2], tonumber(ARGV[1]) + tonumber(ARGV[2]), KEYS[1])",
'return 1',
].join('\n'),
'4',
payloadKey,
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(timestampMS),
String(AUCTION_DURABILITY_RETRY_DELAY_MS),
String(AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS),
String(AUCTION_DURABILITY_MAX_PENDING_DURATION_MS),
formatErrorMessageForRedis(error),
]);
return Number(result) === 2
? 'dead-lettered'
: Number(result) === 1
? 'requeued'
: 'gone';
}
private async recordDurabilityPayloadFailure(
payloadKey: string,
currentRedisTimestampMS: number,
@@ -1012,24 +1415,17 @@ export class AuctionDurabilityScheduler
Number.isFinite(attempts) &&
attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS
) {
await this.auctionRedisService
.raw()
.sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]);
await this.auctionRedisService
.raw()
.sendCommand([
'HSET',
'EVAL',
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1]); redis.call('ZREM', KEYS[2], KEYS[1]); redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1]); redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1]); return 1",
'4',
payloadKey,
'deadLetteredAtRedisTimestampMS',
String(currentRedisTimestampMS),
]);
await this.auctionRedisService
.raw()
.sendCommand([
'ZADD',
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(currentRedisTimestampMS),
payloadKey,
]);
return 'dead-lettered';
}
@@ -1064,7 +1460,7 @@ export class AuctionDurabilityScheduler
'end',
'return 1',
].join('\n'),
'11',
'12',
keys.liveStateKey,
keys.deadlineQueueKey,
keys.structureStateKey,
@@ -1076,6 +1472,7 @@ export class AuctionDurabilityScheduler
keys.actionRequestResultOrderKey,
keys.rehydrateLockKey,
keys.stateTransitionLockKey,
keys.rejectedBidRequestResultOrderKey,
matchId,
payload.auctionLifecycleId,
String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS),
@@ -1159,6 +1556,7 @@ async function readAuctionProjectionIdentityForMutation(
? (query as { select: (...args: never[]) => unknown }).select({
auctionLifecycleId: 1,
revision: 1,
matchSideEffectTransitionId: 1,
} as never)
: query;
const scoped =
@@ -1178,6 +1576,10 @@ async function readAuctionProjectionIdentityForMutation(
if (!projection || typeof projection !== 'object') return {};
const record = projection as Record<string, unknown>;
return {
matchSideEffectTransitionId:
typeof record.matchSideEffectTransitionId === 'string'
? record.matchSideEffectTransitionId
: undefined,
auctionLifecycleId:
typeof record.auctionLifecycleId === 'string'
? record.auctionLifecycleId
@@ -1200,6 +1602,7 @@ export function buildAuctionProjectionGuardedUpdate(
},
update: {
$set: payload.afterAuctionMongoProjection,
$unset: buildAuctionProjectionUnset(payload.afterAuctionMongoProjection),
},
};
}