fix: prevent empty-team auction completion state
This commit is contained in:
1 parent
2984bd1604
commit
ab721873f3
6 files changed
+748
-452
No files matched your search
@@ -128,15 +128,17 @@ export class AuctionDurabilityScheduler
|
||||
currentRedisTimestampMS: number,
|
||||
batchLimit: number,
|
||||
): Promise<string[]> {
|
||||
const reply = await this.auctionRedisService.raw().sendCommand([
|
||||
'ZRANGEBYSCORE',
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
'-inf',
|
||||
String(currentRedisTimestampMS),
|
||||
'LIMIT',
|
||||
'0',
|
||||
String(batchLimit),
|
||||
]);
|
||||
const reply = await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'ZRANGEBYSCORE',
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
'-inf',
|
||||
String(currentRedisTimestampMS),
|
||||
'LIMIT',
|
||||
'0',
|
||||
String(batchLimit),
|
||||
]);
|
||||
return Array.isArray(reply) ? reply.map(String) : [];
|
||||
}
|
||||
|
||||
@@ -168,12 +170,11 @@ export class AuctionDurabilityScheduler
|
||||
await this.clearDurabilityPayload(payloadKey);
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
const failureDisposition =
|
||||
await this.recordDurabilityPayloadFailure(
|
||||
payloadKey,
|
||||
currentRedisTimestampMS,
|
||||
error,
|
||||
);
|
||||
const failureDisposition = await this.recordDurabilityPayloadFailure(
|
||||
payloadKey,
|
||||
currentRedisTimestampMS,
|
||||
error,
|
||||
);
|
||||
this.logger.warn(
|
||||
`Auction durability payload failed for ${payloadKey} (${failureDisposition}): ${
|
||||
error instanceof Error ? error.message : error
|
||||
@@ -195,7 +196,9 @@ export class AuctionDurabilityScheduler
|
||||
const payloadJson = hash.payloadJson;
|
||||
if (!payloadJson) return null;
|
||||
const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload;
|
||||
const committedAtRedisTimestampMS = Number(hash.committedAtRedisTimestampMS);
|
||||
const committedAtRedisTimestampMS = Number(
|
||||
hash.committedAtRedisTimestampMS,
|
||||
);
|
||||
if (Number.isFinite(committedAtRedisTimestampMS)) {
|
||||
payload.committedAtRedisTimestampMS = committedAtRedisTimestampMS;
|
||||
}
|
||||
@@ -223,12 +226,15 @@ export class AuctionDurabilityScheduler
|
||||
try {
|
||||
session?.startTransaction();
|
||||
if (typeof AuctionModel.findById === 'function') {
|
||||
const currentProjection = await readAuctionProjectionIdentityForMutation(
|
||||
AuctionModel,
|
||||
payload.auctionId,
|
||||
session,
|
||||
);
|
||||
if (!shouldProcessAuctionDurabilityPayload(currentProjection, payload)) {
|
||||
const currentProjection =
|
||||
await readAuctionProjectionIdentityForMutation(
|
||||
AuctionModel,
|
||||
payload.auctionId,
|
||||
session,
|
||||
);
|
||||
if (
|
||||
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
|
||||
) {
|
||||
if (session) await session.commitTransaction();
|
||||
return;
|
||||
}
|
||||
@@ -370,15 +376,17 @@ export class AuctionDurabilityScheduler
|
||||
);
|
||||
if (!hasMatchedDocument(leaseResult)) return;
|
||||
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
expectedMatchSideEffectRevision: Number(
|
||||
payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
|
||||
),
|
||||
payload,
|
||||
status: 'processing',
|
||||
});
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
|
||||
{
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
expectedMatchSideEffectRevision: Number(
|
||||
payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
|
||||
),
|
||||
payload,
|
||||
status: 'processing',
|
||||
},
|
||||
);
|
||||
if (!processingDelta) return;
|
||||
let expectedMatchSideEffectRevision =
|
||||
processingDelta.matchSideEffectRevision;
|
||||
@@ -396,11 +404,18 @@ export class AuctionDurabilityScheduler
|
||||
teamBuiltAt?: Date;
|
||||
teams: ReturnType<typeof buildMatchTeamsFromProjection>;
|
||||
} | null;
|
||||
if (!match) throw new Error(`Match not found: ${String(payload.matchId)}`);
|
||||
match.teams = buildMatchTeamsFromProjection(
|
||||
if (!match)
|
||||
throw new Error(`Match not found: ${String(payload.matchId)}`);
|
||||
const matchTeams = buildMatchTeamsFromProjection(
|
||||
payload.afterAuctionMongoProjection,
|
||||
match,
|
||||
);
|
||||
if (matchTeams.length === 0) {
|
||||
throw new Error(
|
||||
`Auction complete side effect produced no match teams for match ${String(payload.matchId)}.`,
|
||||
);
|
||||
}
|
||||
match.teams = matchTeams;
|
||||
match.status = 'team-built';
|
||||
match.teamBuiltAt = match.teamBuiltAt ?? new Date();
|
||||
await match.save();
|
||||
@@ -553,15 +568,18 @@ export class AuctionDurabilityScheduler
|
||||
);
|
||||
if (!hasMatchedDocument(leaseResult)) return;
|
||||
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
expectedMatchSideEffectRevision: Number(
|
||||
params.payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
|
||||
),
|
||||
payload: params.payload,
|
||||
status: 'processing',
|
||||
});
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
|
||||
{
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
expectedMatchSideEffectRevision: Number(
|
||||
params.payload.afterAuctionMongoProjection.matchSideEffectRevision ??
|
||||
0,
|
||||
),
|
||||
payload: params.payload,
|
||||
status: 'processing',
|
||||
},
|
||||
);
|
||||
if (!processingDelta) return;
|
||||
let expectedMatchSideEffectRevision =
|
||||
processingDelta.matchSideEffectRevision;
|
||||
@@ -605,7 +623,8 @@ export class AuctionDurabilityScheduler
|
||||
} catch (error) {
|
||||
const failureTimestampMS =
|
||||
await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
@@ -658,7 +677,9 @@ export class AuctionDurabilityScheduler
|
||||
payload: AuctionDurabilityPayload;
|
||||
session: mongoose.ClientSession | null | undefined;
|
||||
}): Promise<void> {
|
||||
const mutation = parseAuctionHistoryMutation(params.payload.historyMutation);
|
||||
const mutation = parseAuctionHistoryMutation(
|
||||
params.payload.historyMutation,
|
||||
);
|
||||
if (isDestructiveHistoryMutation(mutation)) {
|
||||
if (mutation.type === 'deleteAfterSequenceAndAppendUndo') {
|
||||
if (!params.AuctionHistoryModel.deleteMany) {
|
||||
@@ -689,7 +710,9 @@ export class AuctionDurabilityScheduler
|
||||
}
|
||||
}
|
||||
|
||||
const historyOperation = buildAuctionHistoryUpsert(params.payload).updateOne;
|
||||
const historyOperation = buildAuctionHistoryUpsert(
|
||||
params.payload,
|
||||
).updateOne;
|
||||
await execMaybe(
|
||||
params.AuctionHistoryModel.updateOne(
|
||||
historyOperation.filter as never,
|
||||
@@ -731,7 +754,9 @@ export class AuctionDurabilityScheduler
|
||||
: {}),
|
||||
matchSideEffectStatus: params.status,
|
||||
},
|
||||
...(params.errorMessage ? {} : { $unset: { matchSideEffectError: '' } }),
|
||||
...(params.errorMessage
|
||||
? {}
|
||||
: { $unset: { matchSideEffectError: '' } }),
|
||||
} as never,
|
||||
{
|
||||
projection: {
|
||||
@@ -742,13 +767,11 @@ export class AuctionDurabilityScheduler
|
||||
returnDocument: 'after',
|
||||
} as never,
|
||||
),
|
||||
)) as
|
||||
| {
|
||||
auctionLifecycleId?: string;
|
||||
matchSideEffectRevision?: number;
|
||||
matchSideEffectStatus?: AuctionMatchSideEffectStatus;
|
||||
}
|
||||
| null;
|
||||
)) as {
|
||||
auctionLifecycleId?: string;
|
||||
matchSideEffectRevision?: number;
|
||||
matchSideEffectStatus?: AuctionMatchSideEffectStatus;
|
||||
} | null;
|
||||
if (!updatedAuction?.auctionLifecycleId) return null;
|
||||
|
||||
const delta = buildMatchSideEffectDelta({
|
||||
@@ -815,12 +838,14 @@ export class AuctionDurabilityScheduler
|
||||
payloadKey: string,
|
||||
retryAtRedisTimestampMS: number,
|
||||
): Promise<void> {
|
||||
await this.auctionRedisService.raw().sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
String(retryAtRedisTimestampMS),
|
||||
payloadKey,
|
||||
]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
String(retryAtRedisTimestampMS),
|
||||
payloadKey,
|
||||
]);
|
||||
}
|
||||
|
||||
private async recordDurabilityPayloadFailure(
|
||||
@@ -829,43 +854,44 @@ export class AuctionDurabilityScheduler
|
||||
error: unknown,
|
||||
): Promise<'dead-lettered' | 'requeued'> {
|
||||
const errorMessage = formatErrorMessageForRedis(error);
|
||||
const attemptReply = await this.auctionRedisService.raw().sendCommand([
|
||||
'HINCRBY',
|
||||
payloadKey,
|
||||
'attempts',
|
||||
'1',
|
||||
]);
|
||||
const attemptReply = await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['HINCRBY', payloadKey, 'attempts', '1']);
|
||||
const attempts = Number(attemptReply);
|
||||
await this.auctionRedisService.raw().sendCommand([
|
||||
'HSET',
|
||||
payloadKey,
|
||||
'lastError',
|
||||
errorMessage,
|
||||
'lastFailedAtRedisTimestampMS',
|
||||
String(currentRedisTimestampMS),
|
||||
]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'HSET',
|
||||
payloadKey,
|
||||
'lastError',
|
||||
errorMessage,
|
||||
'lastFailedAtRedisTimestampMS',
|
||||
String(currentRedisTimestampMS),
|
||||
]);
|
||||
|
||||
if (
|
||||
Number.isFinite(attempts) &&
|
||||
attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS
|
||||
) {
|
||||
await this.auctionRedisService.raw().sendCommand([
|
||||
'ZREM',
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
payloadKey,
|
||||
]);
|
||||
await this.auctionRedisService.raw().sendCommand([
|
||||
'HSET',
|
||||
payloadKey,
|
||||
'deadLetteredAtRedisTimestampMS',
|
||||
String(currentRedisTimestampMS),
|
||||
]);
|
||||
await this.auctionRedisService.raw().sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
String(currentRedisTimestampMS),
|
||||
payloadKey,
|
||||
]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'HSET',
|
||||
payloadKey,
|
||||
'deadLetteredAtRedisTimestampMS',
|
||||
String(currentRedisTimestampMS),
|
||||
]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
String(currentRedisTimestampMS),
|
||||
payloadKey,
|
||||
]);
|
||||
return 'dead-lettered';
|
||||
}
|
||||
|
||||
@@ -881,38 +907,40 @@ export class AuctionDurabilityScheduler
|
||||
): Promise<void> {
|
||||
const matchId = String(payload.matchId);
|
||||
const keys = buildAuctionRedisKeys(matchId);
|
||||
await this.auctionRedisService.raw().sendCommand([
|
||||
'EVAL',
|
||||
[
|
||||
"local expectedAuctionLifecycleId = ARGV[2]",
|
||||
"local currentAuctionLifecycleId = redis.call('HGET', KEYS[1], 'auctionLifecycleId')",
|
||||
"if currentAuctionLifecycleId ~= expectedAuctionLifecycleId then",
|
||||
' return 0',
|
||||
'end',
|
||||
"redis.call('ZREM', KEYS[2], ARGV[1])",
|
||||
'for index = 1, #KEYS do',
|
||||
' if index ~= 2 then',
|
||||
" redis.call('EXPIRE', KEYS[index], ARGV[3])",
|
||||
' end',
|
||||
'end',
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'11',
|
||||
keys.liveStateKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.structureStateKey,
|
||||
keys.appliedTransitionIdsKey,
|
||||
keys.bidHistoryKey,
|
||||
keys.bidRequestResultsKey,
|
||||
keys.bidRequestResultOrderKey,
|
||||
keys.actionRequestResultsKey,
|
||||
keys.actionRequestResultOrderKey,
|
||||
keys.rehydrateLockKey,
|
||||
keys.stateTransitionLockKey,
|
||||
matchId,
|
||||
payload.auctionLifecycleId,
|
||||
String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS),
|
||||
]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'EVAL',
|
||||
[
|
||||
'local expectedAuctionLifecycleId = ARGV[2]',
|
||||
"local currentAuctionLifecycleId = redis.call('HGET', KEYS[1], 'auctionLifecycleId')",
|
||||
'if currentAuctionLifecycleId ~= expectedAuctionLifecycleId then',
|
||||
' return 0',
|
||||
'end',
|
||||
"redis.call('ZREM', KEYS[2], ARGV[1])",
|
||||
'for index = 1, #KEYS do',
|
||||
' if index ~= 2 then',
|
||||
" redis.call('EXPIRE', KEYS[index], ARGV[3])",
|
||||
' end',
|
||||
'end',
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'11',
|
||||
keys.liveStateKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.structureStateKey,
|
||||
keys.appliedTransitionIdsKey,
|
||||
keys.bidHistoryKey,
|
||||
keys.bidRequestResultsKey,
|
||||
keys.bidRequestResultOrderKey,
|
||||
keys.actionRequestResultsKey,
|
||||
keys.actionRequestResultOrderKey,
|
||||
keys.rehydrateLockKey,
|
||||
keys.stateTransitionLockKey,
|
||||
matchId,
|
||||
payload.auctionLifecycleId,
|
||||
String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -961,7 +989,9 @@ export function parseAuctionHistoryMutation(
|
||||
}
|
||||
if (record.type === 'reset') {
|
||||
const resetMode =
|
||||
record.resetMode === 'preserve-audit' ? 'preserve-audit' : 'clear-history';
|
||||
record.resetMode === 'preserve-audit'
|
||||
? 'preserve-audit'
|
||||
: 'clear-history';
|
||||
return { resetMode, type: 'reset' };
|
||||
}
|
||||
return { type: 'append' };
|
||||
@@ -1055,7 +1085,10 @@ export function buildSideEffectApplicationUpsert(
|
||||
payload: AuctionDurabilityPayload,
|
||||
mutation: ParsedMatchSideEffectMutation,
|
||||
) {
|
||||
const identity = buildSideEffectApplicationIdentityFromPayload(payload, mutation);
|
||||
const identity = buildSideEffectApplicationIdentityFromPayload(
|
||||
payload,
|
||||
mutation,
|
||||
);
|
||||
return {
|
||||
filter: identity,
|
||||
update: {
|
||||
@@ -1100,7 +1133,12 @@ function buildSideEffectApplicationIdentityFromPayload(
|
||||
|
||||
function hashStableJson(value: unknown): string {
|
||||
return createHash('sha256')
|
||||
.update(JSON.stringify(value, Object.keys(value as Record<string, unknown>).sort()))
|
||||
.update(
|
||||
JSON.stringify(
|
||||
value,
|
||||
Object.keys(value as Record<string, unknown>).sort(),
|
||||
),
|
||||
)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
@@ -1140,13 +1178,16 @@ function buildMatchTeamsFromProjection(
|
||||
const members: MatchTeamMember[] = [
|
||||
{
|
||||
assignedPosition:
|
||||
(captain?.preferredPositions?.[0]?.position as Position | undefined) ??
|
||||
'MID',
|
||||
(captain?.preferredPositions?.[0]?.position as
|
||||
| Position
|
||||
| undefined) ?? 'MID',
|
||||
cost: 0,
|
||||
userId: captainUserId,
|
||||
},
|
||||
...rawMembers
|
||||
.filter((member) => member.userId?.toString() !== captainUserId.toString())
|
||||
.filter(
|
||||
(member) => member.userId?.toString() !== captainUserId.toString(),
|
||||
)
|
||||
.map((member) => ({
|
||||
assignedPosition: (member.assignedPosition ?? 'MID') as Position,
|
||||
cost: Number(member.memberAcquisitionBidPointAmount ?? 0),
|
||||
@@ -1198,7 +1239,9 @@ function calculateDurabilityRetryDelayMS(attempts: number): number {
|
||||
);
|
||||
}
|
||||
|
||||
async function execMaybe<T>(queryOrPromise: Promise<T> | { exec(): Promise<T> }) {
|
||||
async function execMaybe<T>(
|
||||
queryOrPromise: Promise<T> | { exec(): Promise<T> },
|
||||
) {
|
||||
if (
|
||||
queryOrPromise &&
|
||||
typeof (queryOrPromise as { exec?: unknown }).exec === 'function'
|
||||
|
||||
Reference in new issue
Block a user