fix(auction): harden lifecycle and durable bid storage
Serialize lifecycle changes and make cancellation, deadlines, and match side effects recoverable and idempotent. Archive accepted bids durably, quarantine invalid payloads, and preserve compatibility with empty round IDs. Synchronize HTTP polling and socket state with scoped resync events. Add cancellation cleanup index migration and regression coverage. Validated with 744 API tests, 65 Redis integration tests, and 28 Mongo integration tests.
This commit is contained in:
1 parent
f2eec479d9
commit
f649490cd2
55 files changed
+5374
-522
No files matched your search
@@ -0,0 +1,82 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
|
||||
const matchId = new mongoose.Types.ObjectId().toString();
|
||||
|
||||
function bid(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
_id: `${matchId}:life:${String(1).padStart(16, '0')}`,
|
||||
matchId,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 1,
|
||||
bidRequestId: 'bid-1',
|
||||
bidderUserId: new mongoose.Types.ObjectId().toString(),
|
||||
playerUserId: new mongoose.Types.ObjectId().toString(),
|
||||
bidderTeamIndex: 1,
|
||||
bidPointAmount: 10,
|
||||
bidPlacedAtTimestampMS: 1_700_000_000_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(payload: Record<string, unknown>) {
|
||||
const raw = JSON.stringify(payload);
|
||||
const commands: string[][] = [];
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
commands.push(command);
|
||||
if (command[0] === 'ZRANGE') return [payload._id];
|
||||
if (command[0] === 'HMGET') return [raw];
|
||||
return 1;
|
||||
});
|
||||
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
|
||||
matchedCount: 0,
|
||||
upsertedCount: operations.length,
|
||||
}));
|
||||
const service = new AuctionBidArchiveService(
|
||||
{ getModel: async () => ({ bulkWrite }) } as never,
|
||||
{ raw: () => ({ sendCommand }) } as never,
|
||||
);
|
||||
return { service, bulkWrite, commands };
|
||||
}
|
||||
|
||||
function savedEvent(bulkWrite: jest.Mock) {
|
||||
const [[operations]] = bulkWrite.mock.calls as unknown as [
|
||||
[
|
||||
Array<{
|
||||
updateOne: { update: { $setOnInsert: Record<string, unknown> } };
|
||||
}>,
|
||||
],
|
||||
];
|
||||
return operations[0].updateOne.update.$setOnInsert;
|
||||
}
|
||||
|
||||
function quarantined(commands: string[][]) {
|
||||
return commands.some(
|
||||
(command) =>
|
||||
command[0] === 'EVAL' &&
|
||||
command[1].includes("redis.call('ZADD', KEYS[4]"),
|
||||
);
|
||||
}
|
||||
|
||||
it('archives a pre-upgrade bid with an empty round ID without the field', async () => {
|
||||
const f = fixture(bid({ bidRoundId: '' }));
|
||||
await expect(f.service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(quarantined(f.commands)).toBe(false);
|
||||
expect(savedEvent(f.bulkWrite)).not.toHaveProperty('bidRoundId');
|
||||
});
|
||||
|
||||
it('keeps a non-empty round ID on the archived bid', async () => {
|
||||
const f = fixture(bid({ bidRoundId: 'round-1' }));
|
||||
await expect(f.service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(savedEvent(f.bulkWrite)).toMatchObject({ bidRoundId: 'round-1' });
|
||||
});
|
||||
|
||||
it.each([' ', null, 1])(
|
||||
'still quarantines an invalid round ID %j',
|
||||
async (bidRoundId) => {
|
||||
const f = fixture(bid({ bidRoundId }));
|
||||
await expect(f.service.persistPendingBids()).resolves.toBe(0);
|
||||
expect(f.bulkWrite).not.toHaveBeenCalled();
|
||||
expect(quarantined(f.commands)).toBe(true);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,242 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Injectable,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import type { AuctionBidRecord } from '../global/database/schema';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { AuctionRedisService } from './redis/auction-redis.service';
|
||||
import {
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY,
|
||||
} from './redis/auction-redis.keys';
|
||||
|
||||
// Compare the payload as well as the ID: a concurrent repair must not be moved
|
||||
// or acknowledged by a worker that read an older value.
|
||||
const QUARANTINE_BID_ARCHIVE_SCRIPT = `
|
||||
local moved = 0
|
||||
for i = 2, #ARGV, 4 do
|
||||
local current = redis.call('HGET', KEYS[1], ARGV[i])
|
||||
local matches = (ARGV[i+1] == '0' and current == false) or (ARGV[i+1] == '1' and current == ARGV[i+2])
|
||||
if matches and redis.call('ZSCORE', KEYS[2], ARGV[i]) then
|
||||
redis.call('HSET', KEYS[3], ARGV[i], ARGV[i+3])
|
||||
redis.call('ZADD', KEYS[4], ARGV[1], ARGV[i])
|
||||
redis.call('HDEL', KEYS[1], ARGV[i])
|
||||
redis.call('ZREM', KEYS[2], ARGV[i])
|
||||
moved = moved + 1
|
||||
end
|
||||
end
|
||||
return moved
|
||||
`;
|
||||
|
||||
@Injectable()
|
||||
export class AuctionBidArchiveService {
|
||||
private readonly logger = new Logger(AuctionBidArchiveService.name);
|
||||
|
||||
constructor(
|
||||
private readonly database: ConnectionService,
|
||||
private readonly redis: AuctionRedisService,
|
||||
) {}
|
||||
|
||||
public async persistPendingBids(limit = 200): Promise<number> {
|
||||
const client = this.redis.raw();
|
||||
const reply = await client.sendCommand([
|
||||
'ZRANGE',
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
'0',
|
||||
String(limit - 1),
|
||||
]);
|
||||
const ids = Array.isArray(reply) ? reply.map(String) : [];
|
||||
if (!ids.length) return 0;
|
||||
const values = await client.sendCommand<Array<string | null>>([
|
||||
'HMGET',
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
...ids,
|
||||
]);
|
||||
const valid: Array<{ event: AuctionBidRecord; raw: string }> = [];
|
||||
const invalid: Array<{ id: string; raw: string | null; error: string }> =
|
||||
[];
|
||||
for (let index = 0; index < ids.length; index++) {
|
||||
const raw = values[index] ?? null;
|
||||
try {
|
||||
valid.push({ event: parseBidArchiveEvent(ids[index], raw), raw: raw! });
|
||||
} catch (error) {
|
||||
invalid.push({
|
||||
id: ids[index],
|
||||
raw,
|
||||
error: String(error).slice(0, 2000),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (invalid.length) {
|
||||
const now = Date.now();
|
||||
const moved = await client.sendCommand<number>([
|
||||
'EVAL',
|
||||
QUARANTINE_BID_ARCHIVE_SCRIPT,
|
||||
'4',
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY,
|
||||
String(now),
|
||||
...invalid.flatMap(({ id, raw, error }) => [
|
||||
id,
|
||||
raw === null ? '0' : '1',
|
||||
raw ?? '',
|
||||
JSON.stringify({
|
||||
id,
|
||||
rawPayload: raw,
|
||||
error,
|
||||
quarantinedAtTimestampMS: now,
|
||||
}),
|
||||
]),
|
||||
]);
|
||||
if (moved > 0)
|
||||
this.logger.warn(`Quarantined ${moved} invalid bid archive events.`);
|
||||
}
|
||||
if (valid.length) {
|
||||
const model = await this.database.getModel('auctionBidRecord');
|
||||
// The deterministic _id is the dedupe key, including after a crash between
|
||||
// Mongo acknowledgement and Redis cleanup. Never trim unpersisted events.
|
||||
const result = await model.bulkWrite(
|
||||
valid.map(({ event }) => ({
|
||||
updateOne: {
|
||||
filter: { _id: event._id },
|
||||
update: { $setOnInsert: event },
|
||||
upsert: true,
|
||||
},
|
||||
})),
|
||||
{ ordered: false, throwOnValidationError: true },
|
||||
);
|
||||
// A partial/ambiguous write leaves valid entries queued. Replaying the
|
||||
// upserts is safe; assuming that a resolved bulkWrite saved all is not.
|
||||
if (result.matchedCount + result.upsertedCount !== valid.length) {
|
||||
throw new Error(
|
||||
'Bid archive batch was not fully acknowledged by MongoDB.',
|
||||
);
|
||||
}
|
||||
await client.sendCommand([
|
||||
'EVAL',
|
||||
"for i = 1, #ARGV, 2 do if redis.call('HGET', KEYS[1], ARGV[i]) == ARGV[i+1] then redis.call('HDEL', KEYS[1], ARGV[i]); redis.call('ZREM', KEYS[2], ARGV[i]) end end return 1",
|
||||
'2',
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
...valid.flatMap(({ event, raw }) => [event._id, raw]),
|
||||
]);
|
||||
}
|
||||
return valid.length;
|
||||
}
|
||||
|
||||
public async list(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
afterRevision = -1,
|
||||
limit = 100,
|
||||
) {
|
||||
if (
|
||||
!Number.isSafeInteger(afterRevision) ||
|
||||
afterRevision < -1 ||
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > 200
|
||||
) {
|
||||
throw new BadRequestException('Invalid bid history cursor or limit.');
|
||||
}
|
||||
const AuctionModel = await this.database.getModel('auction');
|
||||
const auction = await AuctionModel.findOne({ matchId })
|
||||
.select({ auctionLifecycleId: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
if (!auction) throw new NotFoundException('Auction not found.');
|
||||
const prefix = `${matchId.toString()}:${auction.auctionLifecycleId}:`;
|
||||
const model = await this.database.getModel('auctionBidRecord');
|
||||
// The built-in _id index supports ordered pagination without an additional
|
||||
// production index or an unbounded array inside an Auction document.
|
||||
const items = await model
|
||||
.find({
|
||||
_id: {
|
||||
$gt:
|
||||
afterRevision < 0
|
||||
? prefix
|
||||
: prefix + String(afterRevision).padStart(16, '0'),
|
||||
$lt: prefix + '\uffff',
|
||||
},
|
||||
})
|
||||
.sort({ _id: 1 })
|
||||
.limit(limit + 1)
|
||||
.lean()
|
||||
.exec();
|
||||
const hasMore = items.length > limit;
|
||||
if (hasMore) items.pop();
|
||||
return {
|
||||
items,
|
||||
hasMore,
|
||||
nextRevision: items.at(-1)?.revision ?? afterRevision,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function parseBidArchiveEvent(
|
||||
id: string,
|
||||
raw: string | null,
|
||||
): AuctionBidRecord {
|
||||
if (raw === null) throw new Error('Missing bid archive payload.');
|
||||
const value: unknown = JSON.parse(raw);
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error('Bid archive payload must be an object.');
|
||||
}
|
||||
const event = value as Record<string, unknown>;
|
||||
const string = (field: string): string => {
|
||||
const value = event[field];
|
||||
if (typeof value !== 'string' || !value.trim()) {
|
||||
throw new Error(`Invalid bid archive ${field}.`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const integer = (field: string, minimum = 0): number => {
|
||||
const value = event[field];
|
||||
if (
|
||||
typeof value !== 'number' ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < minimum
|
||||
) {
|
||||
throw new Error(`Invalid bid archive ${field}.`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const objectId = (field: string): mongoose.Types.ObjectId => {
|
||||
const value = string(field);
|
||||
if (!mongoose.isObjectIdOrHexString(value)) {
|
||||
throw new Error(`Invalid bid archive ${field}.`);
|
||||
}
|
||||
return new mongoose.Types.ObjectId(value);
|
||||
};
|
||||
const matchId = objectId('matchId');
|
||||
const auctionLifecycleId = string('auctionLifecycleId');
|
||||
const revision = integer('revision');
|
||||
const expectedId = `${matchId.toString()}:${auctionLifecycleId}:${String(revision).padStart(16, '0')}`;
|
||||
if (event._id !== id || id !== expectedId) {
|
||||
throw new Error('Bid archive identity does not match its payload.');
|
||||
}
|
||||
return {
|
||||
_id: id,
|
||||
matchId,
|
||||
auctionLifecycleId,
|
||||
revision,
|
||||
bidRequestId: string('bidRequestId'),
|
||||
bidderUserId: objectId('bidderUserId'),
|
||||
bidderTeamIndex: integer('bidderTeamIndex', 1),
|
||||
bidPointAmount: integer('bidPointAmount'),
|
||||
bidPlacedAtTimestampMS: integer('bidPlacedAtTimestampMS'),
|
||||
...(event.playerUserId !== undefined
|
||||
? { playerUserId: objectId('playerUserId') }
|
||||
: {}),
|
||||
// Pre-upgrade rounds have no Redis round ID; the bid Lua records ''.
|
||||
...(event.bidRoundId !== undefined && event.bidRoundId !== ''
|
||||
? { bidRoundId: string('bidRoundId') }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { NotFoundException } from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionService } from './auction.service';
|
||||
|
||||
const matchId = new mongoose.Types.ObjectId();
|
||||
const communityId = new mongoose.Types.ObjectId();
|
||||
|
||||
function sessionQuery<T>(value: T) {
|
||||
return { session: async () => value };
|
||||
}
|
||||
|
||||
function fixture(options: {
|
||||
auction: Record<string, unknown> | null;
|
||||
match: Record<string, unknown>;
|
||||
failRedisCleanup?: boolean;
|
||||
}) {
|
||||
const session = {
|
||||
startTransaction: jest.fn(),
|
||||
commitTransaction: jest.fn().mockResolvedValue(undefined),
|
||||
abortTransaction: jest.fn().mockResolvedValue(undefined),
|
||||
endSession: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const deleteMany = jest.fn(() => sessionQuery(undefined));
|
||||
const models: Record<string, unknown> = {
|
||||
auction: { findOne: () => sessionQuery(options.auction) },
|
||||
auctionHistory: { deleteMany },
|
||||
auctionSideEffectApplication: { deleteMany },
|
||||
match: { findById: () => sessionQuery(options.match) },
|
||||
};
|
||||
const redisCommands: string[][] = [];
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
redisCommands.push(command);
|
||||
if (command[0] === 'EVAL' && options.failRedisCleanup)
|
||||
throw new Error('redis unavailable');
|
||||
return command[0] === 'SCAN' ? ['0', []] : 1;
|
||||
});
|
||||
const service = new AuctionService(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => session,
|
||||
} as never,
|
||||
{} as never,
|
||||
{ releaseFromMatch: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
{ cancelPolling: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
{
|
||||
acquireTokenLock: async () => true,
|
||||
releaseTokenLock: async () => true,
|
||||
raw: () => ({ sendCommand }),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
return { service, session, redisCommands };
|
||||
}
|
||||
|
||||
function liveMatch(): Record<string, unknown> {
|
||||
return {
|
||||
communityId,
|
||||
status: 'team-built',
|
||||
teams: [],
|
||||
tournamentCodeIds: [],
|
||||
gameSchedule: [],
|
||||
participants: [],
|
||||
save: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function redisAuction() {
|
||||
return {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: 'life',
|
||||
deleteOne: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
it('records the cancelled lifecycle and still succeeds when Redis cleanup fails', async () => {
|
||||
const match = liveMatch();
|
||||
const f = fixture({
|
||||
auction: redisAuction(),
|
||||
match,
|
||||
failRedisCleanup: true,
|
||||
});
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(f.session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(match.lastCancelledAuctionLifecycleId).toBe('life');
|
||||
expect(match.status).toBe('recruiting');
|
||||
});
|
||||
|
||||
it('answers a retransmitted cancellation and durably schedules Redis cleanup', async () => {
|
||||
const match = { ...liveMatch(), lastCancelledAuctionLifecycleId: 'life' };
|
||||
const f = fixture({ auction: null, match });
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(match).toMatchObject({
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: expect.any(Date),
|
||||
});
|
||||
expect(f.redisCommands).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves a committed cancellation even if its lock renewal fails before returning', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const match = liveMatch();
|
||||
const f = fixture({ auction: redisAuction(), match });
|
||||
f.session.commitTransaction.mockImplementation(async () => {
|
||||
await jest.advanceTimersByTimeAsync(10000);
|
||||
});
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(match.auctionCancellationCleanupPending).toBe(true);
|
||||
expect(f.session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps 404 for a missing auction that was not cancelled with this lifecycle', async () => {
|
||||
for (const [observed, recorded] of [
|
||||
['other', 'life'],
|
||||
[undefined, 'life'],
|
||||
['life', undefined],
|
||||
] as const) {
|
||||
const f = fixture({
|
||||
auction: null,
|
||||
match: { ...liveMatch(), lastCancelledAuctionLifecycleId: recorded },
|
||||
});
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: observed,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(f.session.abortTransaction).toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,607 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionService } from './auction.service';
|
||||
import {
|
||||
AuctionDurabilityScheduler,
|
||||
buildAuctionProjectionGuardedUpdate,
|
||||
type AuctionDurabilityPayload,
|
||||
} from './auction-durability.scheduler';
|
||||
import { AuctionDeadlineScheduler } from './auction-deadline.scheduler';
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
|
||||
const query = <T>(value: T) => ({
|
||||
select() {
|
||||
return this;
|
||||
},
|
||||
session() {
|
||||
return this;
|
||||
},
|
||||
lean() {
|
||||
return this;
|
||||
},
|
||||
exec: async () => value,
|
||||
});
|
||||
const session = () => ({
|
||||
startTransaction: jest.fn(),
|
||||
commitTransaction: jest.fn(),
|
||||
abortTransaction: jest.fn(),
|
||||
endSession: jest.fn(),
|
||||
});
|
||||
const state = {
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 12,
|
||||
phase: 'presenting',
|
||||
currentAuctionHistorySequence: 7,
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentAuctionRound: 1,
|
||||
auctionTeams: [],
|
||||
auctionPlayerOrder: [],
|
||||
passedAuctionPlayerQueue: [],
|
||||
currentHighestBidPointAmount: 0,
|
||||
currentHighestBidderTeamIndex: -1,
|
||||
auctionCompletedAtTimestampMS: 0,
|
||||
currentBidRoundDeadlineTimestampMS: 0,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest
|
||||
.spyOn(AuctionLifecycleLock.prototype, 'run')
|
||||
.mockImplementation(async (_id, work) => work());
|
||||
});
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
function matches(
|
||||
document: Record<string, unknown>,
|
||||
filter: Record<string, unknown>,
|
||||
): boolean {
|
||||
return Object.entries(filter).every(([key, value]) => {
|
||||
if (key === '$or')
|
||||
return (value as Record<string, unknown>[]).some((alternative) =>
|
||||
matches(document, alternative),
|
||||
);
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.entries(value).every(([operator, limit]) => {
|
||||
if (operator === '$exists')
|
||||
return (document[key] !== undefined) === limit;
|
||||
if (operator === '$lt') return Number(document[key]) < Number(limit);
|
||||
if (operator === '$lte') return Number(document[key]) <= Number(limit);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
return document[key] === value;
|
||||
});
|
||||
}
|
||||
function update(
|
||||
document: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) {
|
||||
Object.assign(document, mutation.$set);
|
||||
for (const [key, amount] of Object.entries(mutation.$inc ?? {}))
|
||||
document[key] = Number(document[key] ?? 0) + Number(amount);
|
||||
for (const key of Object.keys(mutation.$unset ?? {})) delete document[key];
|
||||
}
|
||||
|
||||
function sideEffectFixture(withReader = false) {
|
||||
const projection: Record<string, unknown> = {
|
||||
_id: 'auction',
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 9,
|
||||
matchSideEffectRevision: 0,
|
||||
};
|
||||
const application: Record<string, unknown> = {};
|
||||
const payload: AuctionDurabilityPayload = {
|
||||
auctionId: 'auction',
|
||||
matchId: 'match',
|
||||
auctionLifecycleId: 'life',
|
||||
auctionStateTransitionId: 'complete',
|
||||
auctionActionType: 'complete',
|
||||
auctionHistorySequence: 10,
|
||||
revision: 10,
|
||||
snapshotBefore: {},
|
||||
historyMutation: { type: 'append' },
|
||||
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
|
||||
afterAuctionMongoProjection: {
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 10,
|
||||
phase: 'complete',
|
||||
matchSideEffectRevision: 1,
|
||||
matchSideEffectStatus: 'pending',
|
||||
matchSideEffectTransitionId: 'complete',
|
||||
auctionTeams: [{ teamIndex: 1, captainUserId: 'captain', members: [] }],
|
||||
},
|
||||
};
|
||||
const AuctionModel = {
|
||||
findById: () => query({ ...projection }),
|
||||
updateOne: (
|
||||
filter: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) => {
|
||||
const matchedCount = Number(matches(projection, filter));
|
||||
if (matchedCount) update(projection, mutation);
|
||||
return query({ matchedCount });
|
||||
},
|
||||
findOneAndUpdate: (
|
||||
filter: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) => {
|
||||
if (!matches(projection, filter)) return query(null);
|
||||
update(projection, mutation);
|
||||
return query({ ...projection });
|
||||
},
|
||||
};
|
||||
const ApplicationModel = {
|
||||
findOne: () => query({ ...application }),
|
||||
updateOne: (
|
||||
filter: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) => {
|
||||
if (mutation.$setOnInsert) {
|
||||
if (!application.state)
|
||||
Object.assign(application, mutation.$setOnInsert);
|
||||
return query({ matchedCount: 1 });
|
||||
}
|
||||
const matchedCount = Number(matches(application, filter));
|
||||
if (matchedCount) update(application, mutation);
|
||||
return query({ matchedCount });
|
||||
},
|
||||
};
|
||||
const match = {
|
||||
participants: [],
|
||||
teams: [],
|
||||
save: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const historyFindOne = jest.fn().mockReturnValue(query(null));
|
||||
const historyUpdateOne = jest
|
||||
.fn()
|
||||
.mockReturnValue(query({ matchedCount: 1 }));
|
||||
const models: Record<string, unknown> = {
|
||||
auction: AuctionModel,
|
||||
auctionHistory: {
|
||||
findOne: historyFindOne,
|
||||
updateOne: historyUpdateOne,
|
||||
},
|
||||
auctionSideEffectApplication: ApplicationModel,
|
||||
match: {
|
||||
findById: () => ({ ...query(match), session: async () => match }),
|
||||
},
|
||||
};
|
||||
const reserveForMatch = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('temporary reservation error'))
|
||||
.mockResolvedValue(match);
|
||||
const scheduleInitialPoll = jest.fn().mockResolvedValue(undefined);
|
||||
const releaseFromMatch = jest.fn().mockResolvedValue(undefined);
|
||||
const cancelPolling = jest.fn().mockResolvedValue(undefined);
|
||||
let queued = true;
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGEBYSCORE') return queued ? ['payload'] : [];
|
||||
if (command[0] === 'HGETALL')
|
||||
return { payloadJson: JSON.stringify(payload) };
|
||||
if (command[0] === 'DEL') queued = false;
|
||||
if (command[0] === 'HINCRBY') return 1;
|
||||
return 1;
|
||||
});
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => session(),
|
||||
} as never,
|
||||
{
|
||||
getCurrentRedisTimestampMS: async () => 10000,
|
||||
raw: () => ({ sendCommand }),
|
||||
} as never,
|
||||
{ reserveForMatch, releaseFromMatch } as never,
|
||||
{ scheduleInitialPoll, cancelPolling } as never,
|
||||
undefined,
|
||||
undefined,
|
||||
withReader
|
||||
? ({
|
||||
readAuctionStateView: async () => ({
|
||||
auctionLifecycleId: 'life',
|
||||
phase: payload.afterAuctionMongoProjection.phase,
|
||||
matchSideEffectTransitionId: 'complete',
|
||||
}),
|
||||
} as never)
|
||||
: undefined,
|
||||
);
|
||||
return {
|
||||
scheduler,
|
||||
payload,
|
||||
projection,
|
||||
application,
|
||||
reserveForMatch,
|
||||
scheduleInitialPoll,
|
||||
sendCommand,
|
||||
match,
|
||||
historyFindOne,
|
||||
historyUpdateOne,
|
||||
releaseFromMatch,
|
||||
cancelPolling,
|
||||
};
|
||||
}
|
||||
|
||||
it('retries failed completion work with the current generation and only then deletes the payload', async () => {
|
||||
const f = sideEffectFixture();
|
||||
expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({
|
||||
failed: 1,
|
||||
});
|
||||
expect(f.projection.matchSideEffectStatus).toBe('failed');
|
||||
expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({
|
||||
applied: 1,
|
||||
});
|
||||
expect(f.reserveForMatch).toHaveBeenCalledTimes(2);
|
||||
expect(f.scheduleInitialPoll).toHaveBeenCalledTimes(1);
|
||||
expect(f.application.state).toBe('completed');
|
||||
expect(f.projection.matchSideEffectStatus).toBe('completed');
|
||||
expect(f.projection.matchSideEffectRevision).toBe(5);
|
||||
expect(f.sendCommand).toHaveBeenCalledWith(['DEL', 'payload']);
|
||||
});
|
||||
|
||||
it('keeps an outbox payload while another worker owns its side-effect lease', async () => {
|
||||
const f = sideEffectFixture();
|
||||
Object.assign(f.application, {
|
||||
auctionId: 'auction',
|
||||
auctionStateTransitionId: 'complete',
|
||||
sideEffectType: 'syncMatchTeamsAfterComplete',
|
||||
state: 'processing',
|
||||
processingOwnerToken: 'other',
|
||||
processingLeaseExpiresAtTimestampMS: 50000,
|
||||
});
|
||||
expect(await f.scheduler.processDueDurabilityPayloads(20)).toEqual({
|
||||
applied: 0,
|
||||
failed: 0,
|
||||
skipped: 1,
|
||||
});
|
||||
expect(f.sendCommand).not.toHaveBeenCalledWith(['DEL', 'payload']);
|
||||
expect(
|
||||
f.sendCommand.mock.calls.some(([command]) => command[0] === 'HINCRBY'),
|
||||
).toBe(false);
|
||||
expect(f.reserveForMatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('recovers side effects when a later ordinary projection was persisted first', async () => {
|
||||
const f = sideEffectFixture(true);
|
||||
f.reserveForMatch.mockReset().mockResolvedValue(undefined);
|
||||
Object.assign(f.projection, { revision: 12 });
|
||||
await f.scheduler.applyDurabilityPayload(f.payload);
|
||||
expect(f.reserveForMatch).toHaveBeenCalledTimes(1);
|
||||
expect(f.projection.revision).toBe(12);
|
||||
expect(f.projection.matchSideEffectTransitionId).toBe('complete');
|
||||
expect(f.projection.matchSideEffectStatus).toBe('completed');
|
||||
});
|
||||
|
||||
it('still cleans up completed Match state when a second undo invalidates the first undo history', async () => {
|
||||
const f = sideEffectFixture(true);
|
||||
Object.assign(f.projection, { revision: 12 });
|
||||
Object.assign(f.payload, {
|
||||
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
|
||||
});
|
||||
f.payload.afterAuctionMongoProjection.phase = 'presenting';
|
||||
Object.assign(f.match, {
|
||||
teams: [{ teamIndex: 1 }],
|
||||
teamBuiltAt: new Date(),
|
||||
tournamentCodeIds: ['code'],
|
||||
gameSchedule: [{}],
|
||||
});
|
||||
f.historyFindOne.mockReturnValue(
|
||||
query({
|
||||
sequence: 12,
|
||||
historyMutation: {
|
||||
type: 'deleteAfterSequenceAndAppendUndo',
|
||||
targetHistorySequence: 2,
|
||||
},
|
||||
}),
|
||||
);
|
||||
await f.scheduler.applyDurabilityPayload(f.payload);
|
||||
expect(f.historyUpdateOne).not.toHaveBeenCalled();
|
||||
expect(f.releaseFromMatch).toHaveBeenCalledTimes(1);
|
||||
expect(f.cancelPolling).toHaveBeenCalledTimes(1);
|
||||
expect(f.match).toMatchObject({
|
||||
teams: [],
|
||||
teamBuiltAt: undefined,
|
||||
tournamentCodeIds: [],
|
||||
gameSchedule: [],
|
||||
});
|
||||
expect(f.projection.matchSideEffectStatus).toBe('completed');
|
||||
});
|
||||
|
||||
it('does not let a superseded side effect overwrite the newer generation', async () => {
|
||||
const f = sideEffectFixture();
|
||||
Object.assign(f.projection, {
|
||||
revision: 12,
|
||||
matchSideEffectTransitionId: 'undo',
|
||||
matchSideEffectRevision: 6,
|
||||
});
|
||||
await f.scheduler.applyDurabilityPayload(f.payload);
|
||||
expect(f.reserveForMatch).not.toHaveBeenCalled();
|
||||
expect(f.projection.matchSideEffectTransitionId).toBe('undo');
|
||||
});
|
||||
|
||||
it('removes obsolete projection fields both from Mongo updates and subsequent undo snapshots', () => {
|
||||
const service = new AuctionService(
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const internal = service as unknown as {
|
||||
buildAuctionMongoProjectionFromState(
|
||||
auction: unknown,
|
||||
state: Record<string, unknown>,
|
||||
): Record<string, unknown>;
|
||||
buildAuctionSnapshotFromRedisState(
|
||||
auction: unknown,
|
||||
state: Record<string, unknown>,
|
||||
): Record<string, unknown>;
|
||||
};
|
||||
const old = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
matchId: new mongoose.Types.ObjectId(),
|
||||
completedAt: new Date(),
|
||||
auctionCompletedAtTimestampMS: 100,
|
||||
currentHighestBidderTeamIndex: 2,
|
||||
currentBidderTeamIndex: 2,
|
||||
};
|
||||
const projection = internal.buildAuctionMongoProjectionFromState(old, state);
|
||||
const mutation = buildAuctionProjectionGuardedUpdate({
|
||||
auctionId: old._id,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 12,
|
||||
afterAuctionMongoProjection: projection,
|
||||
} as AuctionDurabilityPayload).update;
|
||||
const stored = { ...old } as Record<string, unknown>;
|
||||
update(stored, mutation);
|
||||
const snapshot = internal.buildAuctionSnapshotFromRedisState(old, state);
|
||||
for (const record of [stored, snapshot]) {
|
||||
expect(record).not.toHaveProperty('completedAt');
|
||||
expect(record).not.toHaveProperty('currentHighestBidderTeamIndex');
|
||||
expect(record.phase).toBe('presenting');
|
||||
}
|
||||
});
|
||||
|
||||
it('applies delayed undo history cleanup without deleting newer branch entries or rolling back projection', async () => {
|
||||
const deleteMany = jest.fn().mockReturnValue(query({}));
|
||||
const updateOne = jest.fn().mockReturnValue(query({}));
|
||||
const projectionUpdate = jest.fn();
|
||||
const models: Record<string, unknown> = {
|
||||
auction: {
|
||||
findById: () => query({ auctionLifecycleId: 'life', revision: 12 }),
|
||||
updateOne: projectionUpdate,
|
||||
},
|
||||
auctionHistory: { findOne: () => query(null), deleteMany, updateOne },
|
||||
auctionSideEffectApplication: {},
|
||||
};
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => null,
|
||||
} as never,
|
||||
{} as never,
|
||||
);
|
||||
await scheduler.applyDurabilityPayload({
|
||||
auctionId: 'auction',
|
||||
matchId: 'match',
|
||||
auctionStateTransitionId: 'undo',
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 11,
|
||||
auctionHistorySequence: 9,
|
||||
auctionActionType: 'undo',
|
||||
historyMutation: {
|
||||
type: 'deleteAfterSequenceAndAppendUndo',
|
||||
targetHistorySequence: 3,
|
||||
},
|
||||
matchSideEffectMutation: { type: 'none' },
|
||||
afterAuctionMongoProjection: {},
|
||||
snapshotBefore: {},
|
||||
} as AuctionDurabilityPayload);
|
||||
expect(deleteMany).toHaveBeenCalledWith(
|
||||
{ auctionId: 'auction', sequence: { $gt: 3, $lt: 9 } },
|
||||
{ session: null },
|
||||
);
|
||||
expect(updateOne).toHaveBeenCalledTimes(1);
|
||||
expect(projectionUpdate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects a cancellation from a prior lifecycle before deleting anything', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId();
|
||||
const deleteOne = jest.fn();
|
||||
const service = new AuctionService(
|
||||
{
|
||||
createSession: async () => session(),
|
||||
getModel: async () => ({
|
||||
findOne: () => ({
|
||||
session: async () => ({
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: 'new-life',
|
||||
deleteOne,
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
await expect(
|
||||
service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'old-life',
|
||||
}),
|
||||
).rejects.toThrow('lifecycle has changed');
|
||||
expect(deleteOne).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('resumes startup after Mongo creation instead of returning an unfinished auction', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId();
|
||||
const captain = new mongoose.Types.ObjectId(),
|
||||
player = new mongoose.Types.ObjectId(),
|
||||
lateJoiner = new mongoose.Types.ObjectId();
|
||||
const auction = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
matchId,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 1,
|
||||
phase: 'presenting',
|
||||
startupCompleted: false,
|
||||
startAuctionRequestId: 'same-request',
|
||||
teams: [{ captainUserId: captain, members: [], teamIndex: 1 }],
|
||||
order: [{ userId: player }],
|
||||
};
|
||||
const match = {
|
||||
communityId,
|
||||
status: 'recruiting',
|
||||
participants: [captain, player, lateJoiner].map((userId) => ({
|
||||
userId,
|
||||
excluded: false,
|
||||
})),
|
||||
save: jest.fn(),
|
||||
};
|
||||
const updateOne = jest.fn();
|
||||
const historyUpdate = jest.fn();
|
||||
const models: Record<string, unknown> = {
|
||||
auction: { findOne: async () => auction, updateOne },
|
||||
match: { findById: async () => match },
|
||||
auctionHistory: { updateOne: historyUpdate },
|
||||
};
|
||||
const hydrateState = jest.fn().mockResolvedValue({ hydrated: true });
|
||||
const service = new AuctionService(
|
||||
{ getModel: async (name: string) => models[name] } as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{ hydrateState } as never,
|
||||
{} as never,
|
||||
);
|
||||
await service.start({
|
||||
matchId,
|
||||
communityId,
|
||||
startAuctionRequestId: 'same-request',
|
||||
} as never);
|
||||
expect(hydrateState).toHaveBeenCalledTimes(1);
|
||||
expect(historyUpdate).toHaveBeenCalledTimes(1);
|
||||
expect(match.save).toHaveBeenCalledTimes(1);
|
||||
expect(match.status).toBe('team-built');
|
||||
expect(auction.startupCompleted).toBe(true);
|
||||
expect(match.participants.map((entry) => entry.excluded)).toEqual([
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not open bidding until interrupted startup has been completed', async () => {
|
||||
const service = new AuctionService(
|
||||
{
|
||||
getModel: async () => ({
|
||||
findOne: async () => ({
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
startupCompleted: false,
|
||||
}),
|
||||
}),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
await expect(
|
||||
service.openBidding(new mongoose.Types.ObjectId()),
|
||||
).rejects.toThrow('startup is not complete');
|
||||
});
|
||||
|
||||
it('uses Redis completion state to schedule cleanup despite a delayed Mongo projection', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId();
|
||||
const auction = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
matchId,
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: 'life',
|
||||
phase: 'sold',
|
||||
revision: 9,
|
||||
};
|
||||
const snapshot = { ...state, phase: 'bidding', revision: 5 };
|
||||
const models: Record<string, unknown> = {
|
||||
auction: { findOne: () => ({ session: async () => auction }) },
|
||||
match: { findById: () => ({ session: async () => ({ communityId }) }) },
|
||||
auctionHistory: { findOne: () => query({ snapshotBefore: snapshot }) },
|
||||
};
|
||||
const applyTransition = jest.fn().mockResolvedValue({ ok: true });
|
||||
const commands = {
|
||||
reserveActionRequest: async () => ({
|
||||
reserved: true,
|
||||
actionRequestScopeKey: 'scope',
|
||||
auctionStateTransitionId: 'undo',
|
||||
observedProcessingLeaseVersion: 1,
|
||||
}),
|
||||
applyTransition,
|
||||
};
|
||||
const service = new AuctionService(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => session(),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
commands as never,
|
||||
{
|
||||
readAuctionStateView: async () => ({
|
||||
...state,
|
||||
phase: 'complete',
|
||||
revision: 10,
|
||||
}),
|
||||
} as never,
|
||||
);
|
||||
await service.undoTo({ matchId, communityId, sequence: 2 });
|
||||
const input = applyTransition.mock.calls[0][1] as {
|
||||
durabilityPayloadJson: string;
|
||||
};
|
||||
expect(JSON.parse(input.durabilityPayloadJson)).toMatchObject({
|
||||
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['deadline', 'durability'])(
|
||||
'continues %s ticks when lock release fails',
|
||||
async (kind) => {
|
||||
const redis = {
|
||||
acquireTokenLock: jest.fn().mockResolvedValue(true),
|
||||
releaseTokenLock: jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('redis disconnected')),
|
||||
};
|
||||
const scheduler =
|
||||
kind === 'deadline'
|
||||
? new AuctionDeadlineScheduler(
|
||||
redis as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
)
|
||||
: new AuctionDurabilityScheduler({} as never, redis as never);
|
||||
const work = jest.fn().mockResolvedValue({});
|
||||
Object.assign(
|
||||
scheduler,
|
||||
kind === 'deadline'
|
||||
? { processDueBidRoundDeadlines: work }
|
||||
: { processDueDurabilityPayloads: work },
|
||||
);
|
||||
const internal = scheduler as unknown as { safeTick(): Promise<void> };
|
||||
await internal.safeTick();
|
||||
await internal.safeTick();
|
||||
expect(work).toHaveBeenCalledTimes(2);
|
||||
},
|
||||
);
|
||||
@@ -88,10 +88,13 @@ export class AuctionDeadlineScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
);
|
||||
} finally {
|
||||
if (leaderLockAcquired) {
|
||||
await this.auctionRedisService.releaseTokenLock(
|
||||
BID_ROUND_DEADLINE_LEADER_LOCK_KEY,
|
||||
leaderLockToken,
|
||||
);
|
||||
await this.auctionRedisService
|
||||
.releaseTokenLock(BID_ROUND_DEADLINE_LEADER_LOCK_KEY, leaderLockToken)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Auction deadline lock release failed: ${String(error)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
this.tickInProgress = false;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
import { buildAuctionProjectionUnset } from './auction-projection';
|
||||
import {
|
||||
AuctionDurabilityScheduler,
|
||||
buildAuctionHistoryUpsert,
|
||||
@@ -23,6 +25,12 @@ const payload = {
|
||||
} as const;
|
||||
|
||||
describe('auction durability projection helpers', () => {
|
||||
beforeEach(() => {
|
||||
jest
|
||||
.spyOn(AuctionLifecycleLock.prototype, 'run')
|
||||
.mockImplementation(async (_id, work) => work());
|
||||
});
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
it('builds idempotent history upsert by auction and transition id', () => {
|
||||
expect(buildAuctionHistoryUpsert(payload)).toEqual({
|
||||
updateOne: {
|
||||
@@ -59,6 +67,9 @@ describe('auction durability projection helpers', () => {
|
||||
},
|
||||
update: {
|
||||
$set: { phase: 'sold', revision: 9 },
|
||||
$unset: buildAuctionProjectionUnset(
|
||||
payload.afterAuctionMongoProjection,
|
||||
),
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -210,7 +221,12 @@ describe('auction durability projection helpers', () => {
|
||||
auctionLifecycleId: 'life-1',
|
||||
revision: { $lt: 9 },
|
||||
},
|
||||
{ $set: { phase: 'sold', revision: 9 } },
|
||||
{
|
||||
$set: { phase: 'sold', revision: 9 },
|
||||
$unset: buildAuctionProjectionUnset(
|
||||
payload.afterAuctionMongoProjection,
|
||||
),
|
||||
},
|
||||
{ session },
|
||||
);
|
||||
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
@@ -370,7 +386,7 @@ describe('auction durability projection helpers', () => {
|
||||
expect(historyDeleteMany).toHaveBeenCalledWith(
|
||||
{
|
||||
auctionId: 'auction-1',
|
||||
sequence: { $gt: 2 },
|
||||
sequence: { $gt: 2, $lt: 4 },
|
||||
},
|
||||
{ session },
|
||||
);
|
||||
@@ -811,21 +827,14 @@ describe('auction durability projection helpers', () => {
|
||||
skipped: 0,
|
||||
});
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(5, [
|
||||
'ZREM',
|
||||
'EVAL',
|
||||
expect.stringContaining("redis.call('HEXISTS'"),
|
||||
'4',
|
||||
'payload-1',
|
||||
'auction:auctionDurabilityRetryQueue',
|
||||
'payload-1',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(6, [
|
||||
'HSET',
|
||||
'payload-1',
|
||||
'deadLetteredAtRedisTimestampMS',
|
||||
'1000',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(7, [
|
||||
'ZADD',
|
||||
'auction:auctionDurabilityDeadLetterQueue',
|
||||
'auction:deadLetterStatusQueue',
|
||||
'1000',
|
||||
'payload-1',
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -1030,11 +1039,20 @@ describe('auction durability projection helpers', () => {
|
||||
{
|
||||
_id: 'auction-1',
|
||||
auctionLifecycleId: 'life-1',
|
||||
matchSideEffectRevision: 1,
|
||||
$or: [
|
||||
{ matchSideEffectTransitionId: 'transition-1' },
|
||||
{
|
||||
matchSideEffectTransitionId: { $exists: false },
|
||||
matchSideEffectRevision: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
$inc: { matchSideEffectRevision: 1 },
|
||||
$set: { matchSideEffectStatus: 'processing' },
|
||||
$set: {
|
||||
matchSideEffectStatus: 'processing',
|
||||
matchSideEffectTransitionId: 'transition-1',
|
||||
},
|
||||
$unset: { matchSideEffectError: '' },
|
||||
},
|
||||
expect.objectContaining({ returnDocument: 'after' }),
|
||||
@@ -1045,34 +1063,42 @@ describe('auction durability projection helpers', () => {
|
||||
_id: 'auction-1',
|
||||
auctionLifecycleId: 'life-1',
|
||||
matchSideEffectRevision: 2,
|
||||
matchSideEffectTransitionId: 'transition-1',
|
||||
},
|
||||
{
|
||||
$inc: { matchSideEffectRevision: 1 },
|
||||
$set: { matchSideEffectStatus: 'completed' },
|
||||
$set: {
|
||||
matchSideEffectStatus: 'completed',
|
||||
matchSideEffectTransitionId: 'transition-1',
|
||||
},
|
||||
$unset: { matchSideEffectError: '' },
|
||||
},
|
||||
expect.objectContaining({ returnDocument: 'after' }),
|
||||
);
|
||||
expect(mirrorMatchSideEffectStatus).toHaveBeenCalledWith({
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
matchId: 'match-1',
|
||||
matchSideEffectRevision: 2,
|
||||
matchSideEffectStatus: 'processing',
|
||||
});
|
||||
expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
matchId: 'match-1',
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
});
|
||||
expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(1, 'match-1', {
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
currentServerTimestampMS: 2000,
|
||||
matchSideEffectRevision: 2,
|
||||
matchSideEffectStatus: 'processing',
|
||||
});
|
||||
expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(2, 'match-1', {
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
currentServerTimestampMS: 3000,
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
@@ -1080,7 +1106,7 @@ describe('auction durability projection helpers', () => {
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining("redis.call('EXPIRE'"),
|
||||
'11',
|
||||
'12',
|
||||
'auction:match-1:liveAuctionState',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'auction:match-1:auctionStructureState',
|
||||
@@ -1092,6 +1118,7 @@ describe('auction durability projection helpers', () => {
|
||||
'auction:match-1:auctionActionRequestResultOrder',
|
||||
'auction:match-1:rehydrateLock',
|
||||
'auction:match-1:auctionStateTransitionLock',
|
||||
'auction:match-1:rejectedBidRequestResultOrder',
|
||||
'match-1',
|
||||
'life-1',
|
||||
String(7 * 24 * 60 * 60),
|
||||
@@ -1263,7 +1290,12 @@ describe('auction durability projection helpers', () => {
|
||||
const broadcastMatchSideEffect = jest.fn();
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
{
|
||||
createSession: jest.fn().mockResolvedValue(null),
|
||||
createSession: jest.fn().mockResolvedValueOnce(null).mockResolvedValue({
|
||||
startTransaction: jest.fn(),
|
||||
commitTransaction: jest.fn(),
|
||||
abortTransaction: jest.fn(),
|
||||
endSession: jest.fn(),
|
||||
}),
|
||||
getModel: jest.fn((name: string) => {
|
||||
if (name === 'auctionHistory') {
|
||||
return {
|
||||
@@ -1282,7 +1314,11 @@ describe('auction durability projection helpers', () => {
|
||||
return { updateOne: sideEffectUpdateOne };
|
||||
}
|
||||
if (name === 'match')
|
||||
return { findById: jest.fn().mockReturnValue(match) };
|
||||
return {
|
||||
findById: jest.fn().mockReturnValue({
|
||||
session: jest.fn().mockResolvedValue(match),
|
||||
}),
|
||||
};
|
||||
return {};
|
||||
}),
|
||||
} as never,
|
||||
@@ -1301,8 +1337,8 @@ describe('auction durability projection helpers', () => {
|
||||
|
||||
await scheduler.applyDurabilityPayload(resetPayload);
|
||||
|
||||
expect(releaseFromMatch).toHaveBeenCalledWith(match);
|
||||
expect(cancelPolling).toHaveBeenCalledWith(match);
|
||||
expect(releaseFromMatch).toHaveBeenCalledWith(match, expect.any(Object));
|
||||
expect(cancelPolling).toHaveBeenCalledWith(match, expect.any(Object));
|
||||
expect(match).toMatchObject({
|
||||
gameSchedule: [],
|
||||
result: undefined,
|
||||
@@ -1328,12 +1364,14 @@ describe('auction durability projection helpers', () => {
|
||||
);
|
||||
expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
matchId: 'match-1',
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
});
|
||||
expect(broadcastMatchSideEffect).toHaveBeenLastCalledWith('match-1', {
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
currentServerTimestampMS: 3000,
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
Optional,
|
||||
ConflictException,
|
||||
} from '@nestjs/common';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import type mongoose from 'mongoose';
|
||||
@@ -16,6 +17,10 @@ import type {
|
||||
import { TournamentCodeService } from '../tournament-code/tournament-code.service';
|
||||
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||
import { AuctionEventBusService } from './auction-event-bus.service';
|
||||
import { AuctionService } from './auction.service';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
import { buildAuctionProjectionUnset } from './auction-projection';
|
||||
import {
|
||||
AuctionSideEffectService,
|
||||
buildMatchSideEffectDelta,
|
||||
@@ -26,6 +31,7 @@ import { AuctionRedisService } from './redis/auction-redis.service';
|
||||
import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service';
|
||||
import {
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
buildAuctionRedisKeys,
|
||||
} from './redis/auction-redis.keys';
|
||||
@@ -36,6 +42,8 @@ const AUCTION_DURABILITY_BATCH_LIMIT = 20;
|
||||
const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000;
|
||||
const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000;
|
||||
const AUCTION_DURABILITY_MAX_ATTEMPTS = 10;
|
||||
const AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS = 120;
|
||||
const AUCTION_DURABILITY_MAX_PENDING_DURATION_MS = 10 * 60 * 1000;
|
||||
const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000;
|
||||
const AUCTION_DURABILITY_LEADER_LOCK_KEY =
|
||||
'auction:auctionDurabilityScheduler:leader';
|
||||
@@ -45,6 +53,7 @@ const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000;
|
||||
const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete';
|
||||
const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset';
|
||||
const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects';
|
||||
class SideEffectPendingError extends Error {}
|
||||
|
||||
export interface AuctionDurabilityPayload {
|
||||
actorUserId?: unknown;
|
||||
@@ -67,6 +76,7 @@ export interface AuctionDurabilityPayload {
|
||||
export interface AuctionProjectionIdentity {
|
||||
auctionLifecycleId?: string;
|
||||
revision?: number;
|
||||
matchSideEffectTransitionId?: string;
|
||||
}
|
||||
|
||||
type ParsedAuctionHistoryMutation =
|
||||
@@ -85,7 +95,7 @@ interface CleanupSideEffectMatchDocument {
|
||||
participants: Array<{ excluded?: boolean }>;
|
||||
result?: unknown;
|
||||
resultPollState?: unknown;
|
||||
save(): Promise<unknown>;
|
||||
save(options?: { session: mongoose.ClientSession }): Promise<unknown>;
|
||||
status: string;
|
||||
teamBuiltAt?: Date;
|
||||
teams?: unknown[];
|
||||
@@ -99,6 +109,7 @@ export class AuctionDurabilityScheduler
|
||||
private readonly logger = new Logger(AuctionDurabilityScheduler.name);
|
||||
private tickTimer?: NodeJS.Timeout;
|
||||
private tickInProgress = false;
|
||||
private deadLetterScanCursor = '0';
|
||||
|
||||
constructor(
|
||||
private readonly connectionService: ConnectionService,
|
||||
@@ -113,6 +124,9 @@ export class AuctionDurabilityScheduler
|
||||
private readonly auctionEventBusService?: AuctionEventBusService,
|
||||
@Optional()
|
||||
private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService,
|
||||
@Optional() private readonly auctionService?: AuctionService,
|
||||
@Optional()
|
||||
private readonly auctionBidArchiveService?: AuctionBidArchiveService,
|
||||
) {}
|
||||
|
||||
public onModuleInit(): void {
|
||||
@@ -187,6 +201,23 @@ export class AuctionDurabilityScheduler
|
||||
});
|
||||
await this.clearDurabilityPayload(payloadKey);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof SideEffectPendingError ||
|
||||
error instanceof ConflictException
|
||||
) {
|
||||
const disposition = await this.deferDurabilityPayload(
|
||||
payloadKey,
|
||||
currentRedisTimestampMS,
|
||||
error,
|
||||
);
|
||||
if (disposition === 'dead-lettered') {
|
||||
failed += 1;
|
||||
this.logger.warn(
|
||||
`Auction durability payload exceeded its pending limit: ${payloadKey}`,
|
||||
);
|
||||
} else skipped += 1;
|
||||
continue;
|
||||
}
|
||||
failed += 1;
|
||||
const failureDisposition = await this.recordDurabilityPayloadFailure(
|
||||
payloadKey,
|
||||
@@ -214,6 +245,14 @@ export class AuctionDurabilityScheduler
|
||||
const payloadJson = hash.payloadJson;
|
||||
if (!payloadJson) return null;
|
||||
const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload;
|
||||
const committedDeadline = Number(hash.committedBidRoundDeadlineTimestampMS);
|
||||
if (Number.isFinite(committedDeadline) && committedDeadline > 0) {
|
||||
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS =
|
||||
committedDeadline;
|
||||
payload.afterAuctionMongoProjection.currentDeadline = new Date(
|
||||
committedDeadline,
|
||||
).toISOString();
|
||||
}
|
||||
const committedAtRedisTimestampMS = Number(
|
||||
hash.committedAtRedisTimestampMS,
|
||||
);
|
||||
@@ -240,12 +279,12 @@ export class AuctionDurabilityScheduler
|
||||
const sideEffectMutation = parseMatchSideEffectMutation(
|
||||
payload.matchSideEffectMutation,
|
||||
);
|
||||
let shouldUpdateProjection = true;
|
||||
let shouldApplySideEffects = true;
|
||||
let shouldApplyHistoryMutation = true;
|
||||
|
||||
try {
|
||||
session?.startTransaction();
|
||||
const historyMutation = parseAuctionHistoryMutation(
|
||||
payload.historyMutation,
|
||||
);
|
||||
if (typeof AuctionModel.findById === 'function') {
|
||||
const currentProjection =
|
||||
await readAuctionProjectionIdentityForMutation(
|
||||
@@ -256,37 +295,44 @@ export class AuctionDurabilityScheduler
|
||||
if (
|
||||
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
|
||||
) {
|
||||
// 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다.
|
||||
// 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다.
|
||||
if (
|
||||
historyMutation.type === 'append' &&
|
||||
currentProjection.auctionLifecycleId === payload.auctionLifecycleId
|
||||
currentProjection.auctionLifecycleId !== payload.auctionLifecycleId
|
||||
) {
|
||||
await this.applyStaleAppendHistoryRecovery({
|
||||
if (session) await session.commitTransaction();
|
||||
return;
|
||||
}
|
||||
shouldApplyHistoryMutation =
|
||||
!(await this.isAppendHistoryInvalidatedByLaterDestructiveMutation({
|
||||
AuctionHistoryModel,
|
||||
payload,
|
||||
session,
|
||||
});
|
||||
}
|
||||
if (session) await session.commitTransaction();
|
||||
return;
|
||||
}));
|
||||
// History operations have their own sequence bounds. A newer ordinary
|
||||
// projection must not discard an earlier undo's history cleanup.
|
||||
shouldUpdateProjection = false;
|
||||
shouldApplySideEffects =
|
||||
Boolean(this.auctionRedisStateReaderService) ||
|
||||
currentProjection.matchSideEffectTransitionId ===
|
||||
payload.auctionStateTransitionId;
|
||||
}
|
||||
}
|
||||
await this.applyAuctionHistoryMutation({
|
||||
AuctionHistoryModel,
|
||||
AuctionModel,
|
||||
payload,
|
||||
session,
|
||||
});
|
||||
if (shouldApplyHistoryMutation)
|
||||
await this.applyAuctionHistoryMutation({
|
||||
AuctionHistoryModel,
|
||||
AuctionModel,
|
||||
payload,
|
||||
session,
|
||||
});
|
||||
const projectionOperation = buildAuctionProjectionGuardedUpdate(payload);
|
||||
await execMaybe(
|
||||
AuctionModel.updateOne(
|
||||
projectionOperation.filter as never,
|
||||
projectionOperation.update as never,
|
||||
{ session } as never,
|
||||
),
|
||||
);
|
||||
if (sideEffectMutation) {
|
||||
if (shouldUpdateProjection)
|
||||
await execMaybe(
|
||||
AuctionModel.updateOne(
|
||||
projectionOperation.filter as never,
|
||||
projectionOperation.update as never,
|
||||
{ session } as never,
|
||||
),
|
||||
);
|
||||
if (sideEffectMutation && shouldApplySideEffects) {
|
||||
const sideEffectOperation = buildSideEffectApplicationUpsert(
|
||||
payload,
|
||||
sideEffectMutation,
|
||||
@@ -312,7 +358,7 @@ export class AuctionDurabilityScheduler
|
||||
|
||||
await this.broadcastLatestRedisAuctionStateIfRequested(payload);
|
||||
|
||||
if (sideEffectMutation) {
|
||||
if (sideEffectMutation && shouldApplySideEffects) {
|
||||
await this.applyMatchSideEffect(payload, sideEffectMutation);
|
||||
}
|
||||
}
|
||||
@@ -352,6 +398,64 @@ export class AuctionDurabilityScheduler
|
||||
private async applyMatchSideEffect(
|
||||
payload: AuctionDurabilityPayload,
|
||||
mutation: ParsedMatchSideEffectMutation,
|
||||
): Promise<void> {
|
||||
return new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
String(payload.matchId),
|
||||
async () => {
|
||||
if (this.auctionRedisStateReaderService) {
|
||||
const state =
|
||||
await this.auctionRedisStateReaderService.readAuctionStateView(
|
||||
String(payload.matchId),
|
||||
);
|
||||
if (!state)
|
||||
throw new SideEffectPendingError(
|
||||
'Auction realtime state is unavailable.',
|
||||
);
|
||||
if (
|
||||
state.auctionLifecycleId !== payload.auctionLifecycleId ||
|
||||
(state.matchSideEffectTransitionId &&
|
||||
state.matchSideEffectTransitionId !==
|
||||
payload.auctionStateTransitionId) ||
|
||||
(mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE &&
|
||||
state.phase !== 'complete')
|
||||
)
|
||||
return;
|
||||
// An ordinary later projection may have reached Mongo before this
|
||||
// side-effect-bearing transition. Install its pending generation only
|
||||
// while Redis still names it and Mongo has not advanced beyond it.
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
const pendingRevision = Number(
|
||||
payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
|
||||
);
|
||||
await execMaybe(
|
||||
AuctionModel.updateOne(
|
||||
{
|
||||
_id: payload.auctionId,
|
||||
auctionLifecycleId: payload.auctionLifecycleId,
|
||||
$or: [
|
||||
{ matchSideEffectRevision: { $lt: pendingRevision } },
|
||||
{ matchSideEffectRevision: { $exists: false } },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
matchSideEffectTransitionId: payload.auctionStateTransitionId,
|
||||
matchSideEffectRevision: pendingRevision,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
$unset: { matchSideEffectError: '' },
|
||||
} as never,
|
||||
),
|
||||
);
|
||||
}
|
||||
await this.applyMatchSideEffectWhileLocked(payload, mutation);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async applyMatchSideEffectWhileLocked(
|
||||
payload: AuctionDurabilityPayload,
|
||||
mutation: ParsedMatchSideEffectMutation,
|
||||
): Promise<void> {
|
||||
if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) {
|
||||
await this.applySyncMatchTeamsSideEffect(payload, mutation);
|
||||
@@ -407,7 +511,26 @@ export class AuctionDurabilityScheduler
|
||||
}) as never,
|
||||
),
|
||||
);
|
||||
if (!hasMatchedDocument(leaseResult)) return;
|
||||
if (!hasMatchedDocument(leaseResult)) {
|
||||
const existing = await AuctionSideEffectApplicationModel.findOne(
|
||||
identity as never,
|
||||
)
|
||||
.lean()
|
||||
.exec();
|
||||
if (existing?.state === 'completed') {
|
||||
await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
payload,
|
||||
status: 'completed',
|
||||
});
|
||||
await this.expireCompletedAuctionRedisRuntime(payload);
|
||||
return;
|
||||
}
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side effect is still owned by another worker.',
|
||||
);
|
||||
}
|
||||
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
|
||||
{
|
||||
@@ -420,7 +543,11 @@ export class AuctionDurabilityScheduler
|
||||
status: 'processing',
|
||||
},
|
||||
);
|
||||
if (!processingDelta) return;
|
||||
if (!processingDelta) {
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side-effect generation is not ready.',
|
||||
);
|
||||
}
|
||||
const expectedMatchSideEffectRevision =
|
||||
processingDelta.matchSideEffectRevision;
|
||||
|
||||
@@ -489,7 +616,11 @@ export class AuctionDurabilityScheduler
|
||||
await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
completedAtTimestampMS,
|
||||
@@ -518,7 +649,11 @@ export class AuctionDurabilityScheduler
|
||||
await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
lastError: error instanceof Error ? error.message : String(error),
|
||||
@@ -544,33 +679,6 @@ export class AuctionDurabilityScheduler
|
||||
}
|
||||
}
|
||||
|
||||
private async applyStaleAppendHistoryRecovery(params: {
|
||||
AuctionHistoryModel: {
|
||||
findOne?: (...args: never[]) => unknown;
|
||||
updateOne: (...args: never[]) => unknown;
|
||||
};
|
||||
payload: AuctionDurabilityPayload;
|
||||
session: mongoose.ClientSession | null | undefined;
|
||||
}): Promise<void> {
|
||||
const invalidated =
|
||||
await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params);
|
||||
if (invalidated) return;
|
||||
|
||||
const historyOperation = buildAuctionHistoryUpsert(
|
||||
params.payload,
|
||||
).updateOne;
|
||||
await execMaybe(
|
||||
params.AuctionHistoryModel.updateOne(
|
||||
historyOperation.filter as never,
|
||||
historyOperation.update as never,
|
||||
{
|
||||
session: params.session,
|
||||
upsert: true,
|
||||
} as never,
|
||||
) as Promise<unknown> | { exec(): Promise<unknown> },
|
||||
);
|
||||
}
|
||||
|
||||
private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: {
|
||||
AuctionHistoryModel: {
|
||||
findOne?: (...args: never[]) => unknown;
|
||||
@@ -622,9 +730,7 @@ export class AuctionDurabilityScheduler
|
||||
await this.applyMatchCleanupSideEffect({
|
||||
mutation,
|
||||
payload,
|
||||
updateMatch: async (match) => {
|
||||
await this.tournamentCodeService?.releaseFromMatch(match as never);
|
||||
await this.tournamentResultService?.cancelPolling(match as never);
|
||||
updateMatch: (match) => {
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
@@ -635,7 +741,6 @@ export class AuctionDurabilityScheduler
|
||||
match.teamBuiltAt = undefined;
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -647,13 +752,14 @@ export class AuctionDurabilityScheduler
|
||||
await this.applyMatchCleanupSideEffect({
|
||||
mutation,
|
||||
payload,
|
||||
updateMatch: async (match) => {
|
||||
await this.tournamentCodeService?.releaseFromMatch(match as never);
|
||||
await this.tournamentResultService?.cancelPolling(match as never);
|
||||
updateMatch: (match) => {
|
||||
match.status = 'team-built';
|
||||
match.teams = [];
|
||||
match.teamBuiltAt = undefined;
|
||||
match.tournamentCodeIds = [];
|
||||
match.gameSchedule = [];
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -661,7 +767,7 @@ export class AuctionDurabilityScheduler
|
||||
private async applyMatchCleanupSideEffect(params: {
|
||||
mutation: ParsedMatchSideEffectMutation;
|
||||
payload: AuctionDurabilityPayload;
|
||||
updateMatch: (match: CleanupSideEffectMatchDocument) => Promise<void>;
|
||||
updateMatch: (match: CleanupSideEffectMatchDocument) => void;
|
||||
}): Promise<void> {
|
||||
if (!this.tournamentCodeService || !this.tournamentResultService) {
|
||||
throw new Error('Auction side-effect services are not available.');
|
||||
@@ -700,7 +806,25 @@ export class AuctionDurabilityScheduler
|
||||
}) as never,
|
||||
),
|
||||
);
|
||||
if (!hasMatchedDocument(leaseResult)) return;
|
||||
if (!hasMatchedDocument(leaseResult)) {
|
||||
const existing = await AuctionSideEffectApplicationModel.findOne(
|
||||
identity as never,
|
||||
)
|
||||
.lean()
|
||||
.exec();
|
||||
if (existing?.state === 'completed') {
|
||||
await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
payload: params.payload,
|
||||
status: 'completed',
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side effect is still owned by another worker.',
|
||||
);
|
||||
}
|
||||
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
|
||||
{
|
||||
@@ -714,29 +838,56 @@ export class AuctionDurabilityScheduler
|
||||
status: 'processing',
|
||||
},
|
||||
);
|
||||
if (!processingDelta) return;
|
||||
if (!processingDelta) {
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side-effect generation is not ready.',
|
||||
);
|
||||
}
|
||||
const expectedMatchSideEffectRevision =
|
||||
processingDelta.matchSideEffectRevision;
|
||||
|
||||
try {
|
||||
const match = (await execMaybe(
|
||||
MatchModel.findById(params.payload.matchId) as never,
|
||||
)) as CleanupSideEffectMatchDocument | null;
|
||||
if (!match) {
|
||||
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
|
||||
}
|
||||
if (match.competitionId) {
|
||||
throw new Error(
|
||||
'Cannot reset or undo Match teams after the immutable Competition has been created.',
|
||||
const session = await this.connectionService.createSession();
|
||||
if (!session)
|
||||
throw new Error('Failed to start Match cleanup transaction.');
|
||||
try {
|
||||
session.startTransaction();
|
||||
const match = (await MatchModel.findById(
|
||||
params.payload.matchId,
|
||||
).session(session)) as CleanupSideEffectMatchDocument | null;
|
||||
if (!match)
|
||||
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
|
||||
if (match.competitionId)
|
||||
throw new Error(
|
||||
'Cannot reset or undo Match teams after the immutable Competition has been created.',
|
||||
);
|
||||
await this.tournamentCodeService.releaseFromMatch(
|
||||
match as never,
|
||||
session,
|
||||
);
|
||||
await this.tournamentResultService.cancelPolling(
|
||||
match as never,
|
||||
session,
|
||||
);
|
||||
params.updateMatch(match);
|
||||
await match.save({ session });
|
||||
await session.commitTransaction();
|
||||
} catch (error) {
|
||||
await session.abortTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
await params.updateMatch(match);
|
||||
|
||||
const completedAtTimestampMS =
|
||||
await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
completedAtTimestampMS,
|
||||
@@ -766,7 +917,11 @@ export class AuctionDurabilityScheduler
|
||||
error instanceof Error ? error.message : String(error);
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
lastError: errorMessage,
|
||||
@@ -828,7 +983,10 @@ export class AuctionDurabilityScheduler
|
||||
params.AuctionHistoryModel.deleteMany(
|
||||
{
|
||||
auctionId: params.payload.auctionId,
|
||||
sequence: { $gt: mutation.targetHistorySequence },
|
||||
sequence: {
|
||||
$gt: mutation.targetHistorySequence,
|
||||
$lt: params.payload.auctionHistorySequence,
|
||||
},
|
||||
} as never,
|
||||
{ session: params.session } as never,
|
||||
) as Promise<unknown> | { exec(): Promise<unknown> },
|
||||
@@ -842,6 +1000,7 @@ export class AuctionDurabilityScheduler
|
||||
params.AuctionHistoryModel.deleteMany(
|
||||
{
|
||||
auctionId: params.payload.auctionId,
|
||||
sequence: { $lt: params.payload.auctionHistorySequence },
|
||||
} as never,
|
||||
{ session: params.session } as never,
|
||||
) as Promise<unknown> | { exec(): Promise<unknown> },
|
||||
@@ -874,16 +1033,43 @@ export class AuctionDurabilityScheduler
|
||||
};
|
||||
currentServerTimestampMS: number;
|
||||
errorMessage?: string;
|
||||
expectedMatchSideEffectRevision: number;
|
||||
expectedMatchSideEffectRevision?: number;
|
||||
payload: AuctionDurabilityPayload;
|
||||
status: AuctionMatchSideEffectStatus;
|
||||
onlyUnfinished?: boolean;
|
||||
}): Promise<MatchSideEffectDelta | null> {
|
||||
const updatedAuction = (await execMaybe(
|
||||
params.AuctionModel.findOneAndUpdate(
|
||||
{
|
||||
_id: params.payload.auctionId,
|
||||
auctionLifecycleId: params.payload.auctionLifecycleId,
|
||||
matchSideEffectRevision: params.expectedMatchSideEffectRevision,
|
||||
...(params.onlyUnfinished
|
||||
? { matchSideEffectStatus: { $ne: 'completed' } }
|
||||
: {}),
|
||||
...(params.status === 'processing'
|
||||
? {
|
||||
$or: [
|
||||
{
|
||||
matchSideEffectTransitionId:
|
||||
params.payload.auctionStateTransitionId,
|
||||
},
|
||||
{
|
||||
matchSideEffectTransitionId: { $exists: false },
|
||||
matchSideEffectRevision:
|
||||
params.expectedMatchSideEffectRevision,
|
||||
},
|
||||
],
|
||||
}
|
||||
: {
|
||||
matchSideEffectTransitionId:
|
||||
params.payload.auctionStateTransitionId,
|
||||
...(params.expectedMatchSideEffectRevision === undefined
|
||||
? {}
|
||||
: {
|
||||
matchSideEffectRevision:
|
||||
params.expectedMatchSideEffectRevision,
|
||||
}),
|
||||
}),
|
||||
} as never,
|
||||
{
|
||||
$inc: { matchSideEffectRevision: 1 },
|
||||
@@ -892,6 +1078,8 @@ export class AuctionDurabilityScheduler
|
||||
? { matchSideEffectError: params.errorMessage }
|
||||
: {}),
|
||||
matchSideEffectStatus: params.status,
|
||||
matchSideEffectTransitionId:
|
||||
params.payload.auctionStateTransitionId,
|
||||
},
|
||||
...(params.errorMessage
|
||||
? {}
|
||||
@@ -915,6 +1103,7 @@ export class AuctionDurabilityScheduler
|
||||
|
||||
const delta = buildMatchSideEffectDelta({
|
||||
auctionLifecycleId: updatedAuction.auctionLifecycleId,
|
||||
auctionStateTransitionId: params.payload.auctionStateTransitionId,
|
||||
currentServerTimestampMS: params.currentServerTimestampMS,
|
||||
matchSideEffectRevision: Number(
|
||||
updatedAuction.matchSideEffectRevision ?? 0,
|
||||
@@ -923,19 +1112,170 @@ export class AuctionDurabilityScheduler
|
||||
updatedAuction.matchSideEffectStatus ?? params.status,
|
||||
});
|
||||
|
||||
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
|
||||
auctionLifecycleId: delta.auctionLifecycleId,
|
||||
matchId: String(params.payload.matchId),
|
||||
matchSideEffectRevision: delta.matchSideEffectRevision,
|
||||
matchSideEffectStatus: delta.matchSideEffectStatus,
|
||||
});
|
||||
this.auctionEventBusService?.broadcastMatchSideEffect(
|
||||
String(params.payload.matchId),
|
||||
delta,
|
||||
);
|
||||
const mirrored =
|
||||
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
|
||||
auctionLifecycleId: delta.auctionLifecycleId,
|
||||
auctionStateTransitionId: params.payload.auctionStateTransitionId,
|
||||
matchId: String(params.payload.matchId),
|
||||
matchSideEffectRevision: delta.matchSideEffectRevision,
|
||||
matchSideEffectStatus: delta.matchSideEffectStatus,
|
||||
});
|
||||
if (mirrored !== false) {
|
||||
this.auctionEventBusService?.broadcastMatchSideEffect(
|
||||
String(params.payload.matchId),
|
||||
delta,
|
||||
);
|
||||
}
|
||||
return delta;
|
||||
}
|
||||
|
||||
public async reconcileDeadLetterStatuses(limit = 20): Promise<void> {
|
||||
const client = this.auctionRedisService.raw();
|
||||
const now = await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
// Also reconcile entries dead-lettered before status notifications existed.
|
||||
const scanned = await client.sendCommand([
|
||||
'ZSCAN',
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
this.deadLetterScanCursor,
|
||||
'COUNT',
|
||||
String(limit),
|
||||
]);
|
||||
if (Array.isArray(scanned) && Array.isArray(scanned[1])) {
|
||||
this.deadLetterScanCursor = String(scanned[0]);
|
||||
for (let index = 0; index < scanned[1].length; index += 2) {
|
||||
const key = String(scanned[1][index]);
|
||||
if (
|
||||
!(await client.sendCommand([
|
||||
'HGET',
|
||||
key,
|
||||
'deadLetterStatusReconciledAtTimestampMS',
|
||||
]))
|
||||
) {
|
||||
await client.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
'NX',
|
||||
String(now),
|
||||
key,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
const reply = await client.sendCommand([
|
||||
'ZRANGEBYSCORE',
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
'-inf',
|
||||
String(now),
|
||||
'LIMIT',
|
||||
'0',
|
||||
String(limit),
|
||||
]);
|
||||
if (!Array.isArray(reply)) return;
|
||||
for (const key of reply.map(String)) {
|
||||
try {
|
||||
const payload = await this.readDurabilityPayload(key);
|
||||
if (
|
||||
payload &&
|
||||
parseMatchSideEffectMutation(payload.matchSideEffectMutation)
|
||||
) {
|
||||
const state =
|
||||
await this.auctionRedisStateReaderService?.readAuctionStateView(
|
||||
String(payload.matchId),
|
||||
);
|
||||
// A terminal notification for a previous generation must not change
|
||||
// a new completion/undo, even when Mongo is behind Redis.
|
||||
const superseded =
|
||||
state &&
|
||||
(state.auctionLifecycleId !== payload.auctionLifecycleId ||
|
||||
state.matchSideEffectStatus === 'completed' ||
|
||||
(state.matchSideEffectTransitionId &&
|
||||
state.matchSideEffectTransitionId !==
|
||||
payload.auctionStateTransitionId));
|
||||
if (!superseded) {
|
||||
const AuctionModel =
|
||||
await this.connectionService.getModel('auction');
|
||||
if (
|
||||
state?.matchSideEffectTransitionId ===
|
||||
payload.auctionStateTransitionId
|
||||
) {
|
||||
const revision = Number(
|
||||
payload.afterAuctionMongoProjection.matchSideEffectRevision ??
|
||||
0,
|
||||
);
|
||||
await AuctionModel.updateOne(
|
||||
{
|
||||
_id: payload.auctionId,
|
||||
auctionLifecycleId: payload.auctionLifecycleId,
|
||||
$or: [
|
||||
{ matchSideEffectRevision: { $lt: revision } },
|
||||
{ matchSideEffectRevision: { $exists: false } },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
matchSideEffectTransitionId:
|
||||
payload.auctionStateTransitionId,
|
||||
matchSideEffectRevision: revision,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
},
|
||||
);
|
||||
// Heal a same-generation revision mismatch before sending the
|
||||
// terminal status; otherwise Redis could ignore it as older.
|
||||
await AuctionModel.updateOne(
|
||||
{
|
||||
_id: payload.auctionId,
|
||||
auctionLifecycleId: payload.auctionLifecycleId,
|
||||
matchSideEffectTransitionId: payload.auctionStateTransitionId,
|
||||
matchSideEffectStatus: { $ne: 'completed' },
|
||||
} as never,
|
||||
{
|
||||
$max: {
|
||||
matchSideEffectRevision: Number(
|
||||
state.matchSideEffectRevision ?? revision,
|
||||
),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
const error = await client.sendCommand(['HGET', key, 'lastError']);
|
||||
await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel: AuctionModel as never,
|
||||
payload,
|
||||
currentServerTimestampMS: now,
|
||||
status: 'failed',
|
||||
onlyUnfinished: true,
|
||||
errorMessage:
|
||||
typeof error === 'string'
|
||||
? error
|
||||
: 'Auction side effect exhausted its retry limit.',
|
||||
});
|
||||
}
|
||||
}
|
||||
await client.sendCommand([
|
||||
'EVAL',
|
||||
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 1 then redis.call('HSET', KEYS[1], 'deadLetterStatusReconciledAtTimestampMS', ARGV[1]) end redis.call('ZREM', KEYS[2], KEYS[1]); return 1",
|
||||
'2',
|
||||
key,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(now),
|
||||
]);
|
||||
} catch (error) {
|
||||
// Mongo or Redis mirroring may be unavailable at dead-letter time. Keep
|
||||
// the notification durable until the failure can be shown to viewers.
|
||||
await client.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(now + 5000),
|
||||
key,
|
||||
]);
|
||||
this.logger.warn(
|
||||
`Auction terminal status will retry for ${key}: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async safeTick(): Promise<void> {
|
||||
if (this.tickInProgress) return;
|
||||
this.tickInProgress = true;
|
||||
@@ -948,7 +1288,22 @@ export class AuctionDurabilityScheduler
|
||||
AUCTION_DURABILITY_LEADER_LOCK_TTL_MS,
|
||||
);
|
||||
if (!leaderLockAcquired) return;
|
||||
await this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT);
|
||||
// A failed maintenance task must not starve the other durable queues.
|
||||
for (const work of [
|
||||
() => this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT),
|
||||
() => this.reconcileDeadLetterStatuses(AUCTION_DURABILITY_BATCH_LIMIT),
|
||||
() => this.auctionBidArchiveService?.persistPendingBids(),
|
||||
() =>
|
||||
this.auctionService?.retryPendingCancellations(
|
||||
AUCTION_DURABILITY_BATCH_LIMIT,
|
||||
),
|
||||
]) {
|
||||
try {
|
||||
await work();
|
||||
} catch (error) {
|
||||
this.logger.warn(`Auction maintenance task failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Auction durability tick failed: ${
|
||||
@@ -957,10 +1312,13 @@ export class AuctionDurabilityScheduler
|
||||
);
|
||||
} finally {
|
||||
if (leaderLockAcquired) {
|
||||
await this.auctionRedisService.releaseTokenLock(
|
||||
AUCTION_DURABILITY_LEADER_LOCK_KEY,
|
||||
leaderLockToken,
|
||||
);
|
||||
await this.auctionRedisService
|
||||
.releaseTokenLock(AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Auction durability lock release failed: ${String(error)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
this.tickInProgress = false;
|
||||
}
|
||||
@@ -987,6 +1345,51 @@ export class AuctionDurabilityScheduler
|
||||
]);
|
||||
}
|
||||
|
||||
private async deferDurabilityPayload(
|
||||
payloadKey: string,
|
||||
timestampMS: number,
|
||||
error: unknown,
|
||||
): Promise<'gone' | 'requeued' | 'dead-lettered'> {
|
||||
// Keep coordination waits separate from failed work, but bound them across
|
||||
// restarts too. Updating the payload and queue atomically avoids orphan jobs.
|
||||
const result = await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'EVAL',
|
||||
[
|
||||
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end",
|
||||
"local count = redis.call('HINCRBY', KEYS[1], 'pendingAttempts', 1)",
|
||||
"redis.call('HSETNX', KEYS[1], 'pendingSinceTimestampMS', ARGV[1])",
|
||||
"local since = tonumber(redis.call('HGET', KEYS[1], 'pendingSinceTimestampMS'))",
|
||||
"redis.call('HSET', KEYS[1], 'lastPendingReason', ARGV[5], 'lastPendingAtTimestampMS', ARGV[1])",
|
||||
'if count >= tonumber(ARGV[3]) or tonumber(ARGV[1]) - since >= tonumber(ARGV[4]) then',
|
||||
" redis.call('ZREM', KEYS[2], KEYS[1])",
|
||||
" redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1], 'lastError', ARGV[5])",
|
||||
" redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1])",
|
||||
" redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1])",
|
||||
' return 2',
|
||||
'end',
|
||||
"redis.call('ZADD', KEYS[2], tonumber(ARGV[1]) + tonumber(ARGV[2]), KEYS[1])",
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'4',
|
||||
payloadKey,
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(timestampMS),
|
||||
String(AUCTION_DURABILITY_RETRY_DELAY_MS),
|
||||
String(AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS),
|
||||
String(AUCTION_DURABILITY_MAX_PENDING_DURATION_MS),
|
||||
formatErrorMessageForRedis(error),
|
||||
]);
|
||||
return Number(result) === 2
|
||||
? 'dead-lettered'
|
||||
: Number(result) === 1
|
||||
? 'requeued'
|
||||
: 'gone';
|
||||
}
|
||||
|
||||
private async recordDurabilityPayloadFailure(
|
||||
payloadKey: string,
|
||||
currentRedisTimestampMS: number,
|
||||
@@ -1012,24 +1415,17 @@ export class AuctionDurabilityScheduler
|
||||
Number.isFinite(attempts) &&
|
||||
attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS
|
||||
) {
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'HSET',
|
||||
'EVAL',
|
||||
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1]); redis.call('ZREM', KEYS[2], KEYS[1]); redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1]); redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1]); return 1",
|
||||
'4',
|
||||
payloadKey,
|
||||
'deadLetteredAtRedisTimestampMS',
|
||||
String(currentRedisTimestampMS),
|
||||
]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(currentRedisTimestampMS),
|
||||
payloadKey,
|
||||
]);
|
||||
return 'dead-lettered';
|
||||
}
|
||||
@@ -1064,7 +1460,7 @@ export class AuctionDurabilityScheduler
|
||||
'end',
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'11',
|
||||
'12',
|
||||
keys.liveStateKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.structureStateKey,
|
||||
@@ -1076,6 +1472,7 @@ export class AuctionDurabilityScheduler
|
||||
keys.actionRequestResultOrderKey,
|
||||
keys.rehydrateLockKey,
|
||||
keys.stateTransitionLockKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
matchId,
|
||||
payload.auctionLifecycleId,
|
||||
String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS),
|
||||
@@ -1159,6 +1556,7 @@ async function readAuctionProjectionIdentityForMutation(
|
||||
? (query as { select: (...args: never[]) => unknown }).select({
|
||||
auctionLifecycleId: 1,
|
||||
revision: 1,
|
||||
matchSideEffectTransitionId: 1,
|
||||
} as never)
|
||||
: query;
|
||||
const scoped =
|
||||
@@ -1178,6 +1576,10 @@ async function readAuctionProjectionIdentityForMutation(
|
||||
if (!projection || typeof projection !== 'object') return {};
|
||||
const record = projection as Record<string, unknown>;
|
||||
return {
|
||||
matchSideEffectTransitionId:
|
||||
typeof record.matchSideEffectTransitionId === 'string'
|
||||
? record.matchSideEffectTransitionId
|
||||
: undefined,
|
||||
auctionLifecycleId:
|
||||
typeof record.auctionLifecycleId === 'string'
|
||||
? record.auctionLifecycleId
|
||||
@@ -1200,6 +1602,7 @@ export function buildAuctionProjectionGuardedUpdate(
|
||||
},
|
||||
update: {
|
||||
$set: payload.afterAuctionMongoProjection,
|
||||
$unset: buildAuctionProjectionUnset(payload.afterAuctionMongoProjection),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -24,4 +24,10 @@ export class AuctionEventBusService {
|
||||
?.to(matchId)
|
||||
.emit('state', serializeRealtimeAuction(state));
|
||||
}
|
||||
|
||||
public broadcastReset(matchId: string, auctionLifecycleId: string): void {
|
||||
this.auctionNamespace
|
||||
?.to(matchId)
|
||||
.emit('reset', { matchId, auctionLifecycleId });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
|
||||
afterEach(() => jest.useRealTimers());
|
||||
|
||||
it('renews ownership and handles renewal/release failures without unhandled rejection', async () => {
|
||||
jest.useFakeTimers();
|
||||
const sendCommand = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('redis disconnected'));
|
||||
const releaseTokenLock = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('release failed'));
|
||||
const lock = new AuctionLifecycleLock({
|
||||
acquireTokenLock: jest.fn().mockResolvedValue(true),
|
||||
releaseTokenLock,
|
||||
raw: () => ({ sendCommand }),
|
||||
} as never);
|
||||
const result = lock.run('match', async () => {
|
||||
await jest.advanceTimersByTimeAsync(10000);
|
||||
});
|
||||
await expect(result).rejects.toThrow('redis disconnected');
|
||||
expect(releaseTokenLock).toHaveBeenCalledTimes(1);
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('preserves an operation result if only lock release fails', async () => {
|
||||
const lock = new AuctionLifecycleLock({
|
||||
acquireTokenLock: jest.fn().mockResolvedValue(true),
|
||||
releaseTokenLock: jest.fn().mockRejectedValue(new Error('release failed')),
|
||||
} as never);
|
||||
await expect(lock.run('match', async () => 42)).resolves.toBe(42);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { ConflictException, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { AuctionRedisService } from './redis/auction-redis.service';
|
||||
|
||||
export class AuctionLifecycleBusyError extends ConflictException {
|
||||
constructor() {
|
||||
super({
|
||||
code: 'AUCTION_LIFECYCLE_BUSY',
|
||||
message: '다른 경매 작업을 처리 중입니다. 잠시 후 다시 시도해 주세요.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Serializes startup, cancellation, undo and Match side effects across nodes. */
|
||||
export class AuctionLifecycleLock {
|
||||
private readonly logger = new Logger(AuctionLifecycleLock.name);
|
||||
|
||||
constructor(private readonly redis: AuctionRedisService) {}
|
||||
|
||||
async run<T>(matchId: string, operation: () => Promise<T>): Promise<T> {
|
||||
// Kept outside runtime cleanup: cancelling an auction must not release a
|
||||
// lock still held by its cancellation/recovery operation.
|
||||
const key = `auction:${matchId}:lifecycleMutationLock`;
|
||||
const token = randomUUID();
|
||||
if (!(await this.redis.acquireTokenLock(key, token, 30_000))) {
|
||||
throw new AuctionLifecycleBusyError();
|
||||
}
|
||||
let renewal: Promise<void> | undefined;
|
||||
let ownershipError: Error | undefined;
|
||||
const timer = setInterval(() => {
|
||||
if (renewal) return;
|
||||
renewal = this.redis
|
||||
.raw()
|
||||
.sendCommand<string | null>([
|
||||
'SET',
|
||||
key,
|
||||
token,
|
||||
'IFEQ',
|
||||
token,
|
||||
'PX',
|
||||
'30000',
|
||||
])
|
||||
.then((result) => {
|
||||
if (result !== 'OK')
|
||||
throw new Error('Auction lifecycle lock ownership was lost.');
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
ownershipError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
})
|
||||
.finally(() => {
|
||||
renewal = undefined;
|
||||
});
|
||||
}, 10_000);
|
||||
timer.unref?.();
|
||||
try {
|
||||
const result = await operation();
|
||||
clearInterval(timer);
|
||||
await renewal;
|
||||
if (ownershipError) throw ownershipError;
|
||||
return result;
|
||||
} finally {
|
||||
clearInterval(timer);
|
||||
await renewal;
|
||||
await this.redis.releaseTokenLock(key, token).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Auction lifecycle lock release failed: ${String(error)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
/** Missing optional fields in a complete projection mean deletion, not retention. */
|
||||
export const AUCTION_OPTIONAL_PROJECTION_FIELDS = [
|
||||
'completedAt',
|
||||
'auctionCompletedAtTimestampMS',
|
||||
'currentBidderTeamIndex',
|
||||
'currentHighestBidderTeamIndex',
|
||||
'currentDeadline',
|
||||
'currentBidRoundDeadlineTimestampMS',
|
||||
'currentBidRoundId',
|
||||
] as const;
|
||||
|
||||
export function buildAuctionProjectionUnset(
|
||||
projection: Record<string, unknown>,
|
||||
) {
|
||||
return Object.fromEntries(
|
||||
AUCTION_OPTIONAL_PROJECTION_FIELDS.filter(
|
||||
(field) => projection[field] === undefined,
|
||||
).map((field) => [field, '']),
|
||||
);
|
||||
}
|
||||
@@ -5,6 +5,11 @@ import {
|
||||
} from './auction-rejection';
|
||||
|
||||
describe('resolveAuctionRejectionReason', () => {
|
||||
it('distinguishes an outdated client from a stale round and does not suggest a state-only resync', () => {
|
||||
const reason = resolveAuctionRejectionReason('BID_CLIENT_UPDATE_REQUIRED');
|
||||
expect(reason).toBe('BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED');
|
||||
expect(isAuctionResyncRecommended(reason)).toBe(false);
|
||||
});
|
||||
it('reports roster validation failures without recommending a retry', () => {
|
||||
const reason = resolveAuctionRejectionReason(
|
||||
'AUCTION_ROSTER_INVALID',
|
||||
|
||||
@@ -46,6 +46,8 @@ export const AUCTION_REJECTION_REASON = {
|
||||
'AUCTION_TRANSITION_REJECTED_BECAUSE_REQUEST_COORDINATION_RETRYABLE',
|
||||
|
||||
// === 입찰(bid) ===
|
||||
BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED:
|
||||
'BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED',
|
||||
/** 입찰 라운드가 이미 다음 선수/다음 라운드로 넘어감(내부 `STALE_BID_ROUND`). */
|
||||
BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED:
|
||||
'BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED',
|
||||
@@ -86,6 +88,8 @@ export type AuctionRejectionReason =
|
||||
|
||||
/** 내부 짧은 코드 → 설명적 reason. */
|
||||
const INTERNAL_CODE_TO_REASON: Record<string, AuctionRejectionReason> = {
|
||||
BID_CLIENT_UPDATE_REQUIRED:
|
||||
AUCTION_REJECTION_REASON.BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED,
|
||||
AUCTION_ROSTER_INVALID:
|
||||
AUCTION_REJECTION_REASON.AUCTION_TRANSITION_REJECTED_BECAUSE_ROSTER_INVALID,
|
||||
CONFLICT:
|
||||
|
||||
@@ -96,6 +96,7 @@ describe('auction side effect helpers', () => {
|
||||
'life-1',
|
||||
'2',
|
||||
'completed',
|
||||
'',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -25,6 +25,7 @@ export interface BuildSideEffectLeaseAcquireUpdateParams {
|
||||
|
||||
export interface MatchSideEffectDelta {
|
||||
auctionLifecycleId: string;
|
||||
auctionStateTransitionId?: string;
|
||||
currentServerTimestampMS: number;
|
||||
matchSideEffectRevision: number;
|
||||
matchSideEffectStatus: AuctionMatchSideEffectStatus;
|
||||
@@ -36,6 +37,7 @@ export class AuctionSideEffectService {
|
||||
|
||||
public async mirrorMatchSideEffectStatus(params: {
|
||||
auctionLifecycleId: string;
|
||||
auctionStateTransitionId?: string;
|
||||
matchId: string;
|
||||
matchSideEffectRevision: number;
|
||||
matchSideEffectStatus: AuctionMatchSideEffectStatus;
|
||||
@@ -47,6 +49,8 @@ export class AuctionSideEffectService {
|
||||
'EVAL',
|
||||
[
|
||||
"if redis.call('HGET', KEYS[1], 'auctionLifecycleId') ~= ARGV[1] then return 0 end",
|
||||
"local generation = redis.call('HGET', KEYS[1], 'matchSideEffectTransitionId')",
|
||||
"if ARGV[4] ~= '' and generation and generation ~= ARGV[4] then return 0 end",
|
||||
"local currentRevision = tonumber(redis.call('HGET', KEYS[1], 'matchSideEffectRevision') or '-1')",
|
||||
'local nextRevision = tonumber(ARGV[2])',
|
||||
'if currentRevision >= nextRevision then return 0 end',
|
||||
@@ -58,6 +62,7 @@ export class AuctionSideEffectService {
|
||||
params.auctionLifecycleId,
|
||||
String(params.matchSideEffectRevision),
|
||||
params.matchSideEffectStatus,
|
||||
params.auctionStateTransitionId ?? '',
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
@@ -104,6 +109,9 @@ export function buildMatchSideEffectDelta(
|
||||
): MatchSideEffectDelta {
|
||||
return {
|
||||
auctionLifecycleId: params.auctionLifecycleId,
|
||||
...(params.auctionStateTransitionId
|
||||
? { auctionStateTransitionId: params.auctionStateTransitionId }
|
||||
: {}),
|
||||
currentServerTimestampMS: params.currentServerTimestampMS,
|
||||
matchSideEffectRevision: params.matchSideEffectRevision,
|
||||
matchSideEffectStatus: params.matchSideEffectStatus,
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
Param,
|
||||
Post,
|
||||
UseGuards,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import { AllowPublicAccess } from '../auth/allow-public-access.decorator';
|
||||
@@ -19,6 +20,7 @@ import { ParseObjectIdPipe } from '../common/parse-object-id.pipe';
|
||||
import type { UserDocument } from '../global/database/schema';
|
||||
import { AuctionService } from './auction.service';
|
||||
import { AuctionGateway } from './auction.gateway';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
import {
|
||||
NextAuctionDto,
|
||||
OpenBiddingDto,
|
||||
@@ -35,6 +37,7 @@ export class AuctionController {
|
||||
constructor(
|
||||
private readonly auctionService: AuctionService,
|
||||
private readonly auctionGateway: AuctionGateway,
|
||||
private readonly auctionBidArchiveService: AuctionBidArchiveService,
|
||||
) {}
|
||||
|
||||
@Post()
|
||||
@@ -78,6 +81,20 @@ export class AuctionController {
|
||||
}
|
||||
|
||||
// 공개 — 팀장 2명만 쓸 엔드포인트(트러스트 기반, 10명 동시 관전).
|
||||
@Get('bids/history')
|
||||
@AllowPublicAccess()
|
||||
public async getBidHistory(
|
||||
@Param('matchId', ParseObjectIdPipe) matchId: mongoose.Types.ObjectId,
|
||||
@Query('afterRevision') afterRevision?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
return this.auctionBidArchiveService.list(
|
||||
matchId,
|
||||
afterRevision === undefined ? -1 : Number(afterRevision),
|
||||
limit === undefined ? 100 : Number(limit),
|
||||
);
|
||||
}
|
||||
|
||||
@Post('open-bidding')
|
||||
@AllowPublicAccess()
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@@ -100,6 +117,7 @@ export class AuctionController {
|
||||
teamIndex: body.bidderTeamIndex,
|
||||
amount: body.bidPointAmount,
|
||||
bidRequestId: body.bidRequestId,
|
||||
clientObservedBidRoundId: body.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId: body.clientObservedAuctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId:
|
||||
body.clientObservedCurrentBidRoundPlayerUserId,
|
||||
@@ -213,7 +231,10 @@ export class AuctionController {
|
||||
clientObservedPhase: body.clientObservedPhase,
|
||||
clientObservedRevision: body.clientObservedRevision,
|
||||
});
|
||||
this.auctionGateway.broadcastReset(matchId.toString());
|
||||
this.auctionGateway.broadcastReset(
|
||||
matchId.toString(),
|
||||
body.clientObservedAuctionLifecycleId,
|
||||
);
|
||||
}
|
||||
|
||||
private assertSameCommunity(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ServiceUnavailableException } from '@nestjs/common';
|
||||
|
||||
jest.mock('../auth/jwt.service', () => ({
|
||||
JwtService: class JwtService {},
|
||||
@@ -7,6 +8,76 @@ jest.mock('../auth/jwt.service', () => ({
|
||||
import { AuctionGateway } from './auction.gateway';
|
||||
|
||||
describe('AuctionGateway', () => {
|
||||
it('signals pending initial state so the client can retry privately after recovery', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId().toString();
|
||||
const gateway = new AuctionGateway(
|
||||
{
|
||||
getByMatchId: async () => ({}),
|
||||
getStateByMatchId: async () => {
|
||||
throw new ServiceUnavailableException({
|
||||
code: 'AUCTION_STATE_RECOVERING',
|
||||
message: 'recovering',
|
||||
});
|
||||
},
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() };
|
||||
await gateway.handleJoin(socket as never, { matchId });
|
||||
expect(socket.emit).toHaveBeenCalledTimes(1);
|
||||
expect(socket.emit).toHaveBeenCalledWith('state-pending', {
|
||||
retryAfterMS: 500,
|
||||
});
|
||||
});
|
||||
it('sends join and resync snapshots only to the requesting socket and coalesces in-flight resyncs', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId().toString();
|
||||
const getStateByMatchId = jest.fn().mockResolvedValue({ revision: 3 });
|
||||
const gateway = new AuctionGateway(
|
||||
{
|
||||
getByMatchId: async () => ({ chatMessages: [] }),
|
||||
getStateByMatchId,
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const broadcast = jest.fn();
|
||||
(gateway as unknown as { server: unknown }).server = {
|
||||
to: () => ({ emit: broadcast }),
|
||||
};
|
||||
const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() };
|
||||
await gateway.handleJoin(socket as never, { matchId });
|
||||
expect(socket.emit).toHaveBeenCalledWith('state', { revision: 3 });
|
||||
expect(broadcast).not.toHaveBeenCalled();
|
||||
let finish!: (value: unknown) => void;
|
||||
getStateByMatchId.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const ack = jest.fn(),
|
||||
busyAck = jest.fn();
|
||||
const pending = gateway.handleResync(socket as never, {}, ack);
|
||||
await gateway.handleResync(socket as never, {}, busyAck);
|
||||
expect(busyAck).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ code: 'RESYNC_BUSY' }),
|
||||
);
|
||||
finish({ revision: 4 });
|
||||
await pending;
|
||||
expect(ack).toHaveBeenCalledWith({ ok: true, state: { revision: 4 } });
|
||||
expect(getStateByMatchId).toHaveBeenCalledTimes(2);
|
||||
expect(broadcast).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
function createGateway(params?: {
|
||||
getCurrentRedisTimestampMS?: jest.Mock;
|
||||
placeBidDelta?: jest.Mock;
|
||||
|
||||
@@ -44,6 +44,7 @@ interface JoinAuctionPayload {
|
||||
}
|
||||
|
||||
interface BidPayload {
|
||||
clientObservedBidRoundId?: string;
|
||||
bidderTeamIndex: number;
|
||||
bidPointAmount: number;
|
||||
bidRequestId?: string;
|
||||
@@ -79,6 +80,8 @@ interface BidRateLimitState {
|
||||
}
|
||||
|
||||
interface AuctionSocketData {
|
||||
auctionResyncAtTimestampMS?: number;
|
||||
auctionResyncInProgress?: boolean;
|
||||
auctionBidRateLimit?: BidRateLimitState;
|
||||
identity?: IdentityDocument;
|
||||
matchId?: string;
|
||||
@@ -162,9 +165,55 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
|
||||
new mongoose.Types.ObjectId(payload.matchId),
|
||||
);
|
||||
this.emitChatHistory(socket, auction);
|
||||
this.broadcastState(payload.matchId, auctionState);
|
||||
socket.emit('state', serializeRealtimeAuction(auctionState));
|
||||
} catch (error) {
|
||||
this.emitKnownError(socket, error);
|
||||
const rejection = this.resolveErrorPayload(error);
|
||||
if (rejection.code === 'AUCTION_STATE_RECOVERING') {
|
||||
socket.emit('state-pending', { retryAfterMS: 500 });
|
||||
} else {
|
||||
this.emitKnownError(socket, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SubscribeMessage('resync')
|
||||
public async handleResync(
|
||||
@ConnectedSocket() socket: AuctionSocket,
|
||||
@MessageBody() _payload: unknown,
|
||||
@Ack() ack?: (response: Record<string, unknown>) => void,
|
||||
) {
|
||||
const matchId = this.requireJoinedMatchId(socket);
|
||||
if (!matchId) {
|
||||
ack?.({ ok: false, code: 'JOIN_REQUIRED' });
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const retryAfterMS =
|
||||
500 - (now - (socket.data.auctionResyncAtTimestampMS ?? 0));
|
||||
if (socket.data.auctionResyncInProgress || retryAfterMS > 0) {
|
||||
ack?.({
|
||||
ok: false,
|
||||
code: 'RESYNC_BUSY',
|
||||
retryAfterMS: Math.max(500, retryAfterMS),
|
||||
});
|
||||
return;
|
||||
}
|
||||
socket.data.auctionResyncInProgress = true;
|
||||
socket.data.auctionResyncAtTimestampMS = now;
|
||||
try {
|
||||
const state = await this.auctionService.getStateByMatchId(
|
||||
new mongoose.Types.ObjectId(matchId),
|
||||
);
|
||||
// Only the requesting socket receives the full snapshot.
|
||||
ack?.({ ok: true, state: serializeRealtimeAuction(state) });
|
||||
} catch (error) {
|
||||
ack?.({
|
||||
...this.resolveErrorPayload(error),
|
||||
ok: false,
|
||||
retryAfterMS: 500,
|
||||
});
|
||||
} finally {
|
||||
socket.data.auctionResyncInProgress = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -196,6 +245,7 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
|
||||
teamIndex: payload.bidderTeamIndex,
|
||||
amount: payload.bidPointAmount,
|
||||
bidRequestId,
|
||||
clientObservedBidRoundId: payload.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId:
|
||||
payload.clientObservedAuctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId:
|
||||
@@ -609,8 +659,11 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
|
||||
this.server.to(matchId).emit('bid', delta);
|
||||
}
|
||||
|
||||
public broadcastReset(matchId: string) {
|
||||
this.server.to(matchId).emit('reset', { matchId });
|
||||
public broadcastReset(matchId: string, auctionLifecycleId?: string) {
|
||||
this.server.to(matchId).emit('reset', {
|
||||
matchId,
|
||||
...(auctionLifecycleId ? { auctionLifecycleId } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
public broadcastMatchSideEffect(
|
||||
|
||||
@@ -10,6 +10,7 @@ import { AuctionEventBusService } from './auction-event-bus.service';
|
||||
import { AuctionRedisModule } from './redis/auction-redis.module';
|
||||
import { AuctionSideEffectService } from './auction-side-effect.service';
|
||||
import { AuctionService } from './auction.service';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -21,6 +22,7 @@ import { AuctionService } from './auction.service';
|
||||
controllers: [AuctionController],
|
||||
providers: [
|
||||
AuctionService,
|
||||
AuctionBidArchiveService,
|
||||
AuctionGateway,
|
||||
AuctionEventBusService,
|
||||
AuctionDeadlineScheduler,
|
||||
|
||||
@@ -45,6 +45,10 @@ describe('AuctionService', () => {
|
||||
const allModels = {
|
||||
auctionHistory: auctionHistoryModel,
|
||||
...models,
|
||||
auction: {
|
||||
updateOne: jest.fn().mockResolvedValue({ matchedCount: 1 }),
|
||||
...((models.auction as object) ?? {}),
|
||||
},
|
||||
};
|
||||
const connectionService = {
|
||||
getModel: jest.fn((name: string) => allModels[name]),
|
||||
@@ -697,7 +701,7 @@ describe('AuctionService', () => {
|
||||
expect(auctionRedisService.raw().sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining('currentAuctionLifecycleId'),
|
||||
'11',
|
||||
'12',
|
||||
`auction:${matchId.toString()}:liveAuctionState`,
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
`auction:${matchId.toString()}:auctionStructureState`,
|
||||
@@ -709,6 +713,7 @@ describe('AuctionService', () => {
|
||||
`auction:${matchId.toString()}:auctionActionRequestResultOrder`,
|
||||
`auction:${matchId.toString()}:rehydrateLock`,
|
||||
`auction:${matchId.toString()}:auctionStateTransitionLock`,
|
||||
`auction:${matchId.toString()}:rejectedBidRequestResultOrder`,
|
||||
matchId.toString(),
|
||||
'old-life-1',
|
||||
]);
|
||||
@@ -909,9 +914,9 @@ describe('AuctionService', () => {
|
||||
shouldForceAuctionStateResync: true,
|
||||
});
|
||||
expect(auctionRedisService.acquireTokenLock).toHaveBeenCalledWith(
|
||||
`auction:${matchId.toString()}:rehydrateLock`,
|
||||
`auction:${matchId.toString()}:lifecycleMutationLock`,
|
||||
expect.any(String),
|
||||
5000,
|
||||
30000,
|
||||
);
|
||||
expect(auctionRedisCommandService.hydrateState).toHaveBeenCalledWith(
|
||||
matchId.toString(),
|
||||
@@ -2621,8 +2626,14 @@ describe('AuctionService', () => {
|
||||
auctionRedisCommandService.reserveActionRequest,
|
||||
).not.toHaveBeenCalled();
|
||||
expect(auctionRedisCommandService.applyTransition).not.toHaveBeenCalled();
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match);
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(match).toMatchObject({
|
||||
gameSchedule: [],
|
||||
result: undefined,
|
||||
@@ -2642,13 +2653,12 @@ describe('AuctionService', () => {
|
||||
auctionId,
|
||||
});
|
||||
expect(auction.deleteOne).toHaveBeenCalledWith({ session });
|
||||
expect(redisClient.sendCommand).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
matchId.toString(),
|
||||
'life-1',
|
||||
]),
|
||||
);
|
||||
expect(redisClient.sendCommand).not.toHaveBeenCalled();
|
||||
expect(match).toMatchObject({
|
||||
lastCancelledAuctionLifecycleId: 'life-1',
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: expect.any(Date),
|
||||
});
|
||||
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -2732,8 +2742,14 @@ describe('AuctionService', () => {
|
||||
service.undoTo({ matchId, communityId, sequence: 3 }),
|
||||
).resolves.toBe(restored);
|
||||
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match);
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(match.status).toBe('team-built');
|
||||
expect(match.result).toBeUndefined();
|
||||
expect(auctionReplaceOne).toHaveBeenCalledWith(
|
||||
|
||||
@@ -5,8 +5,16 @@ import {
|
||||
InternalServerErrorException,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
ServiceUnavailableException,
|
||||
Optional,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import {
|
||||
AuctionLifecycleLock,
|
||||
AuctionLifecycleBusyError,
|
||||
} from './auction-lifecycle-lock';
|
||||
import { AUCTION_OPTIONAL_PROJECTION_FIELDS } from './auction-projection';
|
||||
import { AuctionEventBusService } from './auction-event-bus.service';
|
||||
import { performance } from 'perf_hooks';
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
@@ -58,8 +66,6 @@ const AUCTION_BID_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000;
|
||||
const AUCTION_MAX_STORED_ACTION_REQUEST_RESULT_COUNT = 500;
|
||||
const AUCTION_ACTION_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000;
|
||||
const AUCTION_ACTION_REQUEST_LEASE_DURATION_MS = 15 * 1000;
|
||||
const AUCTION_REHYDRATE_LOCK_TTL_MS = 5 * 1000;
|
||||
const AUCTION_REHYDRATE_LOCK_WAIT_MS = 50;
|
||||
const AUCTION_HISTORY_LIST_LIMIT = 100;
|
||||
const AUCTION_HISTORY_LIST_CACHE_TTL_MS = 2 * 1000;
|
||||
const AUCTION_HISTORY_LIST_CACHE_MAX_ENTRIES = 256;
|
||||
@@ -82,6 +88,7 @@ interface StartAuctionParams {
|
||||
}
|
||||
|
||||
interface PlaceBidParams {
|
||||
clientObservedBidRoundId?: string;
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
teamIndex: number;
|
||||
amount: number;
|
||||
@@ -160,6 +167,7 @@ interface AuctionHistoryListCacheEntry {
|
||||
export type AuctionStateResponse = AuctionDocument | Record<string, unknown>;
|
||||
|
||||
export interface AuctionBidDeltaResponse {
|
||||
currentBidRoundId?: string;
|
||||
auctionBidHistoryEntry: Record<string, unknown>;
|
||||
auctionLifecycleId?: string;
|
||||
bidServerTiming?: AuctionBidServerTiming;
|
||||
@@ -212,9 +220,20 @@ export class AuctionService {
|
||||
private readonly auctionRedisService: AuctionRedisService,
|
||||
private readonly auctionRedisCommandService: AuctionRedisCommandService,
|
||||
private readonly auctionRedisStateReaderService: AuctionRedisStateReaderService,
|
||||
@Optional()
|
||||
private readonly auctionEventBusService?: AuctionEventBusService,
|
||||
) {}
|
||||
|
||||
public async start(params: StartAuctionParams): Promise<AuctionDocument> {
|
||||
return new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
params.matchId.toString(),
|
||||
() => this.startLocked(params),
|
||||
);
|
||||
}
|
||||
|
||||
private async startLocked(
|
||||
params: StartAuctionParams,
|
||||
): Promise<AuctionDocument> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const UserModel = await this.connectionService.getModel('user');
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
@@ -231,7 +250,7 @@ export class AuctionService {
|
||||
const existing = await AuctionModel.findOne({ matchId: params.matchId });
|
||||
if (existing) {
|
||||
if (existing.startAuctionRequestId === params.startAuctionRequestId) {
|
||||
return existing;
|
||||
return this.finishAuctionStartup(existing, match);
|
||||
}
|
||||
if (this.isRestartableResetRedisAuction(existing)) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
@@ -249,6 +268,19 @@ export class AuctionService {
|
||||
}
|
||||
}
|
||||
|
||||
// No Mongo auction owns this match. Remove runtime left by an interrupted
|
||||
// cancellation before creating a new lifecycle (under the lifecycle lock).
|
||||
const orphanedRuntime =
|
||||
await this.auctionRedisStateReaderService.readLiveStateIdentity(
|
||||
params.matchId.toString(),
|
||||
);
|
||||
if (orphanedRuntime?.auctionLifecycleId) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
params.matchId.toString(),
|
||||
orphanedRuntime.auctionLifecycleId,
|
||||
);
|
||||
}
|
||||
|
||||
const activeParticipants = this.applyAuctionCut(match);
|
||||
const desiredAuctionCount = activeParticipants.length;
|
||||
if (desiredAuctionCount < 10) {
|
||||
@@ -381,6 +413,7 @@ export class AuctionService {
|
||||
auctionStartedByUserId: params.actorUserId,
|
||||
matchSideEffectStatus: 'none',
|
||||
matchSideEffectRevision: 0,
|
||||
startupCompleted: false,
|
||||
teams: legacyTeams,
|
||||
order: prioritizedOrder,
|
||||
passedQueue: [],
|
||||
@@ -419,12 +452,12 @@ export class AuctionService {
|
||||
matchId: params.matchId,
|
||||
});
|
||||
if (raceWinner?.startAuctionRequestId === params.startAuctionRequestId) {
|
||||
return raceWinner;
|
||||
return this.finishAuctionStartup(raceWinner, match);
|
||||
}
|
||||
throw new BadRequestException('Auction already exists for this match.');
|
||||
}
|
||||
await this.recordSnapshot(auction, 'start');
|
||||
await this.auctionRedisCommandService.hydrateState(
|
||||
const hydration = await this.auctionRedisCommandService.hydrateState(
|
||||
params.matchId.toString(),
|
||||
{
|
||||
auctionHistorySequence: 1,
|
||||
@@ -437,9 +470,18 @@ export class AuctionService {
|
||||
structureStateJson,
|
||||
},
|
||||
);
|
||||
if ('rejected' in hydration)
|
||||
throw new ConflictException(
|
||||
`Auction initialization failed: ${hydration.code}.`,
|
||||
);
|
||||
|
||||
match.status = 'team-built';
|
||||
await match.save();
|
||||
auction.startupCompleted = true;
|
||||
await AuctionModel.updateOne(
|
||||
{ _id: auction._id },
|
||||
{ $set: { startupCompleted: true } },
|
||||
);
|
||||
this.enqueueBackgroundOrderEnrichment({
|
||||
auctionId: auction._id,
|
||||
targets: enrichTargets,
|
||||
@@ -450,6 +492,61 @@ export class AuctionService {
|
||||
return auction;
|
||||
}
|
||||
|
||||
private async finishAuctionStartup(
|
||||
auction: AuctionDocument,
|
||||
match: MatchDocument,
|
||||
): Promise<AuctionDocument> {
|
||||
// Documents predating the startup marker already completed the old path.
|
||||
if (auction.startupCompleted !== false) return auction;
|
||||
// Retry the original cut; recruiting may have changed since Mongo creation.
|
||||
const roster = new Set([
|
||||
...auction.teams.map((team) => team.captainUserId.toString()),
|
||||
...auction.order.map((player) => player.userId.toString()),
|
||||
]);
|
||||
const participants = new Set(
|
||||
match.participants.map((entry) => entry.userId.toString()),
|
||||
);
|
||||
if ([...roster].some((id) => !participants.has(id))) {
|
||||
throw new ConflictException(
|
||||
'Auction startup roster has changed. Cancel and restart the auction.',
|
||||
);
|
||||
}
|
||||
const History = await this.connectionService.getModel('auctionHistory');
|
||||
await History.updateOne(
|
||||
{ auctionId: auction._id, sequence: 1 },
|
||||
{
|
||||
$setOnInsert: {
|
||||
auctionId: auction._id,
|
||||
matchId: auction.matchId,
|
||||
sequence: 1,
|
||||
action: 'start',
|
||||
snapshotBefore: this.cloneJson(
|
||||
typeof auction.toObject === 'function'
|
||||
? auction.toObject()
|
||||
: auction,
|
||||
),
|
||||
takenAt: auction.startedAt ?? new Date(),
|
||||
},
|
||||
},
|
||||
{ upsert: true },
|
||||
);
|
||||
await this.hydrateRedisStateFromAuctionDocument(auction);
|
||||
if (match.status === 'recruiting') {
|
||||
match.participants.forEach((participant) => {
|
||||
participant.excluded = !roster.has(participant.userId.toString());
|
||||
});
|
||||
match.status = 'team-built';
|
||||
await match.save();
|
||||
}
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
await AuctionModel.updateOne(
|
||||
{ _id: auction._id },
|
||||
{ $set: { startupCompleted: true } },
|
||||
);
|
||||
auction.startupCompleted = true;
|
||||
return auction;
|
||||
}
|
||||
|
||||
public async getByMatchId(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
): Promise<AuctionDocument> {
|
||||
@@ -462,18 +559,46 @@ export class AuctionService {
|
||||
public async getStateByMatchId(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
): Promise<AuctionStateResponse> {
|
||||
const auction = await this.getByMatchId(matchId);
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await this.getStateByMatchIdOnce(matchId);
|
||||
} catch (error) {
|
||||
if (!(error instanceof AuctionLifecycleBusyError)) throw error;
|
||||
if (attempt >= 4)
|
||||
throw new ServiceUnavailableException({
|
||||
code: 'AUCTION_STATE_RECOVERING',
|
||||
retryAfterMS: 500,
|
||||
message: '경매 상태를 복구 중입니다. 잠시 후 다시 시도해 주세요.',
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async getStateByMatchIdOnce(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
): Promise<AuctionStateResponse> {
|
||||
let auction = await this.getByMatchId(matchId);
|
||||
if (!this.isRedisV1Auction(auction)) return auction;
|
||||
|
||||
if (auction.startupCompleted === false) {
|
||||
auction = await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
matchId.toString(),
|
||||
async () => {
|
||||
const current = await this.getByMatchId(matchId);
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const match = await MatchModel.findById(matchId);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
return this.finishAuctionStartup(current, match);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const redisState = await this.readRedisAuctionStateOrRehydrate(auction);
|
||||
if (!redisState) return auction;
|
||||
|
||||
const mongoState =
|
||||
typeof auction.toObject === 'function'
|
||||
? auction.toObject({ depopulate: true })
|
||||
: auction;
|
||||
return {
|
||||
...this.cloneJson(mongoState),
|
||||
...this.buildAuctionSnapshotFromRedisState(auction, redisState),
|
||||
...redisState,
|
||||
chatMessages: auction.chatMessages ?? [],
|
||||
};
|
||||
@@ -485,12 +610,15 @@ export class AuctionService {
|
||||
const matchId = auction.matchId.toString();
|
||||
const mongoRevision = auction.revision ?? 0;
|
||||
const mongoAuctionLifecycleId = auction.auctionLifecycleId;
|
||||
if (!mongoAuctionLifecycleId || auction.phase === 'complete') {
|
||||
if (!mongoAuctionLifecycleId) {
|
||||
return this.auctionRedisStateReaderService.readAuctionStateView(matchId);
|
||||
}
|
||||
|
||||
const liveIdentity =
|
||||
await this.auctionRedisStateReaderService.readLiveStateIdentity(matchId);
|
||||
// Completed runtimes expire after the retention period. Mongo remains the
|
||||
// archive; viewing it must not create an immortal Redis runtime again.
|
||||
if (!liveIdentity && auction.phase === 'complete') return null;
|
||||
const shouldHydrate = shouldHydrateRedisAuctionState({
|
||||
live: liveIdentity,
|
||||
mongo: {
|
||||
@@ -503,29 +631,32 @@ export class AuctionService {
|
||||
return this.auctionRedisStateReaderService.readAuctionStateView(matchId);
|
||||
}
|
||||
|
||||
const keys = buildAuctionRedisKeys(matchId);
|
||||
const lockToken = randomUUID();
|
||||
const lockAcquired = await this.auctionRedisService.acquireTokenLock(
|
||||
keys.rehydrateLockKey,
|
||||
lockToken,
|
||||
AUCTION_REHYDRATE_LOCK_TTL_MS,
|
||||
await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
matchId,
|
||||
async () => {
|
||||
const currentAuction = await this.getByMatchId(auction.matchId);
|
||||
if (currentAuction.auctionLifecycleId !== mongoAuctionLifecycleId) {
|
||||
throw new ConflictException(
|
||||
'Auction lifecycle has changed. Refresh the auction.',
|
||||
);
|
||||
}
|
||||
const currentLive =
|
||||
await this.auctionRedisStateReaderService.readLiveStateIdentity(
|
||||
matchId,
|
||||
);
|
||||
if (
|
||||
currentLive?.auctionLifecycleId &&
|
||||
currentLive.auctionLifecycleId !== mongoAuctionLifecycleId
|
||||
) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
matchId,
|
||||
currentLive.auctionLifecycleId,
|
||||
);
|
||||
}
|
||||
await this.hydrateRedisStateFromAuctionDocument(currentAuction);
|
||||
},
|
||||
);
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
await this.hydrateRedisStateFromAuctionDocument(auction);
|
||||
} finally {
|
||||
await this.auctionRedisService.releaseTokenLock(
|
||||
keys.rehydrateLockKey,
|
||||
lockToken,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, AUCTION_REHYDRATE_LOCK_WAIT_MS),
|
||||
);
|
||||
}
|
||||
|
||||
return this.auctionRedisStateReaderService.readAuctionStateView(matchId, {
|
||||
shouldForceAuctionStateResync: true,
|
||||
});
|
||||
@@ -814,6 +945,7 @@ export class AuctionService {
|
||||
bidderTeamIndex: params.teamIndex,
|
||||
bidderUserId: bidContext.bidderTeamCaptainUserId,
|
||||
bidRequestId: params.bidRequestId ?? randomUUID(),
|
||||
clientObservedBidRoundId: params.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId: auctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId,
|
||||
matchId,
|
||||
@@ -890,6 +1022,7 @@ export class AuctionService {
|
||||
bidderTeamIndex: params.teamIndex,
|
||||
bidderUserId: this.stringFromState(team.captainUserId),
|
||||
bidRequestId: params.bidRequestId ?? randomUUID(),
|
||||
clientObservedBidRoundId: params.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId: auctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId,
|
||||
matchId,
|
||||
@@ -923,6 +1056,10 @@ export class AuctionService {
|
||||
): AuctionBidDeltaResponse {
|
||||
return {
|
||||
auctionBidHistoryEntry: result.auctionBidHistoryEntry,
|
||||
currentBidRoundId:
|
||||
typeof result.auctionBidHistoryEntry.bidRoundId === 'string'
|
||||
? result.auctionBidHistoryEntry.bidRoundId
|
||||
: undefined,
|
||||
auctionLifecycleId: this.stringFromState(state.auctionLifecycleId),
|
||||
bidRequestId: result.bidRequestId,
|
||||
currentBidRoundDeadlineTimestampMS:
|
||||
@@ -1408,6 +1545,34 @@ export class AuctionService {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
}): Promise<void> {
|
||||
let committed = false;
|
||||
try {
|
||||
await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
params.matchId.toString(),
|
||||
() =>
|
||||
this.resetLocked(params, () => {
|
||||
committed = true;
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!committed) throw error;
|
||||
// A committed cancellation is successful even if lock renewal/teardown
|
||||
// fails afterward. Cleanup is durably queued in the same Mongo transaction.
|
||||
this.logger.warn(
|
||||
`Auction cancellation committed; post-commit coordination failed: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async resetLocked(
|
||||
params: {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
clientObservedAuctionLifecycleId?: string;
|
||||
clientObservedRevision?: number;
|
||||
},
|
||||
onCommitted: () => void,
|
||||
): Promise<void> {
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
const AuctionHistoryModel =
|
||||
await this.connectionService.getModel('auctionHistory');
|
||||
@@ -1419,70 +1584,155 @@ export class AuctionService {
|
||||
if (!session) {
|
||||
throw new InternalServerErrorException('Failed to start DB session.');
|
||||
}
|
||||
let redisRuntimeToClear: {
|
||||
auctionLifecycleId?: string;
|
||||
matchId: string;
|
||||
} | null = null;
|
||||
try {
|
||||
session.startTransaction();
|
||||
const auction = await AuctionModel.findOne({
|
||||
matchId: params.matchId,
|
||||
}).session(session);
|
||||
if (!auction) throw new NotFoundException('Auction not found.');
|
||||
|
||||
const match = await MatchModel.findById(params.matchId).session(session);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
if (!match.communityId.equals(params.communityId)) {
|
||||
throw new BadRequestException('Community mismatch.');
|
||||
}
|
||||
this.assertCompetitionLifecycleMutable(match, 'reset the auction');
|
||||
|
||||
if (this.isRedisV1Auction(auction)) {
|
||||
redisRuntimeToClear = {
|
||||
auctionLifecycleId: auction.auctionLifecycleId,
|
||||
matchId: params.matchId.toString(),
|
||||
};
|
||||
await this.resetMatchAfterAuctionCancellation(match, session);
|
||||
await AuctionSideEffectApplicationModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await auction.deleteOne({ session });
|
||||
if (!auction) {
|
||||
// 이미 커밋된 취소의 재전송이면 성공으로 응답하고 남은 Redis 정리만 재시도한다.
|
||||
const cancelledMatch = params.clientObservedAuctionLifecycleId
|
||||
? await MatchModel.findById(params.matchId).session(session)
|
||||
: null;
|
||||
if (
|
||||
!cancelledMatch ||
|
||||
cancelledMatch.lastCancelledAuctionLifecycleId !==
|
||||
params.clientObservedAuctionLifecycleId
|
||||
) {
|
||||
throw new NotFoundException('Auction not found.');
|
||||
}
|
||||
if (!cancelledMatch.communityId.equals(params.communityId)) {
|
||||
throw new BadRequestException('Community mismatch.');
|
||||
}
|
||||
cancelledMatch.auctionCancellationCleanupPending = true;
|
||||
cancelledMatch.auctionCancellationCleanupRetryAt = new Date();
|
||||
await cancelledMatch.save({ session });
|
||||
} else {
|
||||
await auction.deleteOne({ session });
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
if (match.status === 'team-built') {
|
||||
await this.tournamentCodeService.releaseFromMatch(match);
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
match.gameSchedule = [];
|
||||
match.participants.forEach((participant) => {
|
||||
participant.excluded = false;
|
||||
});
|
||||
match.teamBuiltAt = undefined;
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save({ session });
|
||||
if (
|
||||
this.isRedisV1Auction(auction) &&
|
||||
(!params.clientObservedAuctionLifecycleId ||
|
||||
params.clientObservedAuctionLifecycleId !==
|
||||
auction.auctionLifecycleId)
|
||||
) {
|
||||
throw new ConflictException(
|
||||
'Auction lifecycle has changed. Refresh before cancelling.',
|
||||
);
|
||||
}
|
||||
|
||||
const match = await MatchModel.findById(params.matchId).session(
|
||||
session,
|
||||
);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
if (!match.communityId.equals(params.communityId)) {
|
||||
throw new BadRequestException('Community mismatch.');
|
||||
}
|
||||
this.assertCompetitionLifecycleMutable(match, 'reset the auction');
|
||||
|
||||
if (this.isRedisV1Auction(auction)) {
|
||||
// 같은 트랜잭션에서 취소 완료를 기록해, 재전송을 멱등 처리한다.
|
||||
match.lastCancelledAuctionLifecycleId = auction.auctionLifecycleId;
|
||||
match.auctionCancellationCleanupPending = true;
|
||||
match.auctionCancellationCleanupRetryAt = new Date();
|
||||
match.auctionCancellationCleanupAttempts = 0;
|
||||
match.auctionCancellationCleanupError = undefined;
|
||||
await this.resetMatchAfterAuctionCancellation(match, session);
|
||||
await AuctionSideEffectApplicationModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await auction.deleteOne({ session });
|
||||
} else {
|
||||
await auction.deleteOne({ session });
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
if (match.status === 'team-built') {
|
||||
await this.tournamentCodeService.releaseFromMatch(match, session);
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
match.gameSchedule = [];
|
||||
match.participants.forEach((participant) => {
|
||||
participant.excluded = false;
|
||||
});
|
||||
match.teamBuiltAt = undefined;
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save({ session });
|
||||
}
|
||||
}
|
||||
}
|
||||
await session.commitTransaction();
|
||||
onCommitted();
|
||||
} catch (err) {
|
||||
await session.abortTransaction();
|
||||
throw err;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
}
|
||||
|
||||
if (redisRuntimeToClear) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
redisRuntimeToClear.matchId,
|
||||
redisRuntimeToClear.auctionLifecycleId,
|
||||
);
|
||||
public async retryPendingCancellations(batchLimit = 20): Promise<void> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const pending = await MatchModel.find({
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: { $lte: new Date() },
|
||||
})
|
||||
.sort({ auctionCancellationCleanupRetryAt: 1 })
|
||||
.limit(batchLimit)
|
||||
.lean()
|
||||
.exec();
|
||||
for (const match of pending) {
|
||||
const lifecycle = match.lastCancelledAuctionLifecycleId;
|
||||
if (!lifecycle) continue;
|
||||
const filter = {
|
||||
_id: match._id,
|
||||
lastCancelledAuctionLifecycleId: lifecycle,
|
||||
};
|
||||
try {
|
||||
await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
match._id.toString(),
|
||||
async () => {
|
||||
// Compare-and-delete in Redis also protects a newly started lifecycle.
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
match._id.toString(),
|
||||
lifecycle,
|
||||
);
|
||||
const AuctionModel =
|
||||
await this.connectionService.getModel('auction');
|
||||
if (!(await AuctionModel.exists({ matchId: match._id }))) {
|
||||
this.auctionEventBusService?.broadcastReset(
|
||||
match._id.toString(),
|
||||
lifecycle,
|
||||
);
|
||||
}
|
||||
await MatchModel.updateOne(filter, {
|
||||
$set: { auctionCancellationCleanupPending: false },
|
||||
$unset: {
|
||||
auctionCancellationCleanupRetryAt: '',
|
||||
auctionCancellationCleanupError: '',
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
const attempts = (match.auctionCancellationCleanupAttempts ?? 0) + 1;
|
||||
await MatchModel.updateOne(filter, {
|
||||
$set: {
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: new Date(
|
||||
Date.now() + Math.min(60000, 1000 * 2 ** Math.min(attempts, 6)),
|
||||
),
|
||||
auctionCancellationCleanupError: String(error).slice(0, 2000),
|
||||
},
|
||||
$inc: { auctionCancellationCleanupAttempts: 1 },
|
||||
});
|
||||
this.logger.warn(
|
||||
`Auction cancellation cleanup will retry for ${match._id.toString()}: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1497,6 +1747,19 @@ export class AuctionService {
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
sequence: number;
|
||||
}): Promise<AuctionStateResponse> {
|
||||
return new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
params.matchId.toString(),
|
||||
() => this.undoToLocked(params),
|
||||
);
|
||||
}
|
||||
|
||||
private async undoToLocked(
|
||||
params: AuctionStructureActionRequestParams & {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
sequence: number;
|
||||
},
|
||||
): Promise<AuctionStateResponse> {
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
const AuctionHistoryModel =
|
||||
await this.connectionService.getModel('auctionHistory');
|
||||
@@ -1508,6 +1771,7 @@ export class AuctionService {
|
||||
}
|
||||
|
||||
let redisUndoContext: {
|
||||
state?: Record<string, unknown>;
|
||||
actionRequestParams?: AuctionStructureActionRequestParams;
|
||||
auction: AuctionDocument;
|
||||
snapshot: Record<string, unknown>;
|
||||
@@ -1543,8 +1807,12 @@ export class AuctionService {
|
||||
throw new BadRequestException('Auction history snapshot is invalid.');
|
||||
}
|
||||
|
||||
const undoingCompletedAuction =
|
||||
Boolean(auction.completedAt) && !('completedAt' in snapshot);
|
||||
const liveState = this.isRedisV1Auction(auction)
|
||||
? await this.requireRedisAuctionState(params.matchId.toString())
|
||||
: undefined;
|
||||
const undoingCompletedAuction = liveState
|
||||
? liveState.phase === 'complete' && snapshot.phase !== 'complete'
|
||||
: Boolean(auction.completedAt) && !('completedAt' in snapshot);
|
||||
if (undoingCompletedAuction) {
|
||||
this.assertCompetitionLifecycleMutable(
|
||||
match,
|
||||
@@ -1553,6 +1821,7 @@ export class AuctionService {
|
||||
}
|
||||
if (this.isRedisV1Auction(auction)) {
|
||||
redisUndoContext = {
|
||||
state: liveState,
|
||||
auction,
|
||||
actionRequestParams: params,
|
||||
snapshot: snapshot,
|
||||
@@ -1562,8 +1831,8 @@ export class AuctionService {
|
||||
await session.commitTransaction();
|
||||
} else {
|
||||
if (undoingCompletedAuction) {
|
||||
await this.tournamentCodeService.releaseFromMatch(match);
|
||||
await this.tournamentResultService.cancelPolling(match);
|
||||
await this.tournamentCodeService.releaseFromMatch(match, session);
|
||||
await this.tournamentResultService.cancelPolling(match, session);
|
||||
match.status = 'team-built';
|
||||
match.result = undefined;
|
||||
await match.save({ session });
|
||||
@@ -1758,6 +2027,14 @@ export class AuctionService {
|
||||
currentAuctionHistorySequence: auctionHistorySequence,
|
||||
lastAppliedAuctionStateTransitionId: auctionStateTransitionId,
|
||||
revision: expectedRevision + 1,
|
||||
currentBidRoundId:
|
||||
params.deadline.mode === 'set-relative'
|
||||
? auctionStateTransitionId
|
||||
: params.state.currentBidRoundId,
|
||||
currentBidRoundDeadlineTimestampMS:
|
||||
params.deadline.mode === 'remove'
|
||||
? 0
|
||||
: params.state.currentBidRoundDeadlineTimestampMS,
|
||||
};
|
||||
const afterAuctionMongoProjection =
|
||||
this.buildAuctionMongoProjectionFromState(params.auction, afterState);
|
||||
@@ -1769,8 +2046,18 @@ export class AuctionService {
|
||||
if (matchSideEffectMutation.type !== 'none') {
|
||||
afterAuctionMongoProjection.matchSideEffectStatus = 'pending';
|
||||
afterAuctionMongoProjection.matchSideEffectRevision =
|
||||
(params.auction.matchSideEffectRevision ?? 0) + 1;
|
||||
Math.max(
|
||||
params.auction.matchSideEffectRevision ?? 0,
|
||||
this.numberFromState(params.state.matchSideEffectRevision),
|
||||
) + 1;
|
||||
afterAuctionMongoProjection.matchSideEffectError = undefined;
|
||||
afterAuctionMongoProjection.matchSideEffectTransitionId =
|
||||
auctionStateTransitionId;
|
||||
params.liveStatePatch.matchSideEffectTransitionId =
|
||||
auctionStateTransitionId;
|
||||
params.liveStatePatch.matchSideEffectStatus = 'pending';
|
||||
params.liveStatePatch.matchSideEffectRevision =
|
||||
afterAuctionMongoProjection.matchSideEffectRevision as number;
|
||||
}
|
||||
|
||||
const durabilityPayload = {
|
||||
@@ -1925,6 +2212,7 @@ export class AuctionService {
|
||||
}
|
||||
|
||||
private async undoRedisAuction(params: {
|
||||
state?: Record<string, unknown>;
|
||||
actionRequestParams?: AuctionStructureActionRequestParams;
|
||||
auction: AuctionDocument;
|
||||
snapshot: Record<string, unknown>;
|
||||
@@ -1932,7 +2220,8 @@ export class AuctionService {
|
||||
undoingCompletedAuction: boolean;
|
||||
}): Promise<AuctionStateResponse> {
|
||||
const matchId = params.auction.matchId.toString();
|
||||
const state = await this.requireRedisAuctionState(matchId);
|
||||
const state =
|
||||
params.state ?? (await this.requireRedisAuctionState(matchId));
|
||||
const actionRequest = await this.reserveRedisStructureActionRequest({
|
||||
action: 'undo',
|
||||
matchId,
|
||||
@@ -2118,8 +2407,11 @@ export class AuctionService {
|
||||
typeof auction.toObject === 'function'
|
||||
? (auction.toObject({ depopulate: true }) as Record<string, unknown>)
|
||||
: (auction as unknown as Record<string, unknown>);
|
||||
const snapshotBase = this.cloneJson(base);
|
||||
for (const field of AUCTION_OPTIONAL_PROJECTION_FIELDS)
|
||||
delete snapshotBase[field];
|
||||
return {
|
||||
...this.cloneJson(base),
|
||||
...snapshotBase,
|
||||
...this.buildAuctionMongoProjectionFromState(auction, state),
|
||||
_id: auction._id,
|
||||
matchId: auction.matchId,
|
||||
@@ -2145,6 +2437,7 @@ export class AuctionService {
|
||||
);
|
||||
const phase = this.stringFromState(state.phase) || 'presenting';
|
||||
const projection: Record<string, unknown> = {
|
||||
currentBidRoundId: state.currentBidRoundId || undefined,
|
||||
auctionBidHistory,
|
||||
auctionCompletedAtTimestampMS:
|
||||
phase === 'complete' &&
|
||||
@@ -2285,6 +2578,8 @@ export class AuctionService {
|
||||
|
||||
private formatRedisBidRejectCode(code: string): string {
|
||||
switch (code) {
|
||||
case 'BID_CLIENT_UPDATE_REQUIRED':
|
||||
return '경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.';
|
||||
case 'PHASE':
|
||||
return 'Auction is not in bidding phase.';
|
||||
case 'STALE_BID_ROUND':
|
||||
@@ -2701,8 +2996,8 @@ export class AuctionService {
|
||||
match: MatchDocument,
|
||||
session: mongoose.ClientSession,
|
||||
): Promise<void> {
|
||||
await this.tournamentCodeService.releaseFromMatch(match);
|
||||
await this.tournamentResultService.cancelPolling(match);
|
||||
await this.tournamentCodeService.releaseFromMatch(match, session);
|
||||
await this.tournamentResultService.cancelPolling(match, session);
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
@@ -2717,6 +3012,11 @@ export class AuctionService {
|
||||
}
|
||||
|
||||
private assertLiveAuctionOperationSupported(auction: AuctionDocument): void {
|
||||
if (auction.startupCompleted === false) {
|
||||
throw new ConflictException(
|
||||
'Auction startup is not complete. Refresh to resume initialization.',
|
||||
);
|
||||
}
|
||||
if (auction.auctionEngineVersion && !this.isRedisV1Auction(auction)) {
|
||||
throw new BadRequestException(
|
||||
'Legacy auction live operations are not supported.',
|
||||
@@ -2881,7 +3181,7 @@ export class AuctionService {
|
||||
auction.topTierThreshold ??
|
||||
'NONE',
|
||||
};
|
||||
await this.auctionRedisCommandService.hydrateState(
|
||||
const hydration = await this.auctionRedisCommandService.hydrateState(
|
||||
auction.matchId.toString(),
|
||||
{
|
||||
auctionHistorySequence: auction.currentAuctionHistorySequence ?? 0,
|
||||
@@ -2889,6 +3189,10 @@ export class AuctionService {
|
||||
auction.auctionLifecycleId ??
|
||||
`auction:${auction._id.toString()}:rehydrated`,
|
||||
liveStatePatch: {
|
||||
currentBidRoundId: auction.phase === 'bidding' ? randomUUID() : '',
|
||||
matchSideEffectTransitionId: auction.matchSideEffectTransitionId,
|
||||
matchSideEffectRevision: auction.matchSideEffectRevision,
|
||||
matchSideEffectStatus: auction.matchSideEffectStatus,
|
||||
...buildAuctionRedisLiveStatePatchFromStructure(structureInput),
|
||||
currentHighestBidPointAmount:
|
||||
auction.currentHighestBidPointAmount ?? auction.currentBid ?? 0,
|
||||
@@ -2905,6 +3209,10 @@ export class AuctionService {
|
||||
structureStateJson: stringifyAuctionRedisStructureState(structureInput),
|
||||
},
|
||||
);
|
||||
if ('rejected' in hydration)
|
||||
throw new ConflictException(
|
||||
`Auction recovery failed: ${hydration.code}.`,
|
||||
);
|
||||
}
|
||||
|
||||
private async clearRedisAuctionRuntimeForLifecycle(
|
||||
@@ -2925,10 +3233,10 @@ export class AuctionService {
|
||||
' end',
|
||||
'end',
|
||||
"redis.call('ZREM', KEYS[2], ARGV[1])",
|
||||
"redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11])",
|
||||
"redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11], KEYS[12])",
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'11',
|
||||
'12',
|
||||
keys.liveStateKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.structureStateKey,
|
||||
@@ -2940,6 +3248,7 @@ export class AuctionService {
|
||||
keys.actionRequestResultOrderKey,
|
||||
keys.rehydrateLockKey,
|
||||
keys.stateTransitionLockKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
matchId,
|
||||
expectedAuctionLifecycleId ?? '',
|
||||
]);
|
||||
|
||||
@@ -72,6 +72,10 @@ export class StartAuctionDto {
|
||||
}
|
||||
|
||||
export class PlaceBidDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
clientObservedBidRoundId?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsUUID()
|
||||
bidRequestId?: string;
|
||||
|
||||
@@ -7,7 +7,13 @@ function makeService(
|
||||
) {
|
||||
const sendCommand = jest
|
||||
.fn()
|
||||
.mockImplementation(() => Promise.resolve(commandReplies.shift()));
|
||||
.mockImplementation((command: string[]) =>
|
||||
Promise.resolve(
|
||||
command[0] === 'EVAL'
|
||||
? [currentServerTimestampMS, ...commandReplies.splice(0, 3)]
|
||||
: commandReplies.shift(),
|
||||
),
|
||||
);
|
||||
const auctionRedisService = {
|
||||
getCurrentRedisTimestampMS: jest
|
||||
.fn()
|
||||
@@ -125,19 +131,14 @@ describe('AuctionRedisStateReaderService', () => {
|
||||
});
|
||||
expect(view).not.toHaveProperty('team:2:remainingBidPointAmount');
|
||||
expect(view).not.toHaveProperty('bidRoundExpiryDecision');
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(1, [
|
||||
'HGETALL',
|
||||
expect(sendCommand).toHaveBeenCalledTimes(1);
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining("redis.call('TIME')"),
|
||||
'3',
|
||||
'auction:match-1:liveAuctionState',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(2, [
|
||||
'GET',
|
||||
'auction:match-1:auctionStructureState',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(3, [
|
||||
'LRANGE',
|
||||
'auction:match-1:auctionBidHistory',
|
||||
'0',
|
||||
'-1',
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -111,18 +111,11 @@ export class AuctionRedisStateReaderService {
|
||||
liveReply,
|
||||
structureStateJson,
|
||||
bidEntries,
|
||||
] = await Promise.all([
|
||||
this.auctionRedisService.getCurrentRedisTimestampMS(),
|
||||
this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['HGETALL', keys.liveStateKey]),
|
||||
this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['GET', keys.structureStateKey]),
|
||||
this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['LRANGE', keys.bidHistoryKey, '0', '-1']),
|
||||
]);
|
||||
] = await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand<
|
||||
unknown[]
|
||||
>(['EVAL', ["local now = redis.call('TIME')", 'local timestamp = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)', "return {tostring(timestamp), redis.call('HGETALL', KEYS[1]), redis.call('GET', KEYS[2]), redis.call('LRANGE', KEYS[3], 0, -1)}"].join('\n'), '3', keys.liveStateKey, keys.structureStateKey, keys.bidHistoryKey]);
|
||||
|
||||
const liveState = parseHashReply(liveReply, {
|
||||
includeInternalLiveStateFields:
|
||||
@@ -144,7 +137,7 @@ export class AuctionRedisStateReaderService {
|
||||
auctionBidHistoryEntries: Array.isArray(bidEntries)
|
||||
? bidEntries.map(String)
|
||||
: [],
|
||||
currentServerTimestampMS,
|
||||
currentServerTimestampMS: Number(currentServerTimestampMS),
|
||||
liveState: liveState as Record<string, unknown> & {
|
||||
auctionLifecycleId: string;
|
||||
phase: string;
|
||||
|
||||
@@ -46,6 +46,7 @@ describe('auction apply transition redis script contract', () => {
|
||||
'auction:match-1:bidRequestResults',
|
||||
'auction:match-1:bidRequestResultOrder',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'auction:match-1:rejectedBidRequestResultOrder',
|
||||
]);
|
||||
expect(invocation.values.slice(0, 19)).toEqual([
|
||||
'match-1',
|
||||
|
||||
@@ -79,6 +79,7 @@ export function buildAuctionApplyTransitionScriptInvocation(
|
||||
keys.bidRequestResultsKey,
|
||||
keys.bidRequestResultOrderKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
],
|
||||
script: AUCTION_APPLY_TRANSITION_SCRIPT,
|
||||
values: [
|
||||
@@ -128,6 +129,7 @@ local auctionBidHistoryKey = KEYS[8]
|
||||
local bidRequestResultsKey = KEYS[9]
|
||||
local bidRequestResultOrderKey = KEYS[10]
|
||||
local bidRoundDeadlineQueueKey = KEYS[11]
|
||||
local rejectedBidRequestResultOrderKey = KEYS[12]
|
||||
|
||||
local matchId = ARGV[1]
|
||||
local auctionLifecycleId = ARGV[2]
|
||||
@@ -270,16 +272,27 @@ for i = 1, liveStatePatchPairCount do
|
||||
end
|
||||
redis.call('HSET', unpack(hsetArgs))
|
||||
|
||||
local committedBidRoundDeadlineTimestampMS = ''
|
||||
if deadlineMode == 'set-relative' then
|
||||
local currentBidRoundDeadlineTimestampMS = currentRedisTimestampMS + (deadlineDurationSeconds * 1000)
|
||||
committedBidRoundDeadlineTimestampMS = tostring(currentBidRoundDeadlineTimestampMS)
|
||||
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS', tostring(currentBidRoundDeadlineTimestampMS))
|
||||
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundId', auctionStateTransitionId)
|
||||
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
|
||||
elseif deadlineMode == 'remove' then
|
||||
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
|
||||
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
|
||||
end
|
||||
|
||||
-- A resumed auction must not disappear when its former completion TTL elapses.
|
||||
if redis.call('HGET', liveAuctionStateKey, 'phase') ~= 'complete' then
|
||||
for _, index in ipairs({1, 2, 3, 6, 7, 8, 9, 10, 12}) do
|
||||
redis.call('PERSIST', KEYS[index])
|
||||
end
|
||||
end
|
||||
|
||||
if shouldResetBidRuntime == '1' then
|
||||
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey)
|
||||
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey, rejectedBidRequestResultOrderKey)
|
||||
end
|
||||
|
||||
redis.call('SADD', appliedAuctionStateTransitionIdsKey, transitionDedupeMember)
|
||||
@@ -288,6 +301,8 @@ redis.call(
|
||||
durabilityPayloadKey,
|
||||
'payloadJson',
|
||||
durabilityPayloadJson,
|
||||
'committedBidRoundDeadlineTimestampMS',
|
||||
committedBidRoundDeadlineTimestampMS,
|
||||
'preparedAtRedisTimestampMS',
|
||||
tostring(currentRedisTimestampMS),
|
||||
'committedAtRedisTimestampMS',
|
||||
|
||||
@@ -30,6 +30,9 @@ describe('auction bid redis script contract', () => {
|
||||
'auction:match-1:bidRequestResults',
|
||||
'auction:match-1:bidRequestResultOrder',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'auction:match-1:rejectedBidRequestResultOrder',
|
||||
'auction:bidArchiveQueue',
|
||||
'auction:bidArchivePayloads',
|
||||
]);
|
||||
expect(invocation.values).toEqual([
|
||||
'match-1',
|
||||
@@ -45,6 +48,7 @@ describe('auction bid redis script contract', () => {
|
||||
'100',
|
||||
'60000',
|
||||
'200',
|
||||
'',
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { AuctionRedisKeys } from './auction-redis.keys';
|
||||
import {
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
} from './auction-redis.keys';
|
||||
import { defineAuctionRedisScript } from './auction-redis.scripts';
|
||||
|
||||
export interface AuctionBidScriptInput {
|
||||
@@ -8,6 +12,7 @@ export interface AuctionBidScriptInput {
|
||||
bidderUserId: string;
|
||||
clientObservedAuctionLifecycleId: string;
|
||||
clientObservedCurrentBidRoundPlayerUserId: string;
|
||||
clientObservedBidRoundId?: string;
|
||||
maximumAuctionTeamMemberCount: number;
|
||||
maximumStoredAuctionBidHistoryCount: number;
|
||||
maximumStoredBidRequestResultCount: number;
|
||||
@@ -40,15 +45,19 @@ export function buildBidRequestPayloadFingerprint(params: {
|
||||
bidPointAmount: number;
|
||||
clientObservedAuctionLifecycleId: string;
|
||||
clientObservedCurrentBidRoundPlayerUserId: string;
|
||||
clientObservedBidRoundId?: string;
|
||||
matchId: string;
|
||||
}): string {
|
||||
return [
|
||||
const parts = [
|
||||
params.matchId,
|
||||
params.clientObservedAuctionLifecycleId,
|
||||
params.clientObservedCurrentBidRoundPlayerUserId,
|
||||
params.bidderTeamIndex,
|
||||
params.bidPointAmount,
|
||||
].join('|');
|
||||
];
|
||||
if (params.clientObservedBidRoundId)
|
||||
parts.push(params.clientObservedBidRoundId);
|
||||
return parts.join('|');
|
||||
}
|
||||
|
||||
export function buildAuctionBidScriptInvocation(
|
||||
@@ -62,6 +71,9 @@ export function buildAuctionBidScriptInvocation(
|
||||
keys.bidRequestResultsKey,
|
||||
keys.bidRequestResultOrderKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
],
|
||||
script: AUCTION_BID_SCRIPT,
|
||||
values: [
|
||||
@@ -78,6 +90,7 @@ export function buildAuctionBidScriptInvocation(
|
||||
String(input.maximumStoredBidRequestResultCount),
|
||||
String(input.minimumBidRequestResultReplayWindowMS),
|
||||
String(input.maximumStoredAuctionBidHistoryCount),
|
||||
input.clientObservedBidRoundId ?? '',
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -90,6 +103,9 @@ local auctionBidHistoryKey = KEYS[2]
|
||||
local bidRequestResultsKey = KEYS[3]
|
||||
local bidRequestResultOrderKey = KEYS[4]
|
||||
local bidRoundDeadlineQueueKey = KEYS[5]
|
||||
local rejectedBidRequestResultOrderKey = KEYS[6]
|
||||
local bidArchiveQueueKey = KEYS[7]
|
||||
local bidArchivePayloadsKey = KEYS[8]
|
||||
|
||||
local matchId = ARGV[1]
|
||||
local bidderTeamIndex = ARGV[2]
|
||||
@@ -104,6 +120,7 @@ local bidRequestPayloadFingerprint = ARGV[10]
|
||||
local maximumStoredBidRequestResultCount = tonumber(ARGV[11])
|
||||
local minimumBidRequestResultReplayWindowMS = tonumber(ARGV[12])
|
||||
local maximumStoredAuctionBidHistoryCount = tonumber(ARGV[13])
|
||||
local clientObservedBidRoundId = ARGV[14] or ''
|
||||
|
||||
local redisTime = redis.call('TIME')
|
||||
local currentRedisTimestampMS = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)
|
||||
@@ -168,12 +185,23 @@ if existingFingerprint then
|
||||
return replayExistingRequest()
|
||||
end
|
||||
|
||||
trimOldBidRequestResults()
|
||||
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
|
||||
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
|
||||
-- Upgrade the old shared queue once, preserving every accepted replay record.
|
||||
if redis.call('HSETNX', bidRequestResultsKey, '_acceptedOnlyOrderV1', '1') == 1 then
|
||||
for _, oldRequestId in ipairs(redis.call('LRANGE', bidRequestResultOrderKey, 0, -1)) do
|
||||
if redis.call('HGET', bidRequestResultsKey, oldRequestId .. ':status') == 'rejected' then
|
||||
redis.call('LREM', bidRequestResultOrderKey, 0, oldRequestId)
|
||||
hdelBidRequest(oldRequestId)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local function storeRejectedRequest(rejectCode)
|
||||
-- Rejections have their own bounded retention. They must never consume the
|
||||
-- protected replay capacity of accepted bids.
|
||||
while redis.call('LLEN', rejectedBidRequestResultOrderKey) >= maximumStoredBidRequestResultCount do
|
||||
local oldest = redis.call('LPOP', rejectedBidRequestResultOrderKey)
|
||||
if oldest then hdelBidRequest(oldest) end
|
||||
end
|
||||
local rejectedCurrentBidRoundDeadlineTimestampMS = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or ''
|
||||
redis.call(
|
||||
'HSET',
|
||||
@@ -191,7 +219,7 @@ local function storeRejectedRequest(rejectCode)
|
||||
requestField('createdAtTimestampMS'),
|
||||
tostring(currentRedisTimestampMS)
|
||||
)
|
||||
redis.call('RPUSH', bidRequestResultOrderKey, bidRequestId)
|
||||
redis.call('RPUSH', rejectedBidRequestResultOrderKey, bidRequestId)
|
||||
return {'REJECTED', rejectCode, bidRequestId, tostring(currentRedisTimestampMS), rejectedCurrentBidRoundDeadlineTimestampMS}
|
||||
end
|
||||
|
||||
@@ -202,9 +230,16 @@ end
|
||||
|
||||
local auctionLifecycleId = redis.call('HGET', liveAuctionStateKey, 'auctionLifecycleId')
|
||||
local currentBidRoundPlayerUserId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundPlayerUserId')
|
||||
local currentBidRoundId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundId') or ''
|
||||
if currentBidRoundId ~= '' and clientObservedBidRoundId == '' then
|
||||
return storeRejectedRequest('BID_CLIENT_UPDATE_REQUIRED')
|
||||
end
|
||||
if auctionLifecycleId ~= clientObservedAuctionLifecycleId or currentBidRoundPlayerUserId ~= clientObservedCurrentBidRoundPlayerUserId then
|
||||
return storeRejectedRequest('STALE_BID_ROUND')
|
||||
end
|
||||
if currentBidRoundId ~= clientObservedBidRoundId then
|
||||
return storeRejectedRequest('STALE_BID_ROUND')
|
||||
end
|
||||
|
||||
local currentBidRoundDeadlineTimestampMS = tonumber(redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or '0')
|
||||
if currentBidRoundDeadlineTimestampMS <= currentRedisTimestampMS then
|
||||
@@ -235,8 +270,9 @@ if bidPointAmount + requiredReserveAfterBid > remainingBidPointAmount then
|
||||
return storeRejectedRequest('INSUFFICIENT')
|
||||
end
|
||||
|
||||
if redis.call('LLEN', auctionBidHistoryKey) >= maximumStoredAuctionBidHistoryCount then
|
||||
return storeRejectedRequest('BID_HISTORY_LIMIT')
|
||||
trimOldBidRequestResults()
|
||||
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
|
||||
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
|
||||
end
|
||||
|
||||
local maximumBidRoundDurationSeconds = tonumber(redis.call('HGET', liveAuctionStateKey, 'maximumBidRoundDurationSeconds') or '0')
|
||||
@@ -254,14 +290,36 @@ end
|
||||
local nextBidRoundDeadlineTimestampMS = currentRedisTimestampMS + nextRemainingBidRoundTimeMS
|
||||
|
||||
local revision = redis.call('HINCRBY', liveAuctionStateKey, 'revision', 1)
|
||||
local auctionBidHistoryEntryJson =
|
||||
'{"bidderTeamIndex":' .. bidderTeamIndex ..
|
||||
',"bidPointAmount":' .. tostring(bidPointAmount) ..
|
||||
',"bidderUserId":"' .. bidderUserId ..
|
||||
'","bidPlacedAtTimestampMS":' .. tostring(currentRedisTimestampMS) ..
|
||||
'}'
|
||||
local auctionBidHistoryEntryJson = cjson.encode({
|
||||
bidRequestId = bidRequestId,
|
||||
revision = revision,
|
||||
bidderTeamIndex = tonumber(bidderTeamIndex),
|
||||
bidPointAmount = bidPointAmount,
|
||||
bidderUserId = bidderUserId,
|
||||
playerUserId = currentBidRoundPlayerUserId,
|
||||
bidRoundId = currentBidRoundId,
|
||||
bidPlacedAtTimestampMS = currentRedisTimestampMS
|
||||
})
|
||||
|
||||
-- Accepted replay returns above this point. Persist every accepted event before
|
||||
-- trimming the recent view; the durable archive worker removes only saved events.
|
||||
local archiveId = matchId .. ':' .. auctionLifecycleId .. ':' .. string.format('%016d', revision)
|
||||
redis.call('HSET', bidArchivePayloadsKey, archiveId, cjson.encode({
|
||||
_id = archiveId,
|
||||
matchId = matchId,
|
||||
auctionLifecycleId = auctionLifecycleId,
|
||||
revision = revision,
|
||||
bidRequestId = bidRequestId,
|
||||
bidderTeamIndex = tonumber(bidderTeamIndex),
|
||||
bidPointAmount = bidPointAmount,
|
||||
bidderUserId = bidderUserId,
|
||||
playerUserId = currentBidRoundPlayerUserId,
|
||||
bidRoundId = currentBidRoundId,
|
||||
bidPlacedAtTimestampMS = currentRedisTimestampMS
|
||||
}))
|
||||
redis.call('ZADD', bidArchiveQueueKey, currentRedisTimestampMS, archiveId)
|
||||
redis.call('RPUSH', auctionBidHistoryKey, auctionBidHistoryEntryJson)
|
||||
redis.call('LTRIM', auctionBidHistoryKey, -maximumStoredAuctionBidHistoryCount, -1)
|
||||
redis.call(
|
||||
'HSET',
|
||||
liveAuctionStateKey,
|
||||
|
||||
@@ -4,6 +4,14 @@ export const AUCTION_DURABILITY_RETRY_QUEUE_KEY =
|
||||
'auction:auctionDurabilityRetryQueue';
|
||||
export const AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY =
|
||||
'auction:auctionDurabilityDeadLetterQueue';
|
||||
export const AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY =
|
||||
'auction:deadLetterStatusQueue';
|
||||
export const AUCTION_BID_ARCHIVE_QUEUE_KEY = 'auction:bidArchiveQueue';
|
||||
export const AUCTION_BID_ARCHIVE_PAYLOADS_KEY = 'auction:bidArchivePayloads';
|
||||
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY =
|
||||
'auction:bidArchiveDeadLetterQueue';
|
||||
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY =
|
||||
'auction:bidArchiveDeadLetterPayloads';
|
||||
|
||||
export interface AuctionRedisKeys {
|
||||
actionRequestResultOrderKey: string;
|
||||
@@ -12,6 +20,7 @@ export interface AuctionRedisKeys {
|
||||
bidHistoryKey: string;
|
||||
bidRequestResultOrderKey: string;
|
||||
bidRequestResultsKey: string;
|
||||
rejectedBidRequestResultOrderKey: string;
|
||||
deadlineQueueKey: string;
|
||||
durabilityDeadLetterQueueKey: string;
|
||||
durabilityPayloadKey: (auctionStateTransitionId: string) => string;
|
||||
@@ -33,6 +42,7 @@ export function buildAuctionRedisKeys(matchId: string): AuctionRedisKeys {
|
||||
bidHistoryKey: `${prefix}:auctionBidHistory`,
|
||||
bidRequestResultOrderKey: `${prefix}:bidRequestResultOrder`,
|
||||
bidRequestResultsKey: `${prefix}:bidRequestResults`,
|
||||
rejectedBidRequestResultOrderKey: `${prefix}:rejectedBidRequestResultOrder`,
|
||||
deadlineQueueKey: AUCTION_BID_ROUND_DEADLINE_QUEUE_KEY,
|
||||
durabilityDeadLetterQueueKey: AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
durabilityPayloadKey: (auctionStateTransitionId: string) =>
|
||||
|
||||
@@ -24,7 +24,7 @@ describe('auction redis state assembly', () => {
|
||||
mongo: { auctionLifecycleId: 'new-life', revision: 2 },
|
||||
},
|
||||
{
|
||||
expected: false,
|
||||
expected: true,
|
||||
live: { auctionLifecycleId: 'redis-life', revision: 3 },
|
||||
mongo: { auctionLifecycleId: 'mongo-life', revision: 2 },
|
||||
},
|
||||
|
||||
@@ -62,13 +62,11 @@ export function shouldHydrateRedisAuctionState(
|
||||
params: ShouldHydrateRedisAuctionStateParams,
|
||||
): boolean {
|
||||
if (!params.live) return true;
|
||||
if (params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId)
|
||||
return true;
|
||||
const redisRevision = params.live.revision ?? -1;
|
||||
const mongoRevision = params.mongo.revision;
|
||||
if (mongoRevision > redisRevision) return true;
|
||||
return (
|
||||
mongoRevision === redisRevision &&
|
||||
params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId
|
||||
);
|
||||
return mongoRevision > redisRevision;
|
||||
}
|
||||
|
||||
export function assembleAuctionViewFromRedisState(
|
||||
@@ -140,13 +138,17 @@ if not liveStatePatchPairCount or #ARGV ~= 8 + (liveStatePatchPairCount * 2) the
|
||||
return {'REJECTED', 'INVALID_PATCH_ARITY', tostring(currentRedisTimestampMS)}
|
||||
end
|
||||
|
||||
-- A lifecycle replacement requires Mongo authority and an explicit cleanup
|
||||
-- under the lifecycle lock. Revision numbers from different runs are unrelated.
|
||||
if redisAuctionLifecycleId and redisAuctionLifecycleId ~= mongoAuctionLifecycleId then
|
||||
return {'REJECTED', 'LIFECYCLE_CONFLICT', tostring(currentRedisTimestampMS)}
|
||||
end
|
||||
|
||||
local shouldHydrate = false
|
||||
if redisRevision < 0 then
|
||||
shouldHydrate = true
|
||||
elseif mongoRevision > redisRevision then
|
||||
shouldHydrate = true
|
||||
elseif mongoRevision == redisRevision and mongoAuctionLifecycleId ~= redisAuctionLifecycleId then
|
||||
shouldHydrate = true
|
||||
end
|
||||
|
||||
if not shouldHydrate then
|
||||
@@ -184,6 +186,7 @@ if phase == 'bidding' then
|
||||
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
|
||||
else
|
||||
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
|
||||
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
|
||||
end
|
||||
|
||||
return {'OK', tostring(hydratedRevision), tostring(currentRedisTimestampMS)}
|
||||
|
||||
Reference in new issue
Block a user