fix(auction): harden lifecycle and durable bid storage

Serialize lifecycle changes and make cancellation, deadlines, and match side effects recoverable and idempotent.

Archive accepted bids durably, quarantine invalid payloads, and preserve compatibility with empty round IDs. Synchronize HTTP polling and socket state with scoped resync events.

Add cancellation cleanup index migration and regression coverage. Validated with 744 API tests, 65 Redis integration tests, and 28 Mongo integration tests.
This commit is contained in:
dahyeon committed 2026-11-09 23:24:11 +09:00
1 parent f2eec479d9
commit f649490cd2
55 files changed
+5374 -522

No files matched your search

+53
View File
@@ -62,8 +62,61 @@ $ pnpm run test:e2e
# test coverage
$ pnpm run test:cov
# auction cancellation/undo transactions against a real local MongoDB replica set
$ pnpm run test:mongodb
```
`test:mongodb` requires `mongod` on PATH, or `MONGODB_TEST_SERVER` set to its
executable path. The runner starts an isolated replica set on a random loopback
port, uses a temporary database directory, and removes it after the tests. It
does not use the application's database connection settings.
## Auction durability
Accepted Redis bids now enter `auction:bidArchivePayloads` and
`auction:bidArchiveQueue` atomically with the bid. The durability scheduler saves
them to `auctionBidRecords` using deterministic IDs before removing queue entries.
Recent state snapshots remain capped at 1000 bids; the archive is not capped and
retains accepted bids across undo/cancellation for audit. These records begin with
this deployment; bids already discarded by an older version cannot be recovered.
`GET /communities/:communityId/matches/:matchId/auction/bids/history` pages through
the current lifecycle's persisted archive using `afterRevision` and `limit` (1–200).
Persistence is asynchronous, so an empty last page may gain entries afterward.
Invalid archive entries are moved atomically to `auction:bidArchiveDeadLetterQueue`
and `auction:bidArchiveDeadLetterPayloads`. The payload stores the original text
(or `null` for a missing payload), error, ID, and quarantine time. Monitor these
keys and review quarantined data before repairing/requeuing it; it is not retried
automatically or expired. Other bids continue to persist. Mongo write failures
leave valid entries in the pending queue for idempotent retries.
Older in-flight rounds can produce `bidRoundId: ''`; the archive treats this as
an absent optional round ID. Empty player IDs and other invalid field types
remain validation errors.
Cancellation commits its completion marker and pending cleanup fields in Match.
The scheduler retries cleanup after restarts, with a delay capped at 60 seconds.
Reset notifications include the lifecycle to protect a newly started auction.
Dead-letter status notifications use `auction:deadLetterStatusQueue`; failures to
persist or mirror the terminal status are retried without rerunning the side effect.
Older side-effect dead letters are also reconciled automatically.
Before deploying the cleanup worker, create its partial Match index using the
additive, repeatable migration below. Schema declarations alone do not create
indexes when `MONGODB_AUTO_INDEX` is disabled (the default). Choose the target
environment explicitly; dry-run does not create indexes or modify documents.
```bash
pnpm --filter api auction:indexes -- --env staging --dry-run
pnpm --filter api auction:indexes -- --env staging --apply
# Use --env prod for production.
```
The index is `{ auctionCancellationCleanupRetryAt: 1 }`, restricted to
`{ auctionCancellationCleanupPending: true }`. The worker only reads due pending
entries; it no longer scans legacy cancellation markers without a pending flag.
## Deployment
When you're ready to deploy your NestJS application to production, there are some key steps you can take to ensure it runs as efficiently as possible. Check out the [deployment documentation](https://docs.nestjs.com/deployment) for more information.
+2
View File
@@ -9,6 +9,7 @@
"build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"competition:migrate": "ts-node src/modules/global/database/migrations/run-competition-domain.ts",
"auction:indexes": "ts-node src/modules/global/database/migrations/run-auction-cancellation-index.ts",
"seed:dev": "ts-node src/seed.ts",
"seed:dakao-participant": "ts-node src/seed-dakao-participant.ts",
"start:dev": "NODE_ENV=dev NEW_RELIC_HOME=. node --import newrelic/esm-loader.mjs -r newrelic dist/main.js",
@@ -24,6 +25,7 @@
"test": "jest",
"typecheck": "tsc --noEmit --incremental false",
"test:redis": "node test/run-redis-tests.mjs",
"test:mongodb": "node test/run-mongo-tests.mjs",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
@@ -0,0 +1,82 @@
import mongoose from 'mongoose';
import { AuctionBidArchiveService } from './auction-bid-archive.service';
const matchId = new mongoose.Types.ObjectId().toString();
function bid(overrides: Record<string, unknown> = {}) {
return {
_id: `${matchId}:life:${String(1).padStart(16, '0')}`,
matchId,
auctionLifecycleId: 'life',
revision: 1,
bidRequestId: 'bid-1',
bidderUserId: new mongoose.Types.ObjectId().toString(),
playerUserId: new mongoose.Types.ObjectId().toString(),
bidderTeamIndex: 1,
bidPointAmount: 10,
bidPlacedAtTimestampMS: 1_700_000_000_000,
...overrides,
};
}
function fixture(payload: Record<string, unknown>) {
const raw = JSON.stringify(payload);
const commands: string[][] = [];
const sendCommand = jest.fn(async (command: string[]) => {
commands.push(command);
if (command[0] === 'ZRANGE') return [payload._id];
if (command[0] === 'HMGET') return [raw];
return 1;
});
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
matchedCount: 0,
upsertedCount: operations.length,
}));
const service = new AuctionBidArchiveService(
{ getModel: async () => ({ bulkWrite }) } as never,
{ raw: () => ({ sendCommand }) } as never,
);
return { service, bulkWrite, commands };
}
function savedEvent(bulkWrite: jest.Mock) {
const [[operations]] = bulkWrite.mock.calls as unknown as [
[
Array<{
updateOne: { update: { $setOnInsert: Record<string, unknown> } };
}>,
],
];
return operations[0].updateOne.update.$setOnInsert;
}
function quarantined(commands: string[][]) {
return commands.some(
(command) =>
command[0] === 'EVAL' &&
command[1].includes("redis.call('ZADD', KEYS[4]"),
);
}
it('archives a pre-upgrade bid with an empty round ID without the field', async () => {
const f = fixture(bid({ bidRoundId: '' }));
await expect(f.service.persistPendingBids()).resolves.toBe(1);
expect(quarantined(f.commands)).toBe(false);
expect(savedEvent(f.bulkWrite)).not.toHaveProperty('bidRoundId');
});
it('keeps a non-empty round ID on the archived bid', async () => {
const f = fixture(bid({ bidRoundId: 'round-1' }));
await expect(f.service.persistPendingBids()).resolves.toBe(1);
expect(savedEvent(f.bulkWrite)).toMatchObject({ bidRoundId: 'round-1' });
});
it.each([' ', null, 1])(
'still quarantines an invalid round ID %j',
async (bidRoundId) => {
const f = fixture(bid({ bidRoundId }));
await expect(f.service.persistPendingBids()).resolves.toBe(0);
expect(f.bulkWrite).not.toHaveBeenCalled();
expect(quarantined(f.commands)).toBe(true);
},
);
@@ -0,0 +1,242 @@
import {
BadRequestException,
Injectable,
Logger,
NotFoundException,
} from '@nestjs/common';
import mongoose from 'mongoose';
import type { AuctionBidRecord } from '../global/database/schema';
import { ConnectionService } from '../global/database/connection.service';
import { AuctionRedisService } from './redis/auction-redis.service';
import {
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY,
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY,
} from './redis/auction-redis.keys';
// Compare the payload as well as the ID: a concurrent repair must not be moved
// or acknowledged by a worker that read an older value.
const QUARANTINE_BID_ARCHIVE_SCRIPT = `
local moved = 0
for i = 2, #ARGV, 4 do
local current = redis.call('HGET', KEYS[1], ARGV[i])
local matches = (ARGV[i+1] == '0' and current == false) or (ARGV[i+1] == '1' and current == ARGV[i+2])
if matches and redis.call('ZSCORE', KEYS[2], ARGV[i]) then
redis.call('HSET', KEYS[3], ARGV[i], ARGV[i+3])
redis.call('ZADD', KEYS[4], ARGV[1], ARGV[i])
redis.call('HDEL', KEYS[1], ARGV[i])
redis.call('ZREM', KEYS[2], ARGV[i])
moved = moved + 1
end
end
return moved
`;
@Injectable()
export class AuctionBidArchiveService {
private readonly logger = new Logger(AuctionBidArchiveService.name);
constructor(
private readonly database: ConnectionService,
private readonly redis: AuctionRedisService,
) {}
public async persistPendingBids(limit = 200): Promise<number> {
const client = this.redis.raw();
const reply = await client.sendCommand([
'ZRANGE',
AUCTION_BID_ARCHIVE_QUEUE_KEY,
'0',
String(limit - 1),
]);
const ids = Array.isArray(reply) ? reply.map(String) : [];
if (!ids.length) return 0;
const values = await client.sendCommand<Array<string | null>>([
'HMGET',
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
...ids,
]);
const valid: Array<{ event: AuctionBidRecord; raw: string }> = [];
const invalid: Array<{ id: string; raw: string | null; error: string }> =
[];
for (let index = 0; index < ids.length; index++) {
const raw = values[index] ?? null;
try {
valid.push({ event: parseBidArchiveEvent(ids[index], raw), raw: raw! });
} catch (error) {
invalid.push({
id: ids[index],
raw,
error: String(error).slice(0, 2000),
});
}
}
if (invalid.length) {
const now = Date.now();
const moved = await client.sendCommand<number>([
'EVAL',
QUARANTINE_BID_ARCHIVE_SCRIPT,
'4',
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY,
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY,
String(now),
...invalid.flatMap(({ id, raw, error }) => [
id,
raw === null ? '0' : '1',
raw ?? '',
JSON.stringify({
id,
rawPayload: raw,
error,
quarantinedAtTimestampMS: now,
}),
]),
]);
if (moved > 0)
this.logger.warn(`Quarantined ${moved} invalid bid archive events.`);
}
if (valid.length) {
const model = await this.database.getModel('auctionBidRecord');
// The deterministic _id is the dedupe key, including after a crash between
// Mongo acknowledgement and Redis cleanup. Never trim unpersisted events.
const result = await model.bulkWrite(
valid.map(({ event }) => ({
updateOne: {
filter: { _id: event._id },
update: { $setOnInsert: event },
upsert: true,
},
})),
{ ordered: false, throwOnValidationError: true },
);
// A partial/ambiguous write leaves valid entries queued. Replaying the
// upserts is safe; assuming that a resolved bulkWrite saved all is not.
if (result.matchedCount + result.upsertedCount !== valid.length) {
throw new Error(
'Bid archive batch was not fully acknowledged by MongoDB.',
);
}
await client.sendCommand([
'EVAL',
"for i = 1, #ARGV, 2 do if redis.call('HGET', KEYS[1], ARGV[i]) == ARGV[i+1] then redis.call('HDEL', KEYS[1], ARGV[i]); redis.call('ZREM', KEYS[2], ARGV[i]) end end return 1",
'2',
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
...valid.flatMap(({ event, raw }) => [event._id, raw]),
]);
}
return valid.length;
}
public async list(
matchId: mongoose.Types.ObjectId,
afterRevision = -1,
limit = 100,
) {
if (
!Number.isSafeInteger(afterRevision) ||
afterRevision < -1 ||
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > 200
) {
throw new BadRequestException('Invalid bid history cursor or limit.');
}
const AuctionModel = await this.database.getModel('auction');
const auction = await AuctionModel.findOne({ matchId })
.select({ auctionLifecycleId: 1 })
.lean()
.exec();
if (!auction) throw new NotFoundException('Auction not found.');
const prefix = `${matchId.toString()}:${auction.auctionLifecycleId}:`;
const model = await this.database.getModel('auctionBidRecord');
// The built-in _id index supports ordered pagination without an additional
// production index or an unbounded array inside an Auction document.
const items = await model
.find({
_id: {
$gt:
afterRevision < 0
? prefix
: prefix + String(afterRevision).padStart(16, '0'),
$lt: prefix + '\uffff',
},
})
.sort({ _id: 1 })
.limit(limit + 1)
.lean()
.exec();
const hasMore = items.length > limit;
if (hasMore) items.pop();
return {
items,
hasMore,
nextRevision: items.at(-1)?.revision ?? afterRevision,
};
}
}
function parseBidArchiveEvent(
id: string,
raw: string | null,
): AuctionBidRecord {
if (raw === null) throw new Error('Missing bid archive payload.');
const value: unknown = JSON.parse(raw);
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error('Bid archive payload must be an object.');
}
const event = value as Record<string, unknown>;
const string = (field: string): string => {
const value = event[field];
if (typeof value !== 'string' || !value.trim()) {
throw new Error(`Invalid bid archive ${field}.`);
}
return value;
};
const integer = (field: string, minimum = 0): number => {
const value = event[field];
if (
typeof value !== 'number' ||
!Number.isSafeInteger(value) ||
value < minimum
) {
throw new Error(`Invalid bid archive ${field}.`);
}
return value;
};
const objectId = (field: string): mongoose.Types.ObjectId => {
const value = string(field);
if (!mongoose.isObjectIdOrHexString(value)) {
throw new Error(`Invalid bid archive ${field}.`);
}
return new mongoose.Types.ObjectId(value);
};
const matchId = objectId('matchId');
const auctionLifecycleId = string('auctionLifecycleId');
const revision = integer('revision');
const expectedId = `${matchId.toString()}:${auctionLifecycleId}:${String(revision).padStart(16, '0')}`;
if (event._id !== id || id !== expectedId) {
throw new Error('Bid archive identity does not match its payload.');
}
return {
_id: id,
matchId,
auctionLifecycleId,
revision,
bidRequestId: string('bidRequestId'),
bidderUserId: objectId('bidderUserId'),
bidderTeamIndex: integer('bidderTeamIndex', 1),
bidPointAmount: integer('bidPointAmount'),
bidPlacedAtTimestampMS: integer('bidPlacedAtTimestampMS'),
...(event.playerUserId !== undefined
? { playerUserId: objectId('playerUserId') }
: {}),
// Pre-upgrade rounds have no Redis round ID; the bid Lua records ''.
...(event.bidRoundId !== undefined && event.bidRoundId !== ''
? { bidRoundId: string('bidRoundId') }
: {}),
};
}
@@ -0,0 +1,154 @@
import { NotFoundException } from '@nestjs/common';
import mongoose from 'mongoose';
import { AuctionService } from './auction.service';
const matchId = new mongoose.Types.ObjectId();
const communityId = new mongoose.Types.ObjectId();
function sessionQuery<T>(value: T) {
return { session: async () => value };
}
function fixture(options: {
auction: Record<string, unknown> | null;
match: Record<string, unknown>;
failRedisCleanup?: boolean;
}) {
const session = {
startTransaction: jest.fn(),
commitTransaction: jest.fn().mockResolvedValue(undefined),
abortTransaction: jest.fn().mockResolvedValue(undefined),
endSession: jest.fn().mockResolvedValue(undefined),
};
const deleteMany = jest.fn(() => sessionQuery(undefined));
const models: Record<string, unknown> = {
auction: { findOne: () => sessionQuery(options.auction) },
auctionHistory: { deleteMany },
auctionSideEffectApplication: { deleteMany },
match: { findById: () => sessionQuery(options.match) },
};
const redisCommands: string[][] = [];
const sendCommand = jest.fn(async (command: string[]) => {
redisCommands.push(command);
if (command[0] === 'EVAL' && options.failRedisCleanup)
throw new Error('redis unavailable');
return command[0] === 'SCAN' ? ['0', []] : 1;
});
const service = new AuctionService(
{
getModel: async (name: string) => models[name],
createSession: async () => session,
} as never,
{} as never,
{ releaseFromMatch: jest.fn().mockResolvedValue(undefined) } as never,
{ cancelPolling: jest.fn().mockResolvedValue(undefined) } as never,
{
acquireTokenLock: async () => true,
releaseTokenLock: async () => true,
raw: () => ({ sendCommand }),
} as never,
{} as never,
{} as never,
);
return { service, session, redisCommands };
}
function liveMatch(): Record<string, unknown> {
return {
communityId,
status: 'team-built',
teams: [],
tournamentCodeIds: [],
gameSchedule: [],
participants: [],
save: jest.fn().mockResolvedValue(undefined),
};
}
function redisAuction() {
return {
_id: new mongoose.Types.ObjectId(),
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: 'life',
deleteOne: jest.fn().mockResolvedValue(undefined),
};
}
it('records the cancelled lifecycle and still succeeds when Redis cleanup fails', async () => {
const match = liveMatch();
const f = fixture({
auction: redisAuction(),
match,
failRedisCleanup: true,
});
await expect(
f.service.reset({
matchId,
communityId,
clientObservedAuctionLifecycleId: 'life',
}),
).resolves.toBeUndefined();
expect(f.session.commitTransaction).toHaveBeenCalledTimes(1);
expect(match.lastCancelledAuctionLifecycleId).toBe('life');
expect(match.status).toBe('recruiting');
});
it('answers a retransmitted cancellation and durably schedules Redis cleanup', async () => {
const match = { ...liveMatch(), lastCancelledAuctionLifecycleId: 'life' };
const f = fixture({ auction: null, match });
await expect(
f.service.reset({
matchId,
communityId,
clientObservedAuctionLifecycleId: 'life',
}),
).resolves.toBeUndefined();
expect(match).toMatchObject({
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: expect.any(Date),
});
expect(f.redisCommands).toEqual([]);
});
it('preserves a committed cancellation even if its lock renewal fails before returning', async () => {
jest.useFakeTimers();
try {
const match = liveMatch();
const f = fixture({ auction: redisAuction(), match });
f.session.commitTransaction.mockImplementation(async () => {
await jest.advanceTimersByTimeAsync(10000);
});
await expect(
f.service.reset({
matchId,
communityId,
clientObservedAuctionLifecycleId: 'life',
}),
).resolves.toBeUndefined();
expect(match.auctionCancellationCleanupPending).toBe(true);
expect(f.session.commitTransaction).toHaveBeenCalledTimes(1);
} finally {
jest.useRealTimers();
}
});
it('keeps 404 for a missing auction that was not cancelled with this lifecycle', async () => {
for (const [observed, recorded] of [
['other', 'life'],
[undefined, 'life'],
['life', undefined],
] as const) {
const f = fixture({
auction: null,
match: { ...liveMatch(), lastCancelledAuctionLifecycleId: recorded },
});
await expect(
f.service.reset({
matchId,
communityId,
clientObservedAuctionLifecycleId: observed,
}),
).rejects.toBeInstanceOf(NotFoundException);
expect(f.session.abortTransaction).toHaveBeenCalled();
}
});
@@ -0,0 +1,607 @@
import mongoose from 'mongoose';
import { AuctionService } from './auction.service';
import {
AuctionDurabilityScheduler,
buildAuctionProjectionGuardedUpdate,
type AuctionDurabilityPayload,
} from './auction-durability.scheduler';
import { AuctionDeadlineScheduler } from './auction-deadline.scheduler';
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
const query = <T>(value: T) => ({
select() {
return this;
},
session() {
return this;
},
lean() {
return this;
},
exec: async () => value,
});
const session = () => ({
startTransaction: jest.fn(),
commitTransaction: jest.fn(),
abortTransaction: jest.fn(),
endSession: jest.fn(),
});
const state = {
auctionLifecycleId: 'life',
revision: 12,
phase: 'presenting',
currentAuctionHistorySequence: 7,
currentAuctionPlayerOrderIndex: 0,
currentAuctionRound: 1,
auctionTeams: [],
auctionPlayerOrder: [],
passedAuctionPlayerQueue: [],
currentHighestBidPointAmount: 0,
currentHighestBidderTeamIndex: -1,
auctionCompletedAtTimestampMS: 0,
currentBidRoundDeadlineTimestampMS: 0,
};
beforeEach(() => {
jest
.spyOn(AuctionLifecycleLock.prototype, 'run')
.mockImplementation(async (_id, work) => work());
});
afterEach(() => jest.restoreAllMocks());
function matches(
document: Record<string, unknown>,
filter: Record<string, unknown>,
): boolean {
return Object.entries(filter).every(([key, value]) => {
if (key === '$or')
return (value as Record<string, unknown>[]).some((alternative) =>
matches(document, alternative),
);
if (value && typeof value === 'object') {
return Object.entries(value).every(([operator, limit]) => {
if (operator === '$exists')
return (document[key] !== undefined) === limit;
if (operator === '$lt') return Number(document[key]) < Number(limit);
if (operator === '$lte') return Number(document[key]) <= Number(limit);
return false;
});
}
return document[key] === value;
});
}
function update(
document: Record<string, unknown>,
mutation: Record<string, Record<string, unknown>>,
) {
Object.assign(document, mutation.$set);
for (const [key, amount] of Object.entries(mutation.$inc ?? {}))
document[key] = Number(document[key] ?? 0) + Number(amount);
for (const key of Object.keys(mutation.$unset ?? {})) delete document[key];
}
function sideEffectFixture(withReader = false) {
const projection: Record<string, unknown> = {
_id: 'auction',
auctionLifecycleId: 'life',
revision: 9,
matchSideEffectRevision: 0,
};
const application: Record<string, unknown> = {};
const payload: AuctionDurabilityPayload = {
auctionId: 'auction',
matchId: 'match',
auctionLifecycleId: 'life',
auctionStateTransitionId: 'complete',
auctionActionType: 'complete',
auctionHistorySequence: 10,
revision: 10,
snapshotBefore: {},
historyMutation: { type: 'append' },
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
afterAuctionMongoProjection: {
auctionLifecycleId: 'life',
revision: 10,
phase: 'complete',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
matchSideEffectTransitionId: 'complete',
auctionTeams: [{ teamIndex: 1, captainUserId: 'captain', members: [] }],
},
};
const AuctionModel = {
findById: () => query({ ...projection }),
updateOne: (
filter: Record<string, unknown>,
mutation: Record<string, Record<string, unknown>>,
) => {
const matchedCount = Number(matches(projection, filter));
if (matchedCount) update(projection, mutation);
return query({ matchedCount });
},
findOneAndUpdate: (
filter: Record<string, unknown>,
mutation: Record<string, Record<string, unknown>>,
) => {
if (!matches(projection, filter)) return query(null);
update(projection, mutation);
return query({ ...projection });
},
};
const ApplicationModel = {
findOne: () => query({ ...application }),
updateOne: (
filter: Record<string, unknown>,
mutation: Record<string, Record<string, unknown>>,
) => {
if (mutation.$setOnInsert) {
if (!application.state)
Object.assign(application, mutation.$setOnInsert);
return query({ matchedCount: 1 });
}
const matchedCount = Number(matches(application, filter));
if (matchedCount) update(application, mutation);
return query({ matchedCount });
},
};
const match = {
participants: [],
teams: [],
save: jest.fn().mockResolvedValue(undefined),
};
const historyFindOne = jest.fn().mockReturnValue(query(null));
const historyUpdateOne = jest
.fn()
.mockReturnValue(query({ matchedCount: 1 }));
const models: Record<string, unknown> = {
auction: AuctionModel,
auctionHistory: {
findOne: historyFindOne,
updateOne: historyUpdateOne,
},
auctionSideEffectApplication: ApplicationModel,
match: {
findById: () => ({ ...query(match), session: async () => match }),
},
};
const reserveForMatch = jest
.fn()
.mockRejectedValueOnce(new Error('temporary reservation error'))
.mockResolvedValue(match);
const scheduleInitialPoll = jest.fn().mockResolvedValue(undefined);
const releaseFromMatch = jest.fn().mockResolvedValue(undefined);
const cancelPolling = jest.fn().mockResolvedValue(undefined);
let queued = true;
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGEBYSCORE') return queued ? ['payload'] : [];
if (command[0] === 'HGETALL')
return { payloadJson: JSON.stringify(payload) };
if (command[0] === 'DEL') queued = false;
if (command[0] === 'HINCRBY') return 1;
return 1;
});
const scheduler = new AuctionDurabilityScheduler(
{
getModel: async (name: string) => models[name],
createSession: async () => session(),
} as never,
{
getCurrentRedisTimestampMS: async () => 10000,
raw: () => ({ sendCommand }),
} as never,
{ reserveForMatch, releaseFromMatch } as never,
{ scheduleInitialPoll, cancelPolling } as never,
undefined,
undefined,
withReader
? ({
readAuctionStateView: async () => ({
auctionLifecycleId: 'life',
phase: payload.afterAuctionMongoProjection.phase,
matchSideEffectTransitionId: 'complete',
}),
} as never)
: undefined,
);
return {
scheduler,
payload,
projection,
application,
reserveForMatch,
scheduleInitialPoll,
sendCommand,
match,
historyFindOne,
historyUpdateOne,
releaseFromMatch,
cancelPolling,
};
}
it('retries failed completion work with the current generation and only then deletes the payload', async () => {
const f = sideEffectFixture();
expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({
failed: 1,
});
expect(f.projection.matchSideEffectStatus).toBe('failed');
expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({
applied: 1,
});
expect(f.reserveForMatch).toHaveBeenCalledTimes(2);
expect(f.scheduleInitialPoll).toHaveBeenCalledTimes(1);
expect(f.application.state).toBe('completed');
expect(f.projection.matchSideEffectStatus).toBe('completed');
expect(f.projection.matchSideEffectRevision).toBe(5);
expect(f.sendCommand).toHaveBeenCalledWith(['DEL', 'payload']);
});
it('keeps an outbox payload while another worker owns its side-effect lease', async () => {
const f = sideEffectFixture();
Object.assign(f.application, {
auctionId: 'auction',
auctionStateTransitionId: 'complete',
sideEffectType: 'syncMatchTeamsAfterComplete',
state: 'processing',
processingOwnerToken: 'other',
processingLeaseExpiresAtTimestampMS: 50000,
});
expect(await f.scheduler.processDueDurabilityPayloads(20)).toEqual({
applied: 0,
failed: 0,
skipped: 1,
});
expect(f.sendCommand).not.toHaveBeenCalledWith(['DEL', 'payload']);
expect(
f.sendCommand.mock.calls.some(([command]) => command[0] === 'HINCRBY'),
).toBe(false);
expect(f.reserveForMatch).not.toHaveBeenCalled();
});
it('recovers side effects when a later ordinary projection was persisted first', async () => {
const f = sideEffectFixture(true);
f.reserveForMatch.mockReset().mockResolvedValue(undefined);
Object.assign(f.projection, { revision: 12 });
await f.scheduler.applyDurabilityPayload(f.payload);
expect(f.reserveForMatch).toHaveBeenCalledTimes(1);
expect(f.projection.revision).toBe(12);
expect(f.projection.matchSideEffectTransitionId).toBe('complete');
expect(f.projection.matchSideEffectStatus).toBe('completed');
});
it('still cleans up completed Match state when a second undo invalidates the first undo history', async () => {
const f = sideEffectFixture(true);
Object.assign(f.projection, { revision: 12 });
Object.assign(f.payload, {
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
});
f.payload.afterAuctionMongoProjection.phase = 'presenting';
Object.assign(f.match, {
teams: [{ teamIndex: 1 }],
teamBuiltAt: new Date(),
tournamentCodeIds: ['code'],
gameSchedule: [{}],
});
f.historyFindOne.mockReturnValue(
query({
sequence: 12,
historyMutation: {
type: 'deleteAfterSequenceAndAppendUndo',
targetHistorySequence: 2,
},
}),
);
await f.scheduler.applyDurabilityPayload(f.payload);
expect(f.historyUpdateOne).not.toHaveBeenCalled();
expect(f.releaseFromMatch).toHaveBeenCalledTimes(1);
expect(f.cancelPolling).toHaveBeenCalledTimes(1);
expect(f.match).toMatchObject({
teams: [],
teamBuiltAt: undefined,
tournamentCodeIds: [],
gameSchedule: [],
});
expect(f.projection.matchSideEffectStatus).toBe('completed');
});
it('does not let a superseded side effect overwrite the newer generation', async () => {
const f = sideEffectFixture();
Object.assign(f.projection, {
revision: 12,
matchSideEffectTransitionId: 'undo',
matchSideEffectRevision: 6,
});
await f.scheduler.applyDurabilityPayload(f.payload);
expect(f.reserveForMatch).not.toHaveBeenCalled();
expect(f.projection.matchSideEffectTransitionId).toBe('undo');
});
it('removes obsolete projection fields both from Mongo updates and subsequent undo snapshots', () => {
const service = new AuctionService(
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
);
const internal = service as unknown as {
buildAuctionMongoProjectionFromState(
auction: unknown,
state: Record<string, unknown>,
): Record<string, unknown>;
buildAuctionSnapshotFromRedisState(
auction: unknown,
state: Record<string, unknown>,
): Record<string, unknown>;
};
const old = {
_id: new mongoose.Types.ObjectId(),
matchId: new mongoose.Types.ObjectId(),
completedAt: new Date(),
auctionCompletedAtTimestampMS: 100,
currentHighestBidderTeamIndex: 2,
currentBidderTeamIndex: 2,
};
const projection = internal.buildAuctionMongoProjectionFromState(old, state);
const mutation = buildAuctionProjectionGuardedUpdate({
auctionId: old._id,
auctionLifecycleId: 'life',
revision: 12,
afterAuctionMongoProjection: projection,
} as AuctionDurabilityPayload).update;
const stored = { ...old } as Record<string, unknown>;
update(stored, mutation);
const snapshot = internal.buildAuctionSnapshotFromRedisState(old, state);
for (const record of [stored, snapshot]) {
expect(record).not.toHaveProperty('completedAt');
expect(record).not.toHaveProperty('currentHighestBidderTeamIndex');
expect(record.phase).toBe('presenting');
}
});
it('applies delayed undo history cleanup without deleting newer branch entries or rolling back projection', async () => {
const deleteMany = jest.fn().mockReturnValue(query({}));
const updateOne = jest.fn().mockReturnValue(query({}));
const projectionUpdate = jest.fn();
const models: Record<string, unknown> = {
auction: {
findById: () => query({ auctionLifecycleId: 'life', revision: 12 }),
updateOne: projectionUpdate,
},
auctionHistory: { findOne: () => query(null), deleteMany, updateOne },
auctionSideEffectApplication: {},
};
const scheduler = new AuctionDurabilityScheduler(
{
getModel: async (name: string) => models[name],
createSession: async () => null,
} as never,
{} as never,
);
await scheduler.applyDurabilityPayload({
auctionId: 'auction',
matchId: 'match',
auctionStateTransitionId: 'undo',
auctionLifecycleId: 'life',
revision: 11,
auctionHistorySequence: 9,
auctionActionType: 'undo',
historyMutation: {
type: 'deleteAfterSequenceAndAppendUndo',
targetHistorySequence: 3,
},
matchSideEffectMutation: { type: 'none' },
afterAuctionMongoProjection: {},
snapshotBefore: {},
} as AuctionDurabilityPayload);
expect(deleteMany).toHaveBeenCalledWith(
{ auctionId: 'auction', sequence: { $gt: 3, $lt: 9 } },
{ session: null },
);
expect(updateOne).toHaveBeenCalledTimes(1);
expect(projectionUpdate).not.toHaveBeenCalled();
});
it('rejects a cancellation from a prior lifecycle before deleting anything', async () => {
const matchId = new mongoose.Types.ObjectId(),
communityId = new mongoose.Types.ObjectId();
const deleteOne = jest.fn();
const service = new AuctionService(
{
createSession: async () => session(),
getModel: async () => ({
findOne: () => ({
session: async () => ({
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: 'new-life',
deleteOne,
}),
}),
}),
} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
);
await expect(
service.reset({
matchId,
communityId,
clientObservedAuctionLifecycleId: 'old-life',
}),
).rejects.toThrow('lifecycle has changed');
expect(deleteOne).not.toHaveBeenCalled();
});
it('resumes startup after Mongo creation instead of returning an unfinished auction', async () => {
const matchId = new mongoose.Types.ObjectId(),
communityId = new mongoose.Types.ObjectId();
const captain = new mongoose.Types.ObjectId(),
player = new mongoose.Types.ObjectId(),
lateJoiner = new mongoose.Types.ObjectId();
const auction = {
_id: new mongoose.Types.ObjectId(),
matchId,
auctionLifecycleId: 'life',
revision: 1,
phase: 'presenting',
startupCompleted: false,
startAuctionRequestId: 'same-request',
teams: [{ captainUserId: captain, members: [], teamIndex: 1 }],
order: [{ userId: player }],
};
const match = {
communityId,
status: 'recruiting',
participants: [captain, player, lateJoiner].map((userId) => ({
userId,
excluded: false,
})),
save: jest.fn(),
};
const updateOne = jest.fn();
const historyUpdate = jest.fn();
const models: Record<string, unknown> = {
auction: { findOne: async () => auction, updateOne },
match: { findById: async () => match },
auctionHistory: { updateOne: historyUpdate },
};
const hydrateState = jest.fn().mockResolvedValue({ hydrated: true });
const service = new AuctionService(
{ getModel: async (name: string) => models[name] } as never,
{} as never,
{} as never,
{} as never,
{} as never,
{ hydrateState } as never,
{} as never,
);
await service.start({
matchId,
communityId,
startAuctionRequestId: 'same-request',
} as never);
expect(hydrateState).toHaveBeenCalledTimes(1);
expect(historyUpdate).toHaveBeenCalledTimes(1);
expect(match.save).toHaveBeenCalledTimes(1);
expect(match.status).toBe('team-built');
expect(auction.startupCompleted).toBe(true);
expect(match.participants.map((entry) => entry.excluded)).toEqual([
false,
false,
true,
]);
});
it('does not open bidding until interrupted startup has been completed', async () => {
const service = new AuctionService(
{
getModel: async () => ({
findOne: async () => ({
auctionEngineVersion: 'redis-v1',
startupCompleted: false,
}),
}),
} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
);
await expect(
service.openBidding(new mongoose.Types.ObjectId()),
).rejects.toThrow('startup is not complete');
});
it('uses Redis completion state to schedule cleanup despite a delayed Mongo projection', async () => {
const matchId = new mongoose.Types.ObjectId(),
communityId = new mongoose.Types.ObjectId();
const auction = {
_id: new mongoose.Types.ObjectId(),
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: 'life',
phase: 'sold',
revision: 9,
};
const snapshot = { ...state, phase: 'bidding', revision: 5 };
const models: Record<string, unknown> = {
auction: { findOne: () => ({ session: async () => auction }) },
match: { findById: () => ({ session: async () => ({ communityId }) }) },
auctionHistory: { findOne: () => query({ snapshotBefore: snapshot }) },
};
const applyTransition = jest.fn().mockResolvedValue({ ok: true });
const commands = {
reserveActionRequest: async () => ({
reserved: true,
actionRequestScopeKey: 'scope',
auctionStateTransitionId: 'undo',
observedProcessingLeaseVersion: 1,
}),
applyTransition,
};
const service = new AuctionService(
{
getModel: async (name: string) => models[name],
createSession: async () => session(),
} as never,
{} as never,
{} as never,
{} as never,
{} as never,
commands as never,
{
readAuctionStateView: async () => ({
...state,
phase: 'complete',
revision: 10,
}),
} as never,
);
await service.undoTo({ matchId, communityId, sequence: 2 });
const input = applyTransition.mock.calls[0][1] as {
durabilityPayloadJson: string;
};
expect(JSON.parse(input.durabilityPayloadJson)).toMatchObject({
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
});
});
it.each(['deadline', 'durability'])(
'continues %s ticks when lock release fails',
async (kind) => {
const redis = {
acquireTokenLock: jest.fn().mockResolvedValue(true),
releaseTokenLock: jest
.fn()
.mockRejectedValue(new Error('redis disconnected')),
};
const scheduler =
kind === 'deadline'
? new AuctionDeadlineScheduler(
redis as never,
{} as never,
{} as never,
{} as never,
)
: new AuctionDurabilityScheduler({} as never, redis as never);
const work = jest.fn().mockResolvedValue({});
Object.assign(
scheduler,
kind === 'deadline'
? { processDueBidRoundDeadlines: work }
: { processDueDurabilityPayloads: work },
);
const internal = scheduler as unknown as { safeTick(): Promise<void> };
await internal.safeTick();
await internal.safeTick();
expect(work).toHaveBeenCalledTimes(2);
},
);
@@ -88,10 +88,13 @@ export class AuctionDeadlineScheduler implements OnModuleInit, OnModuleDestroy {
);
} finally {
if (leaderLockAcquired) {
await this.auctionRedisService.releaseTokenLock(
BID_ROUND_DEADLINE_LEADER_LOCK_KEY,
leaderLockToken,
);
await this.auctionRedisService
.releaseTokenLock(BID_ROUND_DEADLINE_LEADER_LOCK_KEY, leaderLockToken)
.catch((error: unknown) => {
this.logger.warn(
`Auction deadline lock release failed: ${String(error)}`,
);
});
}
this.tickInProgress = false;
}
@@ -1,3 +1,5 @@
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
import { buildAuctionProjectionUnset } from './auction-projection';
import {
AuctionDurabilityScheduler,
buildAuctionHistoryUpsert,
@@ -23,6 +25,12 @@ const payload = {
} as const;
describe('auction durability projection helpers', () => {
beforeEach(() => {
jest
.spyOn(AuctionLifecycleLock.prototype, 'run')
.mockImplementation(async (_id, work) => work());
});
afterEach(() => jest.restoreAllMocks());
it('builds idempotent history upsert by auction and transition id', () => {
expect(buildAuctionHistoryUpsert(payload)).toEqual({
updateOne: {
@@ -59,6 +67,9 @@ describe('auction durability projection helpers', () => {
},
update: {
$set: { phase: 'sold', revision: 9 },
$unset: buildAuctionProjectionUnset(
payload.afterAuctionMongoProjection,
),
},
});
});
@@ -210,7 +221,12 @@ describe('auction durability projection helpers', () => {
auctionLifecycleId: 'life-1',
revision: { $lt: 9 },
},
{ $set: { phase: 'sold', revision: 9 } },
{
$set: { phase: 'sold', revision: 9 },
$unset: buildAuctionProjectionUnset(
payload.afterAuctionMongoProjection,
),
},
{ session },
);
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
@@ -370,7 +386,7 @@ describe('auction durability projection helpers', () => {
expect(historyDeleteMany).toHaveBeenCalledWith(
{
auctionId: 'auction-1',
sequence: { $gt: 2 },
sequence: { $gt: 2, $lt: 4 },
},
{ session },
);
@@ -811,21 +827,14 @@ describe('auction durability projection helpers', () => {
skipped: 0,
});
expect(sendCommand).toHaveBeenNthCalledWith(5, [
'ZREM',
'EVAL',
expect.stringContaining("redis.call('HEXISTS'"),
'4',
'payload-1',
'auction:auctionDurabilityRetryQueue',
'payload-1',
]);
expect(sendCommand).toHaveBeenNthCalledWith(6, [
'HSET',
'payload-1',
'deadLetteredAtRedisTimestampMS',
'1000',
]);
expect(sendCommand).toHaveBeenNthCalledWith(7, [
'ZADD',
'auction:auctionDurabilityDeadLetterQueue',
'auction:deadLetterStatusQueue',
'1000',
'payload-1',
]);
});
@@ -1030,11 +1039,20 @@ describe('auction durability projection helpers', () => {
{
_id: 'auction-1',
auctionLifecycleId: 'life-1',
matchSideEffectRevision: 1,
$or: [
{ matchSideEffectTransitionId: 'transition-1' },
{
matchSideEffectTransitionId: { $exists: false },
matchSideEffectRevision: 1,
},
],
},
{
$inc: { matchSideEffectRevision: 1 },
$set: { matchSideEffectStatus: 'processing' },
$set: {
matchSideEffectStatus: 'processing',
matchSideEffectTransitionId: 'transition-1',
},
$unset: { matchSideEffectError: '' },
},
expect.objectContaining({ returnDocument: 'after' }),
@@ -1045,34 +1063,42 @@ describe('auction durability projection helpers', () => {
_id: 'auction-1',
auctionLifecycleId: 'life-1',
matchSideEffectRevision: 2,
matchSideEffectTransitionId: 'transition-1',
},
{
$inc: { matchSideEffectRevision: 1 },
$set: { matchSideEffectStatus: 'completed' },
$set: {
matchSideEffectStatus: 'completed',
matchSideEffectTransitionId: 'transition-1',
},
$unset: { matchSideEffectError: '' },
},
expect.objectContaining({ returnDocument: 'after' }),
);
expect(mirrorMatchSideEffectStatus).toHaveBeenCalledWith({
auctionLifecycleId: 'life-1',
auctionStateTransitionId: 'transition-1',
matchId: 'match-1',
matchSideEffectRevision: 2,
matchSideEffectStatus: 'processing',
});
expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({
auctionLifecycleId: 'life-1',
auctionStateTransitionId: 'transition-1',
matchId: 'match-1',
matchSideEffectRevision: 3,
matchSideEffectStatus: 'completed',
});
expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(1, 'match-1', {
auctionLifecycleId: 'life-1',
auctionStateTransitionId: 'transition-1',
currentServerTimestampMS: 2000,
matchSideEffectRevision: 2,
matchSideEffectStatus: 'processing',
});
expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(2, 'match-1', {
auctionLifecycleId: 'life-1',
auctionStateTransitionId: 'transition-1',
currentServerTimestampMS: 3000,
matchSideEffectRevision: 3,
matchSideEffectStatus: 'completed',
@@ -1080,7 +1106,7 @@ describe('auction durability projection helpers', () => {
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining("redis.call('EXPIRE'"),
'11',
'12',
'auction:match-1:liveAuctionState',
'auction:bidRoundDeadlineQueue',
'auction:match-1:auctionStructureState',
@@ -1092,6 +1118,7 @@ describe('auction durability projection helpers', () => {
'auction:match-1:auctionActionRequestResultOrder',
'auction:match-1:rehydrateLock',
'auction:match-1:auctionStateTransitionLock',
'auction:match-1:rejectedBidRequestResultOrder',
'match-1',
'life-1',
String(7 * 24 * 60 * 60),
@@ -1263,7 +1290,12 @@ describe('auction durability projection helpers', () => {
const broadcastMatchSideEffect = jest.fn();
const scheduler = new AuctionDurabilityScheduler(
{
createSession: jest.fn().mockResolvedValue(null),
createSession: jest.fn().mockResolvedValueOnce(null).mockResolvedValue({
startTransaction: jest.fn(),
commitTransaction: jest.fn(),
abortTransaction: jest.fn(),
endSession: jest.fn(),
}),
getModel: jest.fn((name: string) => {
if (name === 'auctionHistory') {
return {
@@ -1282,7 +1314,11 @@ describe('auction durability projection helpers', () => {
return { updateOne: sideEffectUpdateOne };
}
if (name === 'match')
return { findById: jest.fn().mockReturnValue(match) };
return {
findById: jest.fn().mockReturnValue({
session: jest.fn().mockResolvedValue(match),
}),
};
return {};
}),
} as never,
@@ -1301,8 +1337,8 @@ describe('auction durability projection helpers', () => {
await scheduler.applyDurabilityPayload(resetPayload);
expect(releaseFromMatch).toHaveBeenCalledWith(match);
expect(cancelPolling).toHaveBeenCalledWith(match);
expect(releaseFromMatch).toHaveBeenCalledWith(match, expect.any(Object));
expect(cancelPolling).toHaveBeenCalledWith(match, expect.any(Object));
expect(match).toMatchObject({
gameSchedule: [],
result: undefined,
@@ -1328,12 +1364,14 @@ describe('auction durability projection helpers', () => {
);
expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({
auctionLifecycleId: 'life-1',
auctionStateTransitionId: 'transition-1',
matchId: 'match-1',
matchSideEffectRevision: 3,
matchSideEffectStatus: 'completed',
});
expect(broadcastMatchSideEffect).toHaveBeenLastCalledWith('match-1', {
auctionLifecycleId: 'life-1',
auctionStateTransitionId: 'transition-1',
currentServerTimestampMS: 3000,
matchSideEffectRevision: 3,
matchSideEffectStatus: 'completed',
@@ -4,6 +4,7 @@ import {
OnModuleDestroy,
OnModuleInit,
Optional,
ConflictException,
} from '@nestjs/common';
import { createHash, randomUUID } from 'crypto';
import type mongoose from 'mongoose';
@@ -16,6 +17,10 @@ import type {
import { TournamentCodeService } from '../tournament-code/tournament-code.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { AuctionEventBusService } from './auction-event-bus.service';
import { AuctionService } from './auction.service';
import { AuctionBidArchiveService } from './auction-bid-archive.service';
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
import { buildAuctionProjectionUnset } from './auction-projection';
import {
AuctionSideEffectService,
buildMatchSideEffectDelta,
@@ -26,6 +31,7 @@ import { AuctionRedisService } from './redis/auction-redis.service';
import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service';
import {
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
buildAuctionRedisKeys,
} from './redis/auction-redis.keys';
@@ -36,6 +42,8 @@ const AUCTION_DURABILITY_BATCH_LIMIT = 20;
const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000;
const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000;
const AUCTION_DURABILITY_MAX_ATTEMPTS = 10;
const AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS = 120;
const AUCTION_DURABILITY_MAX_PENDING_DURATION_MS = 10 * 60 * 1000;
const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000;
const AUCTION_DURABILITY_LEADER_LOCK_KEY =
'auction:auctionDurabilityScheduler:leader';
@@ -45,6 +53,7 @@ const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000;
const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete';
const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset';
const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects';
class SideEffectPendingError extends Error {}
export interface AuctionDurabilityPayload {
actorUserId?: unknown;
@@ -67,6 +76,7 @@ export interface AuctionDurabilityPayload {
export interface AuctionProjectionIdentity {
auctionLifecycleId?: string;
revision?: number;
matchSideEffectTransitionId?: string;
}
type ParsedAuctionHistoryMutation =
@@ -85,7 +95,7 @@ interface CleanupSideEffectMatchDocument {
participants: Array<{ excluded?: boolean }>;
result?: unknown;
resultPollState?: unknown;
save(): Promise<unknown>;
save(options?: { session: mongoose.ClientSession }): Promise<unknown>;
status: string;
teamBuiltAt?: Date;
teams?: unknown[];
@@ -99,6 +109,7 @@ export class AuctionDurabilityScheduler
private readonly logger = new Logger(AuctionDurabilityScheduler.name);
private tickTimer?: NodeJS.Timeout;
private tickInProgress = false;
private deadLetterScanCursor = '0';
constructor(
private readonly connectionService: ConnectionService,
@@ -113,6 +124,9 @@ export class AuctionDurabilityScheduler
private readonly auctionEventBusService?: AuctionEventBusService,
@Optional()
private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService,
@Optional() private readonly auctionService?: AuctionService,
@Optional()
private readonly auctionBidArchiveService?: AuctionBidArchiveService,
) {}
public onModuleInit(): void {
@@ -187,6 +201,23 @@ export class AuctionDurabilityScheduler
});
await this.clearDurabilityPayload(payloadKey);
} catch (error) {
if (
error instanceof SideEffectPendingError ||
error instanceof ConflictException
) {
const disposition = await this.deferDurabilityPayload(
payloadKey,
currentRedisTimestampMS,
error,
);
if (disposition === 'dead-lettered') {
failed += 1;
this.logger.warn(
`Auction durability payload exceeded its pending limit: ${payloadKey}`,
);
} else skipped += 1;
continue;
}
failed += 1;
const failureDisposition = await this.recordDurabilityPayloadFailure(
payloadKey,
@@ -214,6 +245,14 @@ export class AuctionDurabilityScheduler
const payloadJson = hash.payloadJson;
if (!payloadJson) return null;
const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload;
const committedDeadline = Number(hash.committedBidRoundDeadlineTimestampMS);
if (Number.isFinite(committedDeadline) && committedDeadline > 0) {
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS =
committedDeadline;
payload.afterAuctionMongoProjection.currentDeadline = new Date(
committedDeadline,
).toISOString();
}
const committedAtRedisTimestampMS = Number(
hash.committedAtRedisTimestampMS,
);
@@ -240,12 +279,12 @@ export class AuctionDurabilityScheduler
const sideEffectMutation = parseMatchSideEffectMutation(
payload.matchSideEffectMutation,
);
let shouldUpdateProjection = true;
let shouldApplySideEffects = true;
let shouldApplyHistoryMutation = true;
try {
session?.startTransaction();
const historyMutation = parseAuctionHistoryMutation(
payload.historyMutation,
);
if (typeof AuctionModel.findById === 'function') {
const currentProjection =
await readAuctionProjectionIdentityForMutation(
@@ -256,37 +295,44 @@ export class AuctionDurabilityScheduler
if (
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
) {
// 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다.
// 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다.
if (
historyMutation.type === 'append' &&
currentProjection.auctionLifecycleId === payload.auctionLifecycleId
currentProjection.auctionLifecycleId !== payload.auctionLifecycleId
) {
await this.applyStaleAppendHistoryRecovery({
if (session) await session.commitTransaction();
return;
}
shouldApplyHistoryMutation =
!(await this.isAppendHistoryInvalidatedByLaterDestructiveMutation({
AuctionHistoryModel,
payload,
session,
});
}
if (session) await session.commitTransaction();
return;
}));
// History operations have their own sequence bounds. A newer ordinary
// projection must not discard an earlier undo's history cleanup.
shouldUpdateProjection = false;
shouldApplySideEffects =
Boolean(this.auctionRedisStateReaderService) ||
currentProjection.matchSideEffectTransitionId ===
payload.auctionStateTransitionId;
}
}
await this.applyAuctionHistoryMutation({
AuctionHistoryModel,
AuctionModel,
payload,
session,
});
if (shouldApplyHistoryMutation)
await this.applyAuctionHistoryMutation({
AuctionHistoryModel,
AuctionModel,
payload,
session,
});
const projectionOperation = buildAuctionProjectionGuardedUpdate(payload);
await execMaybe(
AuctionModel.updateOne(
projectionOperation.filter as never,
projectionOperation.update as never,
{ session } as never,
),
);
if (sideEffectMutation) {
if (shouldUpdateProjection)
await execMaybe(
AuctionModel.updateOne(
projectionOperation.filter as never,
projectionOperation.update as never,
{ session } as never,
),
);
if (sideEffectMutation && shouldApplySideEffects) {
const sideEffectOperation = buildSideEffectApplicationUpsert(
payload,
sideEffectMutation,
@@ -312,7 +358,7 @@ export class AuctionDurabilityScheduler
await this.broadcastLatestRedisAuctionStateIfRequested(payload);
if (sideEffectMutation) {
if (sideEffectMutation && shouldApplySideEffects) {
await this.applyMatchSideEffect(payload, sideEffectMutation);
}
}
@@ -352,6 +398,64 @@ export class AuctionDurabilityScheduler
private async applyMatchSideEffect(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
return new AuctionLifecycleLock(this.auctionRedisService).run(
String(payload.matchId),
async () => {
if (this.auctionRedisStateReaderService) {
const state =
await this.auctionRedisStateReaderService.readAuctionStateView(
String(payload.matchId),
);
if (!state)
throw new SideEffectPendingError(
'Auction realtime state is unavailable.',
);
if (
state.auctionLifecycleId !== payload.auctionLifecycleId ||
(state.matchSideEffectTransitionId &&
state.matchSideEffectTransitionId !==
payload.auctionStateTransitionId) ||
(mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE &&
state.phase !== 'complete')
)
return;
// An ordinary later projection may have reached Mongo before this
// side-effect-bearing transition. Install its pending generation only
// while Redis still names it and Mongo has not advanced beyond it.
const AuctionModel = await this.connectionService.getModel('auction');
const pendingRevision = Number(
payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
);
await execMaybe(
AuctionModel.updateOne(
{
_id: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
$or: [
{ matchSideEffectRevision: { $lt: pendingRevision } },
{ matchSideEffectRevision: { $exists: false } },
],
} as never,
{
$set: {
matchSideEffectTransitionId: payload.auctionStateTransitionId,
matchSideEffectRevision: pendingRevision,
matchSideEffectStatus: 'pending',
},
$unset: { matchSideEffectError: '' },
} as never,
),
);
}
await this.applyMatchSideEffectWhileLocked(payload, mutation);
},
);
}
private async applyMatchSideEffectWhileLocked(
payload: AuctionDurabilityPayload,
mutation: ParsedMatchSideEffectMutation,
): Promise<void> {
if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) {
await this.applySyncMatchTeamsSideEffect(payload, mutation);
@@ -407,7 +511,26 @@ export class AuctionDurabilityScheduler
}) as never,
),
);
if (!hasMatchedDocument(leaseResult)) return;
if (!hasMatchedDocument(leaseResult)) {
const existing = await AuctionSideEffectApplicationModel.findOne(
identity as never,
)
.lean()
.exec();
if (existing?.state === 'completed') {
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: currentRedisTimestampMS,
payload,
status: 'completed',
});
await this.expireCompletedAuctionRedisRuntime(payload);
return;
}
throw new SideEffectPendingError(
'Auction side effect is still owned by another worker.',
);
}
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
{
@@ -420,7 +543,11 @@ export class AuctionDurabilityScheduler
status: 'processing',
},
);
if (!processingDelta) return;
if (!processingDelta) {
throw new SideEffectPendingError(
'Auction side-effect generation is not ready.',
);
}
const expectedMatchSideEffectRevision =
processingDelta.matchSideEffectRevision;
@@ -489,7 +616,11 @@ export class AuctionDurabilityScheduler
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
completedAtTimestampMS,
@@ -518,7 +649,11 @@ export class AuctionDurabilityScheduler
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
lastError: error instanceof Error ? error.message : String(error),
@@ -544,33 +679,6 @@ export class AuctionDurabilityScheduler
}
}
private async applyStaleAppendHistoryRecovery(params: {
AuctionHistoryModel: {
findOne?: (...args: never[]) => unknown;
updateOne: (...args: never[]) => unknown;
};
payload: AuctionDurabilityPayload;
session: mongoose.ClientSession | null | undefined;
}): Promise<void> {
const invalidated =
await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params);
if (invalidated) return;
const historyOperation = buildAuctionHistoryUpsert(
params.payload,
).updateOne;
await execMaybe(
params.AuctionHistoryModel.updateOne(
historyOperation.filter as never,
historyOperation.update as never,
{
session: params.session,
upsert: true,
} as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
);
}
private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: {
AuctionHistoryModel: {
findOne?: (...args: never[]) => unknown;
@@ -622,9 +730,7 @@ export class AuctionDurabilityScheduler
await this.applyMatchCleanupSideEffect({
mutation,
payload,
updateMatch: async (match) => {
await this.tournamentCodeService?.releaseFromMatch(match as never);
await this.tournamentResultService?.cancelPolling(match as never);
updateMatch: (match) => {
match.status = 'recruiting';
match.teams = [];
match.tournamentCodeIds = [];
@@ -635,7 +741,6 @@ export class AuctionDurabilityScheduler
match.teamBuiltAt = undefined;
match.result = undefined;
match.resultPollState = undefined;
await match.save();
},
});
}
@@ -647,13 +752,14 @@ export class AuctionDurabilityScheduler
await this.applyMatchCleanupSideEffect({
mutation,
payload,
updateMatch: async (match) => {
await this.tournamentCodeService?.releaseFromMatch(match as never);
await this.tournamentResultService?.cancelPolling(match as never);
updateMatch: (match) => {
match.status = 'team-built';
match.teams = [];
match.teamBuiltAt = undefined;
match.tournamentCodeIds = [];
match.gameSchedule = [];
match.result = undefined;
match.resultPollState = undefined;
await match.save();
},
});
}
@@ -661,7 +767,7 @@ export class AuctionDurabilityScheduler
private async applyMatchCleanupSideEffect(params: {
mutation: ParsedMatchSideEffectMutation;
payload: AuctionDurabilityPayload;
updateMatch: (match: CleanupSideEffectMatchDocument) => Promise<void>;
updateMatch: (match: CleanupSideEffectMatchDocument) => void;
}): Promise<void> {
if (!this.tournamentCodeService || !this.tournamentResultService) {
throw new Error('Auction side-effect services are not available.');
@@ -700,7 +806,25 @@ export class AuctionDurabilityScheduler
}) as never,
),
);
if (!hasMatchedDocument(leaseResult)) return;
if (!hasMatchedDocument(leaseResult)) {
const existing = await AuctionSideEffectApplicationModel.findOne(
identity as never,
)
.lean()
.exec();
if (existing?.state === 'completed') {
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel,
currentServerTimestampMS: currentRedisTimestampMS,
payload: params.payload,
status: 'completed',
});
return;
}
throw new SideEffectPendingError(
'Auction side effect is still owned by another worker.',
);
}
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
{
@@ -714,29 +838,56 @@ export class AuctionDurabilityScheduler
status: 'processing',
},
);
if (!processingDelta) return;
if (!processingDelta) {
throw new SideEffectPendingError(
'Auction side-effect generation is not ready.',
);
}
const expectedMatchSideEffectRevision =
processingDelta.matchSideEffectRevision;
try {
const match = (await execMaybe(
MatchModel.findById(params.payload.matchId) as never,
)) as CleanupSideEffectMatchDocument | null;
if (!match) {
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
}
if (match.competitionId) {
throw new Error(
'Cannot reset or undo Match teams after the immutable Competition has been created.',
const session = await this.connectionService.createSession();
if (!session)
throw new Error('Failed to start Match cleanup transaction.');
try {
session.startTransaction();
const match = (await MatchModel.findById(
params.payload.matchId,
).session(session)) as CleanupSideEffectMatchDocument | null;
if (!match)
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
if (match.competitionId)
throw new Error(
'Cannot reset or undo Match teams after the immutable Competition has been created.',
);
await this.tournamentCodeService.releaseFromMatch(
match as never,
session,
);
await this.tournamentResultService.cancelPolling(
match as never,
session,
);
params.updateMatch(match);
await match.save({ session });
await session.commitTransaction();
} catch (error) {
await session.abortTransaction();
throw error;
} finally {
await session.endSession();
}
await params.updateMatch(match);
const completedAtTimestampMS =
await this.auctionRedisService.getCurrentRedisTimestampMS();
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
completedAtTimestampMS,
@@ -766,7 +917,11 @@ export class AuctionDurabilityScheduler
error instanceof Error ? error.message : String(error);
await execMaybe(
AuctionSideEffectApplicationModel.updateOne(
identity as never,
{
...identity,
processingOwnerToken: ownerToken,
state: 'processing',
} as never,
{
$set: {
lastError: errorMessage,
@@ -828,7 +983,10 @@ export class AuctionDurabilityScheduler
params.AuctionHistoryModel.deleteMany(
{
auctionId: params.payload.auctionId,
sequence: { $gt: mutation.targetHistorySequence },
sequence: {
$gt: mutation.targetHistorySequence,
$lt: params.payload.auctionHistorySequence,
},
} as never,
{ session: params.session } as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
@@ -842,6 +1000,7 @@ export class AuctionDurabilityScheduler
params.AuctionHistoryModel.deleteMany(
{
auctionId: params.payload.auctionId,
sequence: { $lt: params.payload.auctionHistorySequence },
} as never,
{ session: params.session } as never,
) as Promise<unknown> | { exec(): Promise<unknown> },
@@ -874,16 +1033,43 @@ export class AuctionDurabilityScheduler
};
currentServerTimestampMS: number;
errorMessage?: string;
expectedMatchSideEffectRevision: number;
expectedMatchSideEffectRevision?: number;
payload: AuctionDurabilityPayload;
status: AuctionMatchSideEffectStatus;
onlyUnfinished?: boolean;
}): Promise<MatchSideEffectDelta | null> {
const updatedAuction = (await execMaybe(
params.AuctionModel.findOneAndUpdate(
{
_id: params.payload.auctionId,
auctionLifecycleId: params.payload.auctionLifecycleId,
matchSideEffectRevision: params.expectedMatchSideEffectRevision,
...(params.onlyUnfinished
? { matchSideEffectStatus: { $ne: 'completed' } }
: {}),
...(params.status === 'processing'
? {
$or: [
{
matchSideEffectTransitionId:
params.payload.auctionStateTransitionId,
},
{
matchSideEffectTransitionId: { $exists: false },
matchSideEffectRevision:
params.expectedMatchSideEffectRevision,
},
],
}
: {
matchSideEffectTransitionId:
params.payload.auctionStateTransitionId,
...(params.expectedMatchSideEffectRevision === undefined
? {}
: {
matchSideEffectRevision:
params.expectedMatchSideEffectRevision,
}),
}),
} as never,
{
$inc: { matchSideEffectRevision: 1 },
@@ -892,6 +1078,8 @@ export class AuctionDurabilityScheduler
? { matchSideEffectError: params.errorMessage }
: {}),
matchSideEffectStatus: params.status,
matchSideEffectTransitionId:
params.payload.auctionStateTransitionId,
},
...(params.errorMessage
? {}
@@ -915,6 +1103,7 @@ export class AuctionDurabilityScheduler
const delta = buildMatchSideEffectDelta({
auctionLifecycleId: updatedAuction.auctionLifecycleId,
auctionStateTransitionId: params.payload.auctionStateTransitionId,
currentServerTimestampMS: params.currentServerTimestampMS,
matchSideEffectRevision: Number(
updatedAuction.matchSideEffectRevision ?? 0,
@@ -923,19 +1112,170 @@ export class AuctionDurabilityScheduler
updatedAuction.matchSideEffectStatus ?? params.status,
});
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
auctionLifecycleId: delta.auctionLifecycleId,
matchId: String(params.payload.matchId),
matchSideEffectRevision: delta.matchSideEffectRevision,
matchSideEffectStatus: delta.matchSideEffectStatus,
});
this.auctionEventBusService?.broadcastMatchSideEffect(
String(params.payload.matchId),
delta,
);
const mirrored =
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
auctionLifecycleId: delta.auctionLifecycleId,
auctionStateTransitionId: params.payload.auctionStateTransitionId,
matchId: String(params.payload.matchId),
matchSideEffectRevision: delta.matchSideEffectRevision,
matchSideEffectStatus: delta.matchSideEffectStatus,
});
if (mirrored !== false) {
this.auctionEventBusService?.broadcastMatchSideEffect(
String(params.payload.matchId),
delta,
);
}
return delta;
}
public async reconcileDeadLetterStatuses(limit = 20): Promise<void> {
const client = this.auctionRedisService.raw();
const now = await this.auctionRedisService.getCurrentRedisTimestampMS();
// Also reconcile entries dead-lettered before status notifications existed.
const scanned = await client.sendCommand([
'ZSCAN',
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
this.deadLetterScanCursor,
'COUNT',
String(limit),
]);
if (Array.isArray(scanned) && Array.isArray(scanned[1])) {
this.deadLetterScanCursor = String(scanned[0]);
for (let index = 0; index < scanned[1].length; index += 2) {
const key = String(scanned[1][index]);
if (
!(await client.sendCommand([
'HGET',
key,
'deadLetterStatusReconciledAtTimestampMS',
]))
) {
await client.sendCommand([
'ZADD',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
'NX',
String(now),
key,
]);
}
}
}
const reply = await client.sendCommand([
'ZRANGEBYSCORE',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
'-inf',
String(now),
'LIMIT',
'0',
String(limit),
]);
if (!Array.isArray(reply)) return;
for (const key of reply.map(String)) {
try {
const payload = await this.readDurabilityPayload(key);
if (
payload &&
parseMatchSideEffectMutation(payload.matchSideEffectMutation)
) {
const state =
await this.auctionRedisStateReaderService?.readAuctionStateView(
String(payload.matchId),
);
// A terminal notification for a previous generation must not change
// a new completion/undo, even when Mongo is behind Redis.
const superseded =
state &&
(state.auctionLifecycleId !== payload.auctionLifecycleId ||
state.matchSideEffectStatus === 'completed' ||
(state.matchSideEffectTransitionId &&
state.matchSideEffectTransitionId !==
payload.auctionStateTransitionId));
if (!superseded) {
const AuctionModel =
await this.connectionService.getModel('auction');
if (
state?.matchSideEffectTransitionId ===
payload.auctionStateTransitionId
) {
const revision = Number(
payload.afterAuctionMongoProjection.matchSideEffectRevision ??
0,
);
await AuctionModel.updateOne(
{
_id: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
$or: [
{ matchSideEffectRevision: { $lt: revision } },
{ matchSideEffectRevision: { $exists: false } },
],
} as never,
{
$set: {
matchSideEffectTransitionId:
payload.auctionStateTransitionId,
matchSideEffectRevision: revision,
matchSideEffectStatus: 'pending',
},
},
);
// Heal a same-generation revision mismatch before sending the
// terminal status; otherwise Redis could ignore it as older.
await AuctionModel.updateOne(
{
_id: payload.auctionId,
auctionLifecycleId: payload.auctionLifecycleId,
matchSideEffectTransitionId: payload.auctionStateTransitionId,
matchSideEffectStatus: { $ne: 'completed' },
} as never,
{
$max: {
matchSideEffectRevision: Number(
state.matchSideEffectRevision ?? revision,
),
},
},
);
}
const error = await client.sendCommand(['HGET', key, 'lastError']);
await this.persistAndBroadcastMatchSideEffectStatus({
AuctionModel: AuctionModel as never,
payload,
currentServerTimestampMS: now,
status: 'failed',
onlyUnfinished: true,
errorMessage:
typeof error === 'string'
? error
: 'Auction side effect exhausted its retry limit.',
});
}
}
await client.sendCommand([
'EVAL',
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 1 then redis.call('HSET', KEYS[1], 'deadLetterStatusReconciledAtTimestampMS', ARGV[1]) end redis.call('ZREM', KEYS[2], KEYS[1]); return 1",
'2',
key,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(now),
]);
} catch (error) {
// Mongo or Redis mirroring may be unavailable at dead-letter time. Keep
// the notification durable until the failure can be shown to viewers.
await client.sendCommand([
'ZADD',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(now + 5000),
key,
]);
this.logger.warn(
`Auction terminal status will retry for ${key}: ${String(error)}`,
);
}
}
}
private async safeTick(): Promise<void> {
if (this.tickInProgress) return;
this.tickInProgress = true;
@@ -948,7 +1288,22 @@ export class AuctionDurabilityScheduler
AUCTION_DURABILITY_LEADER_LOCK_TTL_MS,
);
if (!leaderLockAcquired) return;
await this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT);
// A failed maintenance task must not starve the other durable queues.
for (const work of [
() => this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT),
() => this.reconcileDeadLetterStatuses(AUCTION_DURABILITY_BATCH_LIMIT),
() => this.auctionBidArchiveService?.persistPendingBids(),
() =>
this.auctionService?.retryPendingCancellations(
AUCTION_DURABILITY_BATCH_LIMIT,
),
]) {
try {
await work();
} catch (error) {
this.logger.warn(`Auction maintenance task failed: ${String(error)}`);
}
}
} catch (error) {
this.logger.warn(
`Auction durability tick failed: ${
@@ -957,10 +1312,13 @@ export class AuctionDurabilityScheduler
);
} finally {
if (leaderLockAcquired) {
await this.auctionRedisService.releaseTokenLock(
AUCTION_DURABILITY_LEADER_LOCK_KEY,
leaderLockToken,
);
await this.auctionRedisService
.releaseTokenLock(AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken)
.catch((error: unknown) => {
this.logger.warn(
`Auction durability lock release failed: ${String(error)}`,
);
});
}
this.tickInProgress = false;
}
@@ -987,6 +1345,51 @@ export class AuctionDurabilityScheduler
]);
}
private async deferDurabilityPayload(
payloadKey: string,
timestampMS: number,
error: unknown,
): Promise<'gone' | 'requeued' | 'dead-lettered'> {
// Keep coordination waits separate from failed work, but bound them across
// restarts too. Updating the payload and queue atomically avoids orphan jobs.
const result = await this.auctionRedisService
.raw()
.sendCommand([
'EVAL',
[
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end",
"local count = redis.call('HINCRBY', KEYS[1], 'pendingAttempts', 1)",
"redis.call('HSETNX', KEYS[1], 'pendingSinceTimestampMS', ARGV[1])",
"local since = tonumber(redis.call('HGET', KEYS[1], 'pendingSinceTimestampMS'))",
"redis.call('HSET', KEYS[1], 'lastPendingReason', ARGV[5], 'lastPendingAtTimestampMS', ARGV[1])",
'if count >= tonumber(ARGV[3]) or tonumber(ARGV[1]) - since >= tonumber(ARGV[4]) then',
" redis.call('ZREM', KEYS[2], KEYS[1])",
" redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1], 'lastError', ARGV[5])",
" redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1])",
" redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1])",
' return 2',
'end',
"redis.call('ZADD', KEYS[2], tonumber(ARGV[1]) + tonumber(ARGV[2]), KEYS[1])",
'return 1',
].join('\n'),
'4',
payloadKey,
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(timestampMS),
String(AUCTION_DURABILITY_RETRY_DELAY_MS),
String(AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS),
String(AUCTION_DURABILITY_MAX_PENDING_DURATION_MS),
formatErrorMessageForRedis(error),
]);
return Number(result) === 2
? 'dead-lettered'
: Number(result) === 1
? 'requeued'
: 'gone';
}
private async recordDurabilityPayloadFailure(
payloadKey: string,
currentRedisTimestampMS: number,
@@ -1012,24 +1415,17 @@ export class AuctionDurabilityScheduler
Number.isFinite(attempts) &&
attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS
) {
await this.auctionRedisService
.raw()
.sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]);
await this.auctionRedisService
.raw()
.sendCommand([
'HSET',
'EVAL',
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1]); redis.call('ZREM', KEYS[2], KEYS[1]); redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1]); redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1]); return 1",
'4',
payloadKey,
'deadLetteredAtRedisTimestampMS',
String(currentRedisTimestampMS),
]);
await this.auctionRedisService
.raw()
.sendCommand([
'ZADD',
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
String(currentRedisTimestampMS),
payloadKey,
]);
return 'dead-lettered';
}
@@ -1064,7 +1460,7 @@ export class AuctionDurabilityScheduler
'end',
'return 1',
].join('\n'),
'11',
'12',
keys.liveStateKey,
keys.deadlineQueueKey,
keys.structureStateKey,
@@ -1076,6 +1472,7 @@ export class AuctionDurabilityScheduler
keys.actionRequestResultOrderKey,
keys.rehydrateLockKey,
keys.stateTransitionLockKey,
keys.rejectedBidRequestResultOrderKey,
matchId,
payload.auctionLifecycleId,
String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS),
@@ -1159,6 +1556,7 @@ async function readAuctionProjectionIdentityForMutation(
? (query as { select: (...args: never[]) => unknown }).select({
auctionLifecycleId: 1,
revision: 1,
matchSideEffectTransitionId: 1,
} as never)
: query;
const scoped =
@@ -1178,6 +1576,10 @@ async function readAuctionProjectionIdentityForMutation(
if (!projection || typeof projection !== 'object') return {};
const record = projection as Record<string, unknown>;
return {
matchSideEffectTransitionId:
typeof record.matchSideEffectTransitionId === 'string'
? record.matchSideEffectTransitionId
: undefined,
auctionLifecycleId:
typeof record.auctionLifecycleId === 'string'
? record.auctionLifecycleId
@@ -1200,6 +1602,7 @@ export function buildAuctionProjectionGuardedUpdate(
},
update: {
$set: payload.afterAuctionMongoProjection,
$unset: buildAuctionProjectionUnset(payload.afterAuctionMongoProjection),
},
};
}
@@ -24,4 +24,10 @@ export class AuctionEventBusService {
?.to(matchId)
.emit('state', serializeRealtimeAuction(state));
}
public broadcastReset(matchId: string, auctionLifecycleId: string): void {
this.auctionNamespace
?.to(matchId)
.emit('reset', { matchId, auctionLifecycleId });
}
}
@@ -0,0 +1,32 @@
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
afterEach(() => jest.useRealTimers());
it('renews ownership and handles renewal/release failures without unhandled rejection', async () => {
jest.useFakeTimers();
const sendCommand = jest
.fn()
.mockRejectedValue(new Error('redis disconnected'));
const releaseTokenLock = jest
.fn()
.mockRejectedValue(new Error('release failed'));
const lock = new AuctionLifecycleLock({
acquireTokenLock: jest.fn().mockResolvedValue(true),
releaseTokenLock,
raw: () => ({ sendCommand }),
} as never);
const result = lock.run('match', async () => {
await jest.advanceTimersByTimeAsync(10000);
});
await expect(result).rejects.toThrow('redis disconnected');
expect(releaseTokenLock).toHaveBeenCalledTimes(1);
expect(jest.getTimerCount()).toBe(0);
});
it('preserves an operation result if only lock release fails', async () => {
const lock = new AuctionLifecycleLock({
acquireTokenLock: jest.fn().mockResolvedValue(true),
releaseTokenLock: jest.fn().mockRejectedValue(new Error('release failed')),
} as never);
await expect(lock.run('match', async () => 42)).resolves.toBe(42);
});
@@ -0,0 +1,72 @@
import { ConflictException, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { AuctionRedisService } from './redis/auction-redis.service';
export class AuctionLifecycleBusyError extends ConflictException {
constructor() {
super({
code: 'AUCTION_LIFECYCLE_BUSY',
message: '다른 경매 작업을 처리 중입니다. 잠시 후 다시 시도해 주세요.',
});
}
}
/** Serializes startup, cancellation, undo and Match side effects across nodes. */
export class AuctionLifecycleLock {
private readonly logger = new Logger(AuctionLifecycleLock.name);
constructor(private readonly redis: AuctionRedisService) {}
async run<T>(matchId: string, operation: () => Promise<T>): Promise<T> {
// Kept outside runtime cleanup: cancelling an auction must not release a
// lock still held by its cancellation/recovery operation.
const key = `auction:${matchId}:lifecycleMutationLock`;
const token = randomUUID();
if (!(await this.redis.acquireTokenLock(key, token, 30_000))) {
throw new AuctionLifecycleBusyError();
}
let renewal: Promise<void> | undefined;
let ownershipError: Error | undefined;
const timer = setInterval(() => {
if (renewal) return;
renewal = this.redis
.raw()
.sendCommand<string | null>([
'SET',
key,
token,
'IFEQ',
token,
'PX',
'30000',
])
.then((result) => {
if (result !== 'OK')
throw new Error('Auction lifecycle lock ownership was lost.');
})
.catch((error: unknown) => {
ownershipError =
error instanceof Error ? error : new Error(String(error));
})
.finally(() => {
renewal = undefined;
});
}, 10_000);
timer.unref?.();
try {
const result = await operation();
clearInterval(timer);
await renewal;
if (ownershipError) throw ownershipError;
return result;
} finally {
clearInterval(timer);
await renewal;
await this.redis.releaseTokenLock(key, token).catch((error: unknown) => {
this.logger.warn(
`Auction lifecycle lock release failed: ${String(error)}`,
);
});
}
}
}
@@ -0,0 +1,20 @@
/** Missing optional fields in a complete projection mean deletion, not retention. */
export const AUCTION_OPTIONAL_PROJECTION_FIELDS = [
'completedAt',
'auctionCompletedAtTimestampMS',
'currentBidderTeamIndex',
'currentHighestBidderTeamIndex',
'currentDeadline',
'currentBidRoundDeadlineTimestampMS',
'currentBidRoundId',
] as const;
export function buildAuctionProjectionUnset(
projection: Record<string, unknown>,
) {
return Object.fromEntries(
AUCTION_OPTIONAL_PROJECTION_FIELDS.filter(
(field) => projection[field] === undefined,
).map((field) => [field, '']),
);
}
@@ -5,6 +5,11 @@ import {
} from './auction-rejection';
describe('resolveAuctionRejectionReason', () => {
it('distinguishes an outdated client from a stale round and does not suggest a state-only resync', () => {
const reason = resolveAuctionRejectionReason('BID_CLIENT_UPDATE_REQUIRED');
expect(reason).toBe('BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED');
expect(isAuctionResyncRecommended(reason)).toBe(false);
});
it('reports roster validation failures without recommending a retry', () => {
const reason = resolveAuctionRejectionReason(
'AUCTION_ROSTER_INVALID',
@@ -46,6 +46,8 @@ export const AUCTION_REJECTION_REASON = {
'AUCTION_TRANSITION_REJECTED_BECAUSE_REQUEST_COORDINATION_RETRYABLE',
// === 입찰(bid) ===
BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED:
'BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED',
/** 입찰 라운드가 이미 다음 선수/다음 라운드로 넘어감(내부 `STALE_BID_ROUND`). */
BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED:
'BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED',
@@ -86,6 +88,8 @@ export type AuctionRejectionReason =
/** 내부 짧은 코드 → 설명적 reason. */
const INTERNAL_CODE_TO_REASON: Record<string, AuctionRejectionReason> = {
BID_CLIENT_UPDATE_REQUIRED:
AUCTION_REJECTION_REASON.BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED,
AUCTION_ROSTER_INVALID:
AUCTION_REJECTION_REASON.AUCTION_TRANSITION_REJECTED_BECAUSE_ROSTER_INVALID,
CONFLICT:
@@ -96,6 +96,7 @@ describe('auction side effect helpers', () => {
'life-1',
'2',
'completed',
'',
]);
});
});
@@ -25,6 +25,7 @@ export interface BuildSideEffectLeaseAcquireUpdateParams {
export interface MatchSideEffectDelta {
auctionLifecycleId: string;
auctionStateTransitionId?: string;
currentServerTimestampMS: number;
matchSideEffectRevision: number;
matchSideEffectStatus: AuctionMatchSideEffectStatus;
@@ -36,6 +37,7 @@ export class AuctionSideEffectService {
public async mirrorMatchSideEffectStatus(params: {
auctionLifecycleId: string;
auctionStateTransitionId?: string;
matchId: string;
matchSideEffectRevision: number;
matchSideEffectStatus: AuctionMatchSideEffectStatus;
@@ -47,6 +49,8 @@ export class AuctionSideEffectService {
'EVAL',
[
"if redis.call('HGET', KEYS[1], 'auctionLifecycleId') ~= ARGV[1] then return 0 end",
"local generation = redis.call('HGET', KEYS[1], 'matchSideEffectTransitionId')",
"if ARGV[4] ~= '' and generation and generation ~= ARGV[4] then return 0 end",
"local currentRevision = tonumber(redis.call('HGET', KEYS[1], 'matchSideEffectRevision') or '-1')",
'local nextRevision = tonumber(ARGV[2])',
'if currentRevision >= nextRevision then return 0 end',
@@ -58,6 +62,7 @@ export class AuctionSideEffectService {
params.auctionLifecycleId,
String(params.matchSideEffectRevision),
params.matchSideEffectStatus,
params.auctionStateTransitionId ?? '',
]);
return Number(result) === 1;
}
@@ -104,6 +109,9 @@ export function buildMatchSideEffectDelta(
): MatchSideEffectDelta {
return {
auctionLifecycleId: params.auctionLifecycleId,
...(params.auctionStateTransitionId
? { auctionStateTransitionId: params.auctionStateTransitionId }
: {}),
currentServerTimestampMS: params.currentServerTimestampMS,
matchSideEffectRevision: params.matchSideEffectRevision,
matchSideEffectStatus: params.matchSideEffectStatus,
@@ -9,6 +9,7 @@ import {
Param,
Post,
UseGuards,
Query,
} from '@nestjs/common';
import mongoose from 'mongoose';
import { AllowPublicAccess } from '../auth/allow-public-access.decorator';
@@ -19,6 +20,7 @@ import { ParseObjectIdPipe } from '../common/parse-object-id.pipe';
import type { UserDocument } from '../global/database/schema';
import { AuctionService } from './auction.service';
import { AuctionGateway } from './auction.gateway';
import { AuctionBidArchiveService } from './auction-bid-archive.service';
import {
NextAuctionDto,
OpenBiddingDto,
@@ -35,6 +37,7 @@ export class AuctionController {
constructor(
private readonly auctionService: AuctionService,
private readonly auctionGateway: AuctionGateway,
private readonly auctionBidArchiveService: AuctionBidArchiveService,
) {}
@Post()
@@ -78,6 +81,20 @@ export class AuctionController {
}
// 공개 — 팀장 2명만 쓸 엔드포인트(트러스트 기반, 10명 동시 관전).
@Get('bids/history')
@AllowPublicAccess()
public async getBidHistory(
@Param('matchId', ParseObjectIdPipe) matchId: mongoose.Types.ObjectId,
@Query('afterRevision') afterRevision?: string,
@Query('limit') limit?: string,
) {
return this.auctionBidArchiveService.list(
matchId,
afterRevision === undefined ? -1 : Number(afterRevision),
limit === undefined ? 100 : Number(limit),
);
}
@Post('open-bidding')
@AllowPublicAccess()
@HttpCode(HttpStatus.OK)
@@ -100,6 +117,7 @@ export class AuctionController {
teamIndex: body.bidderTeamIndex,
amount: body.bidPointAmount,
bidRequestId: body.bidRequestId,
clientObservedBidRoundId: body.clientObservedBidRoundId,
clientObservedAuctionLifecycleId: body.clientObservedAuctionLifecycleId,
clientObservedCurrentBidRoundPlayerUserId:
body.clientObservedCurrentBidRoundPlayerUserId,
@@ -213,7 +231,10 @@ export class AuctionController {
clientObservedPhase: body.clientObservedPhase,
clientObservedRevision: body.clientObservedRevision,
});
this.auctionGateway.broadcastReset(matchId.toString());
this.auctionGateway.broadcastReset(
matchId.toString(),
body.clientObservedAuctionLifecycleId,
);
}
private assertSameCommunity(
@@ -1,4 +1,5 @@
import mongoose from 'mongoose';
import { ServiceUnavailableException } from '@nestjs/common';
jest.mock('../auth/jwt.service', () => ({
JwtService: class JwtService {},
@@ -7,6 +8,76 @@ jest.mock('../auth/jwt.service', () => ({
import { AuctionGateway } from './auction.gateway';
describe('AuctionGateway', () => {
it('signals pending initial state so the client can retry privately after recovery', async () => {
const matchId = new mongoose.Types.ObjectId().toString();
const gateway = new AuctionGateway(
{
getByMatchId: async () => ({}),
getStateByMatchId: async () => {
throw new ServiceUnavailableException({
code: 'AUCTION_STATE_RECOVERING',
message: 'recovering',
});
},
} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
);
const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() };
await gateway.handleJoin(socket as never, { matchId });
expect(socket.emit).toHaveBeenCalledTimes(1);
expect(socket.emit).toHaveBeenCalledWith('state-pending', {
retryAfterMS: 500,
});
});
it('sends join and resync snapshots only to the requesting socket and coalesces in-flight resyncs', async () => {
const matchId = new mongoose.Types.ObjectId().toString();
const getStateByMatchId = jest.fn().mockResolvedValue({ revision: 3 });
const gateway = new AuctionGateway(
{
getByMatchId: async () => ({ chatMessages: [] }),
getStateByMatchId,
} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
);
const broadcast = jest.fn();
(gateway as unknown as { server: unknown }).server = {
to: () => ({ emit: broadcast }),
};
const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() };
await gateway.handleJoin(socket as never, { matchId });
expect(socket.emit).toHaveBeenCalledWith('state', { revision: 3 });
expect(broadcast).not.toHaveBeenCalled();
let finish!: (value: unknown) => void;
getStateByMatchId.mockImplementationOnce(
() =>
new Promise((resolve) => {
finish = resolve;
}),
);
const ack = jest.fn(),
busyAck = jest.fn();
const pending = gateway.handleResync(socket as never, {}, ack);
await gateway.handleResync(socket as never, {}, busyAck);
expect(busyAck).toHaveBeenCalledWith(
expect.objectContaining({ code: 'RESYNC_BUSY' }),
);
finish({ revision: 4 });
await pending;
expect(ack).toHaveBeenCalledWith({ ok: true, state: { revision: 4 } });
expect(getStateByMatchId).toHaveBeenCalledTimes(2);
expect(broadcast).not.toHaveBeenCalled();
});
function createGateway(params?: {
getCurrentRedisTimestampMS?: jest.Mock;
placeBidDelta?: jest.Mock;
@@ -44,6 +44,7 @@ interface JoinAuctionPayload {
}
interface BidPayload {
clientObservedBidRoundId?: string;
bidderTeamIndex: number;
bidPointAmount: number;
bidRequestId?: string;
@@ -79,6 +80,8 @@ interface BidRateLimitState {
}
interface AuctionSocketData {
auctionResyncAtTimestampMS?: number;
auctionResyncInProgress?: boolean;
auctionBidRateLimit?: BidRateLimitState;
identity?: IdentityDocument;
matchId?: string;
@@ -162,9 +165,55 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
new mongoose.Types.ObjectId(payload.matchId),
);
this.emitChatHistory(socket, auction);
this.broadcastState(payload.matchId, auctionState);
socket.emit('state', serializeRealtimeAuction(auctionState));
} catch (error) {
this.emitKnownError(socket, error);
const rejection = this.resolveErrorPayload(error);
if (rejection.code === 'AUCTION_STATE_RECOVERING') {
socket.emit('state-pending', { retryAfterMS: 500 });
} else {
this.emitKnownError(socket, error);
}
}
}
@SubscribeMessage('resync')
public async handleResync(
@ConnectedSocket() socket: AuctionSocket,
@MessageBody() _payload: unknown,
@Ack() ack?: (response: Record<string, unknown>) => void,
) {
const matchId = this.requireJoinedMatchId(socket);
if (!matchId) {
ack?.({ ok: false, code: 'JOIN_REQUIRED' });
return;
}
const now = Date.now();
const retryAfterMS =
500 - (now - (socket.data.auctionResyncAtTimestampMS ?? 0));
if (socket.data.auctionResyncInProgress || retryAfterMS > 0) {
ack?.({
ok: false,
code: 'RESYNC_BUSY',
retryAfterMS: Math.max(500, retryAfterMS),
});
return;
}
socket.data.auctionResyncInProgress = true;
socket.data.auctionResyncAtTimestampMS = now;
try {
const state = await this.auctionService.getStateByMatchId(
new mongoose.Types.ObjectId(matchId),
);
// Only the requesting socket receives the full snapshot.
ack?.({ ok: true, state: serializeRealtimeAuction(state) });
} catch (error) {
ack?.({
...this.resolveErrorPayload(error),
ok: false,
retryAfterMS: 500,
});
} finally {
socket.data.auctionResyncInProgress = false;
}
}
@@ -196,6 +245,7 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
teamIndex: payload.bidderTeamIndex,
amount: payload.bidPointAmount,
bidRequestId,
clientObservedBidRoundId: payload.clientObservedBidRoundId,
clientObservedAuctionLifecycleId:
payload.clientObservedAuctionLifecycleId,
clientObservedCurrentBidRoundPlayerUserId:
@@ -609,8 +659,11 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
this.server.to(matchId).emit('bid', delta);
}
public broadcastReset(matchId: string) {
this.server.to(matchId).emit('reset', { matchId });
public broadcastReset(matchId: string, auctionLifecycleId?: string) {
this.server.to(matchId).emit('reset', {
matchId,
...(auctionLifecycleId ? { auctionLifecycleId } : {}),
});
}
public broadcastMatchSideEffect(
@@ -10,6 +10,7 @@ import { AuctionEventBusService } from './auction-event-bus.service';
import { AuctionRedisModule } from './redis/auction-redis.module';
import { AuctionSideEffectService } from './auction-side-effect.service';
import { AuctionService } from './auction.service';
import { AuctionBidArchiveService } from './auction-bid-archive.service';
@Module({
imports: [
@@ -21,6 +22,7 @@ import { AuctionService } from './auction.service';
controllers: [AuctionController],
providers: [
AuctionService,
AuctionBidArchiveService,
AuctionGateway,
AuctionEventBusService,
AuctionDeadlineScheduler,
@@ -45,6 +45,10 @@ describe('AuctionService', () => {
const allModels = {
auctionHistory: auctionHistoryModel,
...models,
auction: {
updateOne: jest.fn().mockResolvedValue({ matchedCount: 1 }),
...((models.auction as object) ?? {}),
},
};
const connectionService = {
getModel: jest.fn((name: string) => allModels[name]),
@@ -697,7 +701,7 @@ describe('AuctionService', () => {
expect(auctionRedisService.raw().sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('currentAuctionLifecycleId'),
'11',
'12',
`auction:${matchId.toString()}:liveAuctionState`,
'auction:bidRoundDeadlineQueue',
`auction:${matchId.toString()}:auctionStructureState`,
@@ -709,6 +713,7 @@ describe('AuctionService', () => {
`auction:${matchId.toString()}:auctionActionRequestResultOrder`,
`auction:${matchId.toString()}:rehydrateLock`,
`auction:${matchId.toString()}:auctionStateTransitionLock`,
`auction:${matchId.toString()}:rejectedBidRequestResultOrder`,
matchId.toString(),
'old-life-1',
]);
@@ -909,9 +914,9 @@ describe('AuctionService', () => {
shouldForceAuctionStateResync: true,
});
expect(auctionRedisService.acquireTokenLock).toHaveBeenCalledWith(
`auction:${matchId.toString()}:rehydrateLock`,
`auction:${matchId.toString()}:lifecycleMutationLock`,
expect.any(String),
5000,
30000,
);
expect(auctionRedisCommandService.hydrateState).toHaveBeenCalledWith(
matchId.toString(),
@@ -2621,8 +2626,14 @@ describe('AuctionService', () => {
auctionRedisCommandService.reserveActionRequest,
).not.toHaveBeenCalled();
expect(auctionRedisCommandService.applyTransition).not.toHaveBeenCalled();
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match);
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match);
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(
match,
session,
);
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(
match,
session,
);
expect(match).toMatchObject({
gameSchedule: [],
result: undefined,
@@ -2642,13 +2653,12 @@ describe('AuctionService', () => {
auctionId,
});
expect(auction.deleteOne).toHaveBeenCalledWith({ session });
expect(redisClient.sendCommand).toHaveBeenCalledWith(
expect.arrayContaining([
'auction:bidRoundDeadlineQueue',
matchId.toString(),
'life-1',
]),
);
expect(redisClient.sendCommand).not.toHaveBeenCalled();
expect(match).toMatchObject({
lastCancelledAuctionLifecycleId: 'life-1',
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: expect.any(Date),
});
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
});
@@ -2732,8 +2742,14 @@ describe('AuctionService', () => {
service.undoTo({ matchId, communityId, sequence: 3 }),
).resolves.toBe(restored);
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match);
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match);
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(
match,
session,
);
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(
match,
session,
);
expect(match.status).toBe('team-built');
expect(match.result).toBeUndefined();
expect(auctionReplaceOne).toHaveBeenCalledWith(
+402 -93
View File
@@ -5,8 +5,16 @@ import {
InternalServerErrorException,
Logger,
NotFoundException,
ServiceUnavailableException,
Optional,
} from '@nestjs/common';
import { randomUUID } from 'crypto';
import {
AuctionLifecycleLock,
AuctionLifecycleBusyError,
} from './auction-lifecycle-lock';
import { AUCTION_OPTIONAL_PROJECTION_FIELDS } from './auction-projection';
import { AuctionEventBusService } from './auction-event-bus.service';
import { performance } from 'perf_hooks';
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
@@ -58,8 +66,6 @@ const AUCTION_BID_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000;
const AUCTION_MAX_STORED_ACTION_REQUEST_RESULT_COUNT = 500;
const AUCTION_ACTION_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000;
const AUCTION_ACTION_REQUEST_LEASE_DURATION_MS = 15 * 1000;
const AUCTION_REHYDRATE_LOCK_TTL_MS = 5 * 1000;
const AUCTION_REHYDRATE_LOCK_WAIT_MS = 50;
const AUCTION_HISTORY_LIST_LIMIT = 100;
const AUCTION_HISTORY_LIST_CACHE_TTL_MS = 2 * 1000;
const AUCTION_HISTORY_LIST_CACHE_MAX_ENTRIES = 256;
@@ -82,6 +88,7 @@ interface StartAuctionParams {
}
interface PlaceBidParams {
clientObservedBidRoundId?: string;
matchId: mongoose.Types.ObjectId;
teamIndex: number;
amount: number;
@@ -160,6 +167,7 @@ interface AuctionHistoryListCacheEntry {
export type AuctionStateResponse = AuctionDocument | Record<string, unknown>;
export interface AuctionBidDeltaResponse {
currentBidRoundId?: string;
auctionBidHistoryEntry: Record<string, unknown>;
auctionLifecycleId?: string;
bidServerTiming?: AuctionBidServerTiming;
@@ -212,9 +220,20 @@ export class AuctionService {
private readonly auctionRedisService: AuctionRedisService,
private readonly auctionRedisCommandService: AuctionRedisCommandService,
private readonly auctionRedisStateReaderService: AuctionRedisStateReaderService,
@Optional()
private readonly auctionEventBusService?: AuctionEventBusService,
) {}
public async start(params: StartAuctionParams): Promise<AuctionDocument> {
return new AuctionLifecycleLock(this.auctionRedisService).run(
params.matchId.toString(),
() => this.startLocked(params),
);
}
private async startLocked(
params: StartAuctionParams,
): Promise<AuctionDocument> {
const MatchModel = await this.connectionService.getModel('match');
const UserModel = await this.connectionService.getModel('user');
const AuctionModel = await this.connectionService.getModel('auction');
@@ -231,7 +250,7 @@ export class AuctionService {
const existing = await AuctionModel.findOne({ matchId: params.matchId });
if (existing) {
if (existing.startAuctionRequestId === params.startAuctionRequestId) {
return existing;
return this.finishAuctionStartup(existing, match);
}
if (this.isRestartableResetRedisAuction(existing)) {
await this.clearRedisAuctionRuntimeForLifecycle(
@@ -249,6 +268,19 @@ export class AuctionService {
}
}
// No Mongo auction owns this match. Remove runtime left by an interrupted
// cancellation before creating a new lifecycle (under the lifecycle lock).
const orphanedRuntime =
await this.auctionRedisStateReaderService.readLiveStateIdentity(
params.matchId.toString(),
);
if (orphanedRuntime?.auctionLifecycleId) {
await this.clearRedisAuctionRuntimeForLifecycle(
params.matchId.toString(),
orphanedRuntime.auctionLifecycleId,
);
}
const activeParticipants = this.applyAuctionCut(match);
const desiredAuctionCount = activeParticipants.length;
if (desiredAuctionCount < 10) {
@@ -381,6 +413,7 @@ export class AuctionService {
auctionStartedByUserId: params.actorUserId,
matchSideEffectStatus: 'none',
matchSideEffectRevision: 0,
startupCompleted: false,
teams: legacyTeams,
order: prioritizedOrder,
passedQueue: [],
@@ -419,12 +452,12 @@ export class AuctionService {
matchId: params.matchId,
});
if (raceWinner?.startAuctionRequestId === params.startAuctionRequestId) {
return raceWinner;
return this.finishAuctionStartup(raceWinner, match);
}
throw new BadRequestException('Auction already exists for this match.');
}
await this.recordSnapshot(auction, 'start');
await this.auctionRedisCommandService.hydrateState(
const hydration = await this.auctionRedisCommandService.hydrateState(
params.matchId.toString(),
{
auctionHistorySequence: 1,
@@ -437,9 +470,18 @@ export class AuctionService {
structureStateJson,
},
);
if ('rejected' in hydration)
throw new ConflictException(
`Auction initialization failed: ${hydration.code}.`,
);
match.status = 'team-built';
await match.save();
auction.startupCompleted = true;
await AuctionModel.updateOne(
{ _id: auction._id },
{ $set: { startupCompleted: true } },
);
this.enqueueBackgroundOrderEnrichment({
auctionId: auction._id,
targets: enrichTargets,
@@ -450,6 +492,61 @@ export class AuctionService {
return auction;
}
private async finishAuctionStartup(
auction: AuctionDocument,
match: MatchDocument,
): Promise<AuctionDocument> {
// Documents predating the startup marker already completed the old path.
if (auction.startupCompleted !== false) return auction;
// Retry the original cut; recruiting may have changed since Mongo creation.
const roster = new Set([
...auction.teams.map((team) => team.captainUserId.toString()),
...auction.order.map((player) => player.userId.toString()),
]);
const participants = new Set(
match.participants.map((entry) => entry.userId.toString()),
);
if ([...roster].some((id) => !participants.has(id))) {
throw new ConflictException(
'Auction startup roster has changed. Cancel and restart the auction.',
);
}
const History = await this.connectionService.getModel('auctionHistory');
await History.updateOne(
{ auctionId: auction._id, sequence: 1 },
{
$setOnInsert: {
auctionId: auction._id,
matchId: auction.matchId,
sequence: 1,
action: 'start',
snapshotBefore: this.cloneJson(
typeof auction.toObject === 'function'
? auction.toObject()
: auction,
),
takenAt: auction.startedAt ?? new Date(),
},
},
{ upsert: true },
);
await this.hydrateRedisStateFromAuctionDocument(auction);
if (match.status === 'recruiting') {
match.participants.forEach((participant) => {
participant.excluded = !roster.has(participant.userId.toString());
});
match.status = 'team-built';
await match.save();
}
const AuctionModel = await this.connectionService.getModel('auction');
await AuctionModel.updateOne(
{ _id: auction._id },
{ $set: { startupCompleted: true } },
);
auction.startupCompleted = true;
return auction;
}
public async getByMatchId(
matchId: mongoose.Types.ObjectId,
): Promise<AuctionDocument> {
@@ -462,18 +559,46 @@ export class AuctionService {
public async getStateByMatchId(
matchId: mongoose.Types.ObjectId,
): Promise<AuctionStateResponse> {
const auction = await this.getByMatchId(matchId);
for (let attempt = 0; ; attempt++) {
try {
return await this.getStateByMatchIdOnce(matchId);
} catch (error) {
if (!(error instanceof AuctionLifecycleBusyError)) throw error;
if (attempt >= 4)
throw new ServiceUnavailableException({
code: 'AUCTION_STATE_RECOVERING',
retryAfterMS: 500,
message: '경매 상태를 복구 중입니다. 잠시 후 다시 시도해 주세요.',
});
await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));
}
}
}
private async getStateByMatchIdOnce(
matchId: mongoose.Types.ObjectId,
): Promise<AuctionStateResponse> {
let auction = await this.getByMatchId(matchId);
if (!this.isRedisV1Auction(auction)) return auction;
if (auction.startupCompleted === false) {
auction = await new AuctionLifecycleLock(this.auctionRedisService).run(
matchId.toString(),
async () => {
const current = await this.getByMatchId(matchId);
const MatchModel = await this.connectionService.getModel('match');
const match = await MatchModel.findById(matchId);
if (!match) throw new NotFoundException('Match not found.');
return this.finishAuctionStartup(current, match);
},
);
}
const redisState = await this.readRedisAuctionStateOrRehydrate(auction);
if (!redisState) return auction;
const mongoState =
typeof auction.toObject === 'function'
? auction.toObject({ depopulate: true })
: auction;
return {
...this.cloneJson(mongoState),
...this.buildAuctionSnapshotFromRedisState(auction, redisState),
...redisState,
chatMessages: auction.chatMessages ?? [],
};
@@ -485,12 +610,15 @@ export class AuctionService {
const matchId = auction.matchId.toString();
const mongoRevision = auction.revision ?? 0;
const mongoAuctionLifecycleId = auction.auctionLifecycleId;
if (!mongoAuctionLifecycleId || auction.phase === 'complete') {
if (!mongoAuctionLifecycleId) {
return this.auctionRedisStateReaderService.readAuctionStateView(matchId);
}
const liveIdentity =
await this.auctionRedisStateReaderService.readLiveStateIdentity(matchId);
// Completed runtimes expire after the retention period. Mongo remains the
// archive; viewing it must not create an immortal Redis runtime again.
if (!liveIdentity && auction.phase === 'complete') return null;
const shouldHydrate = shouldHydrateRedisAuctionState({
live: liveIdentity,
mongo: {
@@ -503,29 +631,32 @@ export class AuctionService {
return this.auctionRedisStateReaderService.readAuctionStateView(matchId);
}
const keys = buildAuctionRedisKeys(matchId);
const lockToken = randomUUID();
const lockAcquired = await this.auctionRedisService.acquireTokenLock(
keys.rehydrateLockKey,
lockToken,
AUCTION_REHYDRATE_LOCK_TTL_MS,
await new AuctionLifecycleLock(this.auctionRedisService).run(
matchId,
async () => {
const currentAuction = await this.getByMatchId(auction.matchId);
if (currentAuction.auctionLifecycleId !== mongoAuctionLifecycleId) {
throw new ConflictException(
'Auction lifecycle has changed. Refresh the auction.',
);
}
const currentLive =
await this.auctionRedisStateReaderService.readLiveStateIdentity(
matchId,
);
if (
currentLive?.auctionLifecycleId &&
currentLive.auctionLifecycleId !== mongoAuctionLifecycleId
) {
await this.clearRedisAuctionRuntimeForLifecycle(
matchId,
currentLive.auctionLifecycleId,
);
}
await this.hydrateRedisStateFromAuctionDocument(currentAuction);
},
);
if (lockAcquired) {
try {
await this.hydrateRedisStateFromAuctionDocument(auction);
} finally {
await this.auctionRedisService.releaseTokenLock(
keys.rehydrateLockKey,
lockToken,
);
}
} else {
await new Promise((resolve) =>
setTimeout(resolve, AUCTION_REHYDRATE_LOCK_WAIT_MS),
);
}
return this.auctionRedisStateReaderService.readAuctionStateView(matchId, {
shouldForceAuctionStateResync: true,
});
@@ -814,6 +945,7 @@ export class AuctionService {
bidderTeamIndex: params.teamIndex,
bidderUserId: bidContext.bidderTeamCaptainUserId,
bidRequestId: params.bidRequestId ?? randomUUID(),
clientObservedBidRoundId: params.clientObservedBidRoundId,
clientObservedAuctionLifecycleId: auctionLifecycleId,
clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId,
matchId,
@@ -890,6 +1022,7 @@ export class AuctionService {
bidderTeamIndex: params.teamIndex,
bidderUserId: this.stringFromState(team.captainUserId),
bidRequestId: params.bidRequestId ?? randomUUID(),
clientObservedBidRoundId: params.clientObservedBidRoundId,
clientObservedAuctionLifecycleId: auctionLifecycleId,
clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId,
matchId,
@@ -923,6 +1056,10 @@ export class AuctionService {
): AuctionBidDeltaResponse {
return {
auctionBidHistoryEntry: result.auctionBidHistoryEntry,
currentBidRoundId:
typeof result.auctionBidHistoryEntry.bidRoundId === 'string'
? result.auctionBidHistoryEntry.bidRoundId
: undefined,
auctionLifecycleId: this.stringFromState(state.auctionLifecycleId),
bidRequestId: result.bidRequestId,
currentBidRoundDeadlineTimestampMS:
@@ -1408,6 +1545,34 @@ export class AuctionService {
matchId: mongoose.Types.ObjectId;
communityId: mongoose.Types.ObjectId;
}): Promise<void> {
let committed = false;
try {
await new AuctionLifecycleLock(this.auctionRedisService).run(
params.matchId.toString(),
() =>
this.resetLocked(params, () => {
committed = true;
}),
);
} catch (error) {
if (!committed) throw error;
// A committed cancellation is successful even if lock renewal/teardown
// fails afterward. Cleanup is durably queued in the same Mongo transaction.
this.logger.warn(
`Auction cancellation committed; post-commit coordination failed: ${String(error)}`,
);
}
}
private async resetLocked(
params: {
matchId: mongoose.Types.ObjectId;
communityId: mongoose.Types.ObjectId;
clientObservedAuctionLifecycleId?: string;
clientObservedRevision?: number;
},
onCommitted: () => void,
): Promise<void> {
const AuctionModel = await this.connectionService.getModel('auction');
const AuctionHistoryModel =
await this.connectionService.getModel('auctionHistory');
@@ -1419,70 +1584,155 @@ export class AuctionService {
if (!session) {
throw new InternalServerErrorException('Failed to start DB session.');
}
let redisRuntimeToClear: {
auctionLifecycleId?: string;
matchId: string;
} | null = null;
try {
session.startTransaction();
const auction = await AuctionModel.findOne({
matchId: params.matchId,
}).session(session);
if (!auction) throw new NotFoundException('Auction not found.');
const match = await MatchModel.findById(params.matchId).session(session);
if (!match) throw new NotFoundException('Match not found.');
if (!match.communityId.equals(params.communityId)) {
throw new BadRequestException('Community mismatch.');
}
this.assertCompetitionLifecycleMutable(match, 'reset the auction');
if (this.isRedisV1Auction(auction)) {
redisRuntimeToClear = {
auctionLifecycleId: auction.auctionLifecycleId,
matchId: params.matchId.toString(),
};
await this.resetMatchAfterAuctionCancellation(match, session);
await AuctionSideEffectApplicationModel.deleteMany({
auctionId: auction._id,
}).session(session);
await AuctionHistoryModel.deleteMany({
auctionId: auction._id,
}).session(session);
await auction.deleteOne({ session });
if (!auction) {
// 이미 커밋된 취소의 재전송이면 성공으로 응답하고 남은 Redis 정리만 재시도한다.
const cancelledMatch = params.clientObservedAuctionLifecycleId
? await MatchModel.findById(params.matchId).session(session)
: null;
if (
!cancelledMatch ||
cancelledMatch.lastCancelledAuctionLifecycleId !==
params.clientObservedAuctionLifecycleId
) {
throw new NotFoundException('Auction not found.');
}
if (!cancelledMatch.communityId.equals(params.communityId)) {
throw new BadRequestException('Community mismatch.');
}
cancelledMatch.auctionCancellationCleanupPending = true;
cancelledMatch.auctionCancellationCleanupRetryAt = new Date();
await cancelledMatch.save({ session });
} else {
await auction.deleteOne({ session });
await AuctionHistoryModel.deleteMany({
auctionId: auction._id,
}).session(session);
if (match.status === 'team-built') {
await this.tournamentCodeService.releaseFromMatch(match);
match.status = 'recruiting';
match.teams = [];
match.tournamentCodeIds = [];
match.gameSchedule = [];
match.participants.forEach((participant) => {
participant.excluded = false;
});
match.teamBuiltAt = undefined;
match.result = undefined;
match.resultPollState = undefined;
await match.save({ session });
if (
this.isRedisV1Auction(auction) &&
(!params.clientObservedAuctionLifecycleId ||
params.clientObservedAuctionLifecycleId !==
auction.auctionLifecycleId)
) {
throw new ConflictException(
'Auction lifecycle has changed. Refresh before cancelling.',
);
}
const match = await MatchModel.findById(params.matchId).session(
session,
);
if (!match) throw new NotFoundException('Match not found.');
if (!match.communityId.equals(params.communityId)) {
throw new BadRequestException('Community mismatch.');
}
this.assertCompetitionLifecycleMutable(match, 'reset the auction');
if (this.isRedisV1Auction(auction)) {
// 같은 트랜잭션에서 취소 완료를 기록해, 재전송을 멱등 처리한다.
match.lastCancelledAuctionLifecycleId = auction.auctionLifecycleId;
match.auctionCancellationCleanupPending = true;
match.auctionCancellationCleanupRetryAt = new Date();
match.auctionCancellationCleanupAttempts = 0;
match.auctionCancellationCleanupError = undefined;
await this.resetMatchAfterAuctionCancellation(match, session);
await AuctionSideEffectApplicationModel.deleteMany({
auctionId: auction._id,
}).session(session);
await AuctionHistoryModel.deleteMany({
auctionId: auction._id,
}).session(session);
await auction.deleteOne({ session });
} else {
await auction.deleteOne({ session });
await AuctionHistoryModel.deleteMany({
auctionId: auction._id,
}).session(session);
if (match.status === 'team-built') {
await this.tournamentCodeService.releaseFromMatch(match, session);
match.status = 'recruiting';
match.teams = [];
match.tournamentCodeIds = [];
match.gameSchedule = [];
match.participants.forEach((participant) => {
participant.excluded = false;
});
match.teamBuiltAt = undefined;
match.result = undefined;
match.resultPollState = undefined;
await match.save({ session });
}
}
}
await session.commitTransaction();
onCommitted();
} catch (err) {
await session.abortTransaction();
throw err;
} finally {
await session.endSession();
}
}
if (redisRuntimeToClear) {
await this.clearRedisAuctionRuntimeForLifecycle(
redisRuntimeToClear.matchId,
redisRuntimeToClear.auctionLifecycleId,
);
public async retryPendingCancellations(batchLimit = 20): Promise<void> {
const MatchModel = await this.connectionService.getModel('match');
const pending = await MatchModel.find({
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: { $lte: new Date() },
})
.sort({ auctionCancellationCleanupRetryAt: 1 })
.limit(batchLimit)
.lean()
.exec();
for (const match of pending) {
const lifecycle = match.lastCancelledAuctionLifecycleId;
if (!lifecycle) continue;
const filter = {
_id: match._id,
lastCancelledAuctionLifecycleId: lifecycle,
};
try {
await new AuctionLifecycleLock(this.auctionRedisService).run(
match._id.toString(),
async () => {
// Compare-and-delete in Redis also protects a newly started lifecycle.
await this.clearRedisAuctionRuntimeForLifecycle(
match._id.toString(),
lifecycle,
);
const AuctionModel =
await this.connectionService.getModel('auction');
if (!(await AuctionModel.exists({ matchId: match._id }))) {
this.auctionEventBusService?.broadcastReset(
match._id.toString(),
lifecycle,
);
}
await MatchModel.updateOne(filter, {
$set: { auctionCancellationCleanupPending: false },
$unset: {
auctionCancellationCleanupRetryAt: '',
auctionCancellationCleanupError: '',
},
});
},
);
} catch (error) {
const attempts = (match.auctionCancellationCleanupAttempts ?? 0) + 1;
await MatchModel.updateOne(filter, {
$set: {
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: new Date(
Date.now() + Math.min(60000, 1000 * 2 ** Math.min(attempts, 6)),
),
auctionCancellationCleanupError: String(error).slice(0, 2000),
},
$inc: { auctionCancellationCleanupAttempts: 1 },
});
this.logger.warn(
`Auction cancellation cleanup will retry for ${match._id.toString()}: ${String(error)}`,
);
}
}
}
@@ -1497,6 +1747,19 @@ export class AuctionService {
communityId: mongoose.Types.ObjectId;
sequence: number;
}): Promise<AuctionStateResponse> {
return new AuctionLifecycleLock(this.auctionRedisService).run(
params.matchId.toString(),
() => this.undoToLocked(params),
);
}
private async undoToLocked(
params: AuctionStructureActionRequestParams & {
matchId: mongoose.Types.ObjectId;
communityId: mongoose.Types.ObjectId;
sequence: number;
},
): Promise<AuctionStateResponse> {
const AuctionModel = await this.connectionService.getModel('auction');
const AuctionHistoryModel =
await this.connectionService.getModel('auctionHistory');
@@ -1508,6 +1771,7 @@ export class AuctionService {
}
let redisUndoContext: {
state?: Record<string, unknown>;
actionRequestParams?: AuctionStructureActionRequestParams;
auction: AuctionDocument;
snapshot: Record<string, unknown>;
@@ -1543,8 +1807,12 @@ export class AuctionService {
throw new BadRequestException('Auction history snapshot is invalid.');
}
const undoingCompletedAuction =
Boolean(auction.completedAt) && !('completedAt' in snapshot);
const liveState = this.isRedisV1Auction(auction)
? await this.requireRedisAuctionState(params.matchId.toString())
: undefined;
const undoingCompletedAuction = liveState
? liveState.phase === 'complete' && snapshot.phase !== 'complete'
: Boolean(auction.completedAt) && !('completedAt' in snapshot);
if (undoingCompletedAuction) {
this.assertCompetitionLifecycleMutable(
match,
@@ -1553,6 +1821,7 @@ export class AuctionService {
}
if (this.isRedisV1Auction(auction)) {
redisUndoContext = {
state: liveState,
auction,
actionRequestParams: params,
snapshot: snapshot,
@@ -1562,8 +1831,8 @@ export class AuctionService {
await session.commitTransaction();
} else {
if (undoingCompletedAuction) {
await this.tournamentCodeService.releaseFromMatch(match);
await this.tournamentResultService.cancelPolling(match);
await this.tournamentCodeService.releaseFromMatch(match, session);
await this.tournamentResultService.cancelPolling(match, session);
match.status = 'team-built';
match.result = undefined;
await match.save({ session });
@@ -1758,6 +2027,14 @@ export class AuctionService {
currentAuctionHistorySequence: auctionHistorySequence,
lastAppliedAuctionStateTransitionId: auctionStateTransitionId,
revision: expectedRevision + 1,
currentBidRoundId:
params.deadline.mode === 'set-relative'
? auctionStateTransitionId
: params.state.currentBidRoundId,
currentBidRoundDeadlineTimestampMS:
params.deadline.mode === 'remove'
? 0
: params.state.currentBidRoundDeadlineTimestampMS,
};
const afterAuctionMongoProjection =
this.buildAuctionMongoProjectionFromState(params.auction, afterState);
@@ -1769,8 +2046,18 @@ export class AuctionService {
if (matchSideEffectMutation.type !== 'none') {
afterAuctionMongoProjection.matchSideEffectStatus = 'pending';
afterAuctionMongoProjection.matchSideEffectRevision =
(params.auction.matchSideEffectRevision ?? 0) + 1;
Math.max(
params.auction.matchSideEffectRevision ?? 0,
this.numberFromState(params.state.matchSideEffectRevision),
) + 1;
afterAuctionMongoProjection.matchSideEffectError = undefined;
afterAuctionMongoProjection.matchSideEffectTransitionId =
auctionStateTransitionId;
params.liveStatePatch.matchSideEffectTransitionId =
auctionStateTransitionId;
params.liveStatePatch.matchSideEffectStatus = 'pending';
params.liveStatePatch.matchSideEffectRevision =
afterAuctionMongoProjection.matchSideEffectRevision as number;
}
const durabilityPayload = {
@@ -1925,6 +2212,7 @@ export class AuctionService {
}
private async undoRedisAuction(params: {
state?: Record<string, unknown>;
actionRequestParams?: AuctionStructureActionRequestParams;
auction: AuctionDocument;
snapshot: Record<string, unknown>;
@@ -1932,7 +2220,8 @@ export class AuctionService {
undoingCompletedAuction: boolean;
}): Promise<AuctionStateResponse> {
const matchId = params.auction.matchId.toString();
const state = await this.requireRedisAuctionState(matchId);
const state =
params.state ?? (await this.requireRedisAuctionState(matchId));
const actionRequest = await this.reserveRedisStructureActionRequest({
action: 'undo',
matchId,
@@ -2118,8 +2407,11 @@ export class AuctionService {
typeof auction.toObject === 'function'
? (auction.toObject({ depopulate: true }) as Record<string, unknown>)
: (auction as unknown as Record<string, unknown>);
const snapshotBase = this.cloneJson(base);
for (const field of AUCTION_OPTIONAL_PROJECTION_FIELDS)
delete snapshotBase[field];
return {
...this.cloneJson(base),
...snapshotBase,
...this.buildAuctionMongoProjectionFromState(auction, state),
_id: auction._id,
matchId: auction.matchId,
@@ -2145,6 +2437,7 @@ export class AuctionService {
);
const phase = this.stringFromState(state.phase) || 'presenting';
const projection: Record<string, unknown> = {
currentBidRoundId: state.currentBidRoundId || undefined,
auctionBidHistory,
auctionCompletedAtTimestampMS:
phase === 'complete' &&
@@ -2285,6 +2578,8 @@ export class AuctionService {
private formatRedisBidRejectCode(code: string): string {
switch (code) {
case 'BID_CLIENT_UPDATE_REQUIRED':
return '경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.';
case 'PHASE':
return 'Auction is not in bidding phase.';
case 'STALE_BID_ROUND':
@@ -2701,8 +2996,8 @@ export class AuctionService {
match: MatchDocument,
session: mongoose.ClientSession,
): Promise<void> {
await this.tournamentCodeService.releaseFromMatch(match);
await this.tournamentResultService.cancelPolling(match);
await this.tournamentCodeService.releaseFromMatch(match, session);
await this.tournamentResultService.cancelPolling(match, session);
match.status = 'recruiting';
match.teams = [];
match.tournamentCodeIds = [];
@@ -2717,6 +3012,11 @@ export class AuctionService {
}
private assertLiveAuctionOperationSupported(auction: AuctionDocument): void {
if (auction.startupCompleted === false) {
throw new ConflictException(
'Auction startup is not complete. Refresh to resume initialization.',
);
}
if (auction.auctionEngineVersion && !this.isRedisV1Auction(auction)) {
throw new BadRequestException(
'Legacy auction live operations are not supported.',
@@ -2881,7 +3181,7 @@ export class AuctionService {
auction.topTierThreshold ??
'NONE',
};
await this.auctionRedisCommandService.hydrateState(
const hydration = await this.auctionRedisCommandService.hydrateState(
auction.matchId.toString(),
{
auctionHistorySequence: auction.currentAuctionHistorySequence ?? 0,
@@ -2889,6 +3189,10 @@ export class AuctionService {
auction.auctionLifecycleId ??
`auction:${auction._id.toString()}:rehydrated`,
liveStatePatch: {
currentBidRoundId: auction.phase === 'bidding' ? randomUUID() : '',
matchSideEffectTransitionId: auction.matchSideEffectTransitionId,
matchSideEffectRevision: auction.matchSideEffectRevision,
matchSideEffectStatus: auction.matchSideEffectStatus,
...buildAuctionRedisLiveStatePatchFromStructure(structureInput),
currentHighestBidPointAmount:
auction.currentHighestBidPointAmount ?? auction.currentBid ?? 0,
@@ -2905,6 +3209,10 @@ export class AuctionService {
structureStateJson: stringifyAuctionRedisStructureState(structureInput),
},
);
if ('rejected' in hydration)
throw new ConflictException(
`Auction recovery failed: ${hydration.code}.`,
);
}
private async clearRedisAuctionRuntimeForLifecycle(
@@ -2925,10 +3233,10 @@ export class AuctionService {
' end',
'end',
"redis.call('ZREM', KEYS[2], ARGV[1])",
"redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11])",
"redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11], KEYS[12])",
'return 1',
].join('\n'),
'11',
'12',
keys.liveStateKey,
keys.deadlineQueueKey,
keys.structureStateKey,
@@ -2940,6 +3248,7 @@ export class AuctionService {
keys.actionRequestResultOrderKey,
keys.rehydrateLockKey,
keys.stateTransitionLockKey,
keys.rejectedBidRequestResultOrderKey,
matchId,
expectedAuctionLifecycleId ?? '',
]);
@@ -72,6 +72,10 @@ export class StartAuctionDto {
}
export class PlaceBidDto {
@IsOptional()
@IsString()
clientObservedBidRoundId?: string;
@IsOptional()
@IsUUID()
bidRequestId?: string;
@@ -7,7 +7,13 @@ function makeService(
) {
const sendCommand = jest
.fn()
.mockImplementation(() => Promise.resolve(commandReplies.shift()));
.mockImplementation((command: string[]) =>
Promise.resolve(
command[0] === 'EVAL'
? [currentServerTimestampMS, ...commandReplies.splice(0, 3)]
: commandReplies.shift(),
),
);
const auctionRedisService = {
getCurrentRedisTimestampMS: jest
.fn()
@@ -125,19 +131,14 @@ describe('AuctionRedisStateReaderService', () => {
});
expect(view).not.toHaveProperty('team:2:remainingBidPointAmount');
expect(view).not.toHaveProperty('bidRoundExpiryDecision');
expect(sendCommand).toHaveBeenNthCalledWith(1, [
'HGETALL',
expect(sendCommand).toHaveBeenCalledTimes(1);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining("redis.call('TIME')"),
'3',
'auction:match-1:liveAuctionState',
]);
expect(sendCommand).toHaveBeenNthCalledWith(2, [
'GET',
'auction:match-1:auctionStructureState',
]);
expect(sendCommand).toHaveBeenNthCalledWith(3, [
'LRANGE',
'auction:match-1:auctionBidHistory',
'0',
'-1',
]);
});
@@ -111,18 +111,11 @@ export class AuctionRedisStateReaderService {
liveReply,
structureStateJson,
bidEntries,
] = await Promise.all([
this.auctionRedisService.getCurrentRedisTimestampMS(),
this.auctionRedisService
.raw()
.sendCommand(['HGETALL', keys.liveStateKey]),
this.auctionRedisService
.raw()
.sendCommand(['GET', keys.structureStateKey]),
this.auctionRedisService
.raw()
.sendCommand(['LRANGE', keys.bidHistoryKey, '0', '-1']),
]);
] = await this.auctionRedisService
.raw()
.sendCommand<
unknown[]
>(['EVAL', ["local now = redis.call('TIME')", 'local timestamp = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)', "return {tostring(timestamp), redis.call('HGETALL', KEYS[1]), redis.call('GET', KEYS[2]), redis.call('LRANGE', KEYS[3], 0, -1)}"].join('\n'), '3', keys.liveStateKey, keys.structureStateKey, keys.bidHistoryKey]);
const liveState = parseHashReply(liveReply, {
includeInternalLiveStateFields:
@@ -144,7 +137,7 @@ export class AuctionRedisStateReaderService {
auctionBidHistoryEntries: Array.isArray(bidEntries)
? bidEntries.map(String)
: [],
currentServerTimestampMS,
currentServerTimestampMS: Number(currentServerTimestampMS),
liveState: liveState as Record<string, unknown> & {
auctionLifecycleId: string;
phase: string;
@@ -46,6 +46,7 @@ describe('auction apply transition redis script contract', () => {
'auction:match-1:bidRequestResults',
'auction:match-1:bidRequestResultOrder',
'auction:bidRoundDeadlineQueue',
'auction:match-1:rejectedBidRequestResultOrder',
]);
expect(invocation.values.slice(0, 19)).toEqual([
'match-1',
@@ -79,6 +79,7 @@ export function buildAuctionApplyTransitionScriptInvocation(
keys.bidRequestResultsKey,
keys.bidRequestResultOrderKey,
keys.deadlineQueueKey,
keys.rejectedBidRequestResultOrderKey,
],
script: AUCTION_APPLY_TRANSITION_SCRIPT,
values: [
@@ -128,6 +129,7 @@ local auctionBidHistoryKey = KEYS[8]
local bidRequestResultsKey = KEYS[9]
local bidRequestResultOrderKey = KEYS[10]
local bidRoundDeadlineQueueKey = KEYS[11]
local rejectedBidRequestResultOrderKey = KEYS[12]
local matchId = ARGV[1]
local auctionLifecycleId = ARGV[2]
@@ -270,16 +272,27 @@ for i = 1, liveStatePatchPairCount do
end
redis.call('HSET', unpack(hsetArgs))
local committedBidRoundDeadlineTimestampMS = ''
if deadlineMode == 'set-relative' then
local currentBidRoundDeadlineTimestampMS = currentRedisTimestampMS + (deadlineDurationSeconds * 1000)
committedBidRoundDeadlineTimestampMS = tostring(currentBidRoundDeadlineTimestampMS)
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS', tostring(currentBidRoundDeadlineTimestampMS))
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundId', auctionStateTransitionId)
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
elseif deadlineMode == 'remove' then
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
end
-- A resumed auction must not disappear when its former completion TTL elapses.
if redis.call('HGET', liveAuctionStateKey, 'phase') ~= 'complete' then
for _, index in ipairs({1, 2, 3, 6, 7, 8, 9, 10, 12}) do
redis.call('PERSIST', KEYS[index])
end
end
if shouldResetBidRuntime == '1' then
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey)
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey, rejectedBidRequestResultOrderKey)
end
redis.call('SADD', appliedAuctionStateTransitionIdsKey, transitionDedupeMember)
@@ -288,6 +301,8 @@ redis.call(
durabilityPayloadKey,
'payloadJson',
durabilityPayloadJson,
'committedBidRoundDeadlineTimestampMS',
committedBidRoundDeadlineTimestampMS,
'preparedAtRedisTimestampMS',
tostring(currentRedisTimestampMS),
'committedAtRedisTimestampMS',
@@ -30,6 +30,9 @@ describe('auction bid redis script contract', () => {
'auction:match-1:bidRequestResults',
'auction:match-1:bidRequestResultOrder',
'auction:bidRoundDeadlineQueue',
'auction:match-1:rejectedBidRequestResultOrder',
'auction:bidArchiveQueue',
'auction:bidArchivePayloads',
]);
expect(invocation.values).toEqual([
'match-1',
@@ -45,6 +48,7 @@ describe('auction bid redis script contract', () => {
'100',
'60000',
'200',
'',
]);
});
@@ -1,4 +1,8 @@
import type { AuctionRedisKeys } from './auction-redis.keys';
import {
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
} from './auction-redis.keys';
import { defineAuctionRedisScript } from './auction-redis.scripts';
export interface AuctionBidScriptInput {
@@ -8,6 +12,7 @@ export interface AuctionBidScriptInput {
bidderUserId: string;
clientObservedAuctionLifecycleId: string;
clientObservedCurrentBidRoundPlayerUserId: string;
clientObservedBidRoundId?: string;
maximumAuctionTeamMemberCount: number;
maximumStoredAuctionBidHistoryCount: number;
maximumStoredBidRequestResultCount: number;
@@ -40,15 +45,19 @@ export function buildBidRequestPayloadFingerprint(params: {
bidPointAmount: number;
clientObservedAuctionLifecycleId: string;
clientObservedCurrentBidRoundPlayerUserId: string;
clientObservedBidRoundId?: string;
matchId: string;
}): string {
return [
const parts = [
params.matchId,
params.clientObservedAuctionLifecycleId,
params.clientObservedCurrentBidRoundPlayerUserId,
params.bidderTeamIndex,
params.bidPointAmount,
].join('|');
];
if (params.clientObservedBidRoundId)
parts.push(params.clientObservedBidRoundId);
return parts.join('|');
}
export function buildAuctionBidScriptInvocation(
@@ -62,6 +71,9 @@ export function buildAuctionBidScriptInvocation(
keys.bidRequestResultsKey,
keys.bidRequestResultOrderKey,
keys.deadlineQueueKey,
keys.rejectedBidRequestResultOrderKey,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
],
script: AUCTION_BID_SCRIPT,
values: [
@@ -78,6 +90,7 @@ export function buildAuctionBidScriptInvocation(
String(input.maximumStoredBidRequestResultCount),
String(input.minimumBidRequestResultReplayWindowMS),
String(input.maximumStoredAuctionBidHistoryCount),
input.clientObservedBidRoundId ?? '',
],
};
}
@@ -90,6 +103,9 @@ local auctionBidHistoryKey = KEYS[2]
local bidRequestResultsKey = KEYS[3]
local bidRequestResultOrderKey = KEYS[4]
local bidRoundDeadlineQueueKey = KEYS[5]
local rejectedBidRequestResultOrderKey = KEYS[6]
local bidArchiveQueueKey = KEYS[7]
local bidArchivePayloadsKey = KEYS[8]
local matchId = ARGV[1]
local bidderTeamIndex = ARGV[2]
@@ -104,6 +120,7 @@ local bidRequestPayloadFingerprint = ARGV[10]
local maximumStoredBidRequestResultCount = tonumber(ARGV[11])
local minimumBidRequestResultReplayWindowMS = tonumber(ARGV[12])
local maximumStoredAuctionBidHistoryCount = tonumber(ARGV[13])
local clientObservedBidRoundId = ARGV[14] or ''
local redisTime = redis.call('TIME')
local currentRedisTimestampMS = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)
@@ -168,12 +185,23 @@ if existingFingerprint then
return replayExistingRequest()
end
trimOldBidRequestResults()
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
-- Upgrade the old shared queue once, preserving every accepted replay record.
if redis.call('HSETNX', bidRequestResultsKey, '_acceptedOnlyOrderV1', '1') == 1 then
for _, oldRequestId in ipairs(redis.call('LRANGE', bidRequestResultOrderKey, 0, -1)) do
if redis.call('HGET', bidRequestResultsKey, oldRequestId .. ':status') == 'rejected' then
redis.call('LREM', bidRequestResultOrderKey, 0, oldRequestId)
hdelBidRequest(oldRequestId)
end
end
end
local function storeRejectedRequest(rejectCode)
-- Rejections have their own bounded retention. They must never consume the
-- protected replay capacity of accepted bids.
while redis.call('LLEN', rejectedBidRequestResultOrderKey) >= maximumStoredBidRequestResultCount do
local oldest = redis.call('LPOP', rejectedBidRequestResultOrderKey)
if oldest then hdelBidRequest(oldest) end
end
local rejectedCurrentBidRoundDeadlineTimestampMS = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or ''
redis.call(
'HSET',
@@ -191,7 +219,7 @@ local function storeRejectedRequest(rejectCode)
requestField('createdAtTimestampMS'),
tostring(currentRedisTimestampMS)
)
redis.call('RPUSH', bidRequestResultOrderKey, bidRequestId)
redis.call('RPUSH', rejectedBidRequestResultOrderKey, bidRequestId)
return {'REJECTED', rejectCode, bidRequestId, tostring(currentRedisTimestampMS), rejectedCurrentBidRoundDeadlineTimestampMS}
end
@@ -202,9 +230,16 @@ end
local auctionLifecycleId = redis.call('HGET', liveAuctionStateKey, 'auctionLifecycleId')
local currentBidRoundPlayerUserId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundPlayerUserId')
local currentBidRoundId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundId') or ''
if currentBidRoundId ~= '' and clientObservedBidRoundId == '' then
return storeRejectedRequest('BID_CLIENT_UPDATE_REQUIRED')
end
if auctionLifecycleId ~= clientObservedAuctionLifecycleId or currentBidRoundPlayerUserId ~= clientObservedCurrentBidRoundPlayerUserId then
return storeRejectedRequest('STALE_BID_ROUND')
end
if currentBidRoundId ~= clientObservedBidRoundId then
return storeRejectedRequest('STALE_BID_ROUND')
end
local currentBidRoundDeadlineTimestampMS = tonumber(redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or '0')
if currentBidRoundDeadlineTimestampMS <= currentRedisTimestampMS then
@@ -235,8 +270,9 @@ if bidPointAmount + requiredReserveAfterBid > remainingBidPointAmount then
return storeRejectedRequest('INSUFFICIENT')
end
if redis.call('LLEN', auctionBidHistoryKey) >= maximumStoredAuctionBidHistoryCount then
return storeRejectedRequest('BID_HISTORY_LIMIT')
trimOldBidRequestResults()
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
end
local maximumBidRoundDurationSeconds = tonumber(redis.call('HGET', liveAuctionStateKey, 'maximumBidRoundDurationSeconds') or '0')
@@ -254,14 +290,36 @@ end
local nextBidRoundDeadlineTimestampMS = currentRedisTimestampMS + nextRemainingBidRoundTimeMS
local revision = redis.call('HINCRBY', liveAuctionStateKey, 'revision', 1)
local auctionBidHistoryEntryJson =
'{"bidderTeamIndex":' .. bidderTeamIndex ..
',"bidPointAmount":' .. tostring(bidPointAmount) ..
',"bidderUserId":"' .. bidderUserId ..
'","bidPlacedAtTimestampMS":' .. tostring(currentRedisTimestampMS) ..
'}'
local auctionBidHistoryEntryJson = cjson.encode({
bidRequestId = bidRequestId,
revision = revision,
bidderTeamIndex = tonumber(bidderTeamIndex),
bidPointAmount = bidPointAmount,
bidderUserId = bidderUserId,
playerUserId = currentBidRoundPlayerUserId,
bidRoundId = currentBidRoundId,
bidPlacedAtTimestampMS = currentRedisTimestampMS
})
-- Accepted replay returns above this point. Persist every accepted event before
-- trimming the recent view; the durable archive worker removes only saved events.
local archiveId = matchId .. ':' .. auctionLifecycleId .. ':' .. string.format('%016d', revision)
redis.call('HSET', bidArchivePayloadsKey, archiveId, cjson.encode({
_id = archiveId,
matchId = matchId,
auctionLifecycleId = auctionLifecycleId,
revision = revision,
bidRequestId = bidRequestId,
bidderTeamIndex = tonumber(bidderTeamIndex),
bidPointAmount = bidPointAmount,
bidderUserId = bidderUserId,
playerUserId = currentBidRoundPlayerUserId,
bidRoundId = currentBidRoundId,
bidPlacedAtTimestampMS = currentRedisTimestampMS
}))
redis.call('ZADD', bidArchiveQueueKey, currentRedisTimestampMS, archiveId)
redis.call('RPUSH', auctionBidHistoryKey, auctionBidHistoryEntryJson)
redis.call('LTRIM', auctionBidHistoryKey, -maximumStoredAuctionBidHistoryCount, -1)
redis.call(
'HSET',
liveAuctionStateKey,
@@ -4,6 +4,14 @@ export const AUCTION_DURABILITY_RETRY_QUEUE_KEY =
'auction:auctionDurabilityRetryQueue';
export const AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY =
'auction:auctionDurabilityDeadLetterQueue';
export const AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY =
'auction:deadLetterStatusQueue';
export const AUCTION_BID_ARCHIVE_QUEUE_KEY = 'auction:bidArchiveQueue';
export const AUCTION_BID_ARCHIVE_PAYLOADS_KEY = 'auction:bidArchivePayloads';
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY =
'auction:bidArchiveDeadLetterQueue';
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY =
'auction:bidArchiveDeadLetterPayloads';
export interface AuctionRedisKeys {
actionRequestResultOrderKey: string;
@@ -12,6 +20,7 @@ export interface AuctionRedisKeys {
bidHistoryKey: string;
bidRequestResultOrderKey: string;
bidRequestResultsKey: string;
rejectedBidRequestResultOrderKey: string;
deadlineQueueKey: string;
durabilityDeadLetterQueueKey: string;
durabilityPayloadKey: (auctionStateTransitionId: string) => string;
@@ -33,6 +42,7 @@ export function buildAuctionRedisKeys(matchId: string): AuctionRedisKeys {
bidHistoryKey: `${prefix}:auctionBidHistory`,
bidRequestResultOrderKey: `${prefix}:bidRequestResultOrder`,
bidRequestResultsKey: `${prefix}:bidRequestResults`,
rejectedBidRequestResultOrderKey: `${prefix}:rejectedBidRequestResultOrder`,
deadlineQueueKey: AUCTION_BID_ROUND_DEADLINE_QUEUE_KEY,
durabilityDeadLetterQueueKey: AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
durabilityPayloadKey: (auctionStateTransitionId: string) =>
@@ -24,7 +24,7 @@ describe('auction redis state assembly', () => {
mongo: { auctionLifecycleId: 'new-life', revision: 2 },
},
{
expected: false,
expected: true,
live: { auctionLifecycleId: 'redis-life', revision: 3 },
mongo: { auctionLifecycleId: 'mongo-life', revision: 2 },
},
@@ -62,13 +62,11 @@ export function shouldHydrateRedisAuctionState(
params: ShouldHydrateRedisAuctionStateParams,
): boolean {
if (!params.live) return true;
if (params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId)
return true;
const redisRevision = params.live.revision ?? -1;
const mongoRevision = params.mongo.revision;
if (mongoRevision > redisRevision) return true;
return (
mongoRevision === redisRevision &&
params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId
);
return mongoRevision > redisRevision;
}
export function assembleAuctionViewFromRedisState(
@@ -140,13 +138,17 @@ if not liveStatePatchPairCount or #ARGV ~= 8 + (liveStatePatchPairCount * 2) the
return {'REJECTED', 'INVALID_PATCH_ARITY', tostring(currentRedisTimestampMS)}
end
-- A lifecycle replacement requires Mongo authority and an explicit cleanup
-- under the lifecycle lock. Revision numbers from different runs are unrelated.
if redisAuctionLifecycleId and redisAuctionLifecycleId ~= mongoAuctionLifecycleId then
return {'REJECTED', 'LIFECYCLE_CONFLICT', tostring(currentRedisTimestampMS)}
end
local shouldHydrate = false
if redisRevision < 0 then
shouldHydrate = true
elseif mongoRevision > redisRevision then
shouldHydrate = true
elseif mongoRevision == redisRevision and mongoAuctionLifecycleId ~= redisAuctionLifecycleId then
shouldHydrate = true
end
if not shouldHydrate then
@@ -184,6 +186,7 @@ if phase == 'bidding' then
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
else
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
end
return {'OK', tostring(hydratedRevision), tostring(currentRedisTimestampMS)}
@@ -0,0 +1,7 @@
export const AUCTION_CANCELLATION_CLEANUP_INDEX = {
key: { auctionCancellationCleanupRetryAt: 1 as const },
options: {
name: 'auction_cancellation_cleanup_due',
partialFilterExpression: { auctionCancellationCleanupPending: true },
},
};
@@ -0,0 +1,39 @@
import type { mongo } from 'mongoose';
import { AUCTION_CANCELLATION_CLEANUP_INDEX } from '../auction-cancellation-index';
/** Creates only this additive index; never drops or rebuilds other indexes. */
export async function migrateAuctionCancellationIndex(
db: mongo.Db,
{ dryRun = true }: { dryRun?: boolean } = {},
) {
const { key, options } = AUCTION_CANCELLATION_CLEANUP_INDEX;
const collection = db.collection('match');
const indexes = await collection
.listIndexes()
.toArray()
.catch((error: unknown) => {
if (error instanceof Error && 'code' in error && error.code === 26)
return [];
throw error;
});
const existing = indexes.find((index) => index.name === options.name);
if (
existing &&
(JSON.stringify(existing.key) !== JSON.stringify(key) ||
JSON.stringify(existing.partialFilterExpression) !==
JSON.stringify(options.partialFilterExpression) ||
existing.unique === true ||
existing.sparse === true ||
existing.hidden === true)
) {
throw new Error(
`Index ${options.name} already exists with a different definition.`,
);
}
if (!dryRun) await collection.createIndex(key, options);
return {
dryRun,
alreadyPresent: Boolean(existing),
index: { key, ...options },
};
}
@@ -0,0 +1,105 @@
import 'reflect-metadata';
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { NestFactory } from '@nestjs/core';
import { readFile } from 'node:fs/promises';
import { resolve } from 'node:path';
import { ConnectionService, resolveMongoDbName } from '../connection.service';
import { SecretModule } from '../../secret/secret.module';
import { migrateAuctionCancellationIndex } from './20261109-auction-cancellation-index';
@Module({
imports: [
ConfigModule.forRoot({
isGlobal: true,
envFilePath: [
resolve(__dirname, '../../../../../../project.env'),
resolve(process.cwd(), '../project.env'),
resolve(process.cwd(), 'apps/project.env'),
resolve(process.cwd(), 'project.env'),
],
}),
SecretModule,
],
providers: [ConnectionService],
})
class MigrationModule {}
export function parseAuctionIndexOptions(argv: string[]) {
let dryRun = true;
let help = false;
let nodeEnv = process.env.NODE_ENV ?? 'dev';
for (let index = 0; index < argv.length; index++) {
const argument = argv[index];
if (argument === '--') continue;
if (argument === '--apply') dryRun = false;
else if (argument === '--dry-run') dryRun = true;
else if (argument === '--help' || argument === '-h') help = true;
else if (argument === '--env') nodeEnv = argv[++index] ?? '';
else if (argument.startsWith('--env=')) nodeEnv = argument.slice(6);
else throw new Error(`Unknown argument: ${argument}`);
}
if (!help && !['dev', 'staging', 'prod'].includes(nodeEnv)) {
throw new Error('--env must be dev, staging, or prod.');
}
return { dryRun, help, nodeEnv };
}
async function main(): Promise<void> {
const options = parseAuctionIndexOptions(process.argv.slice(2));
if (options.help) {
console.log(`Usage:
pnpm --filter api auction:indexes -- --env staging --dry-run
pnpm --filter api auction:indexes -- --env prod --apply
Defaults to dry-run. --env accepts dev, staging, or prod (NODE_ENV or dev by default).
Creates the Match cancellation cleanup partial index; does not modify documents.`);
return;
}
// Use the same optional machine environment as the existing migration runners.
const machineHome = process.env.HOME ?? process.env.USERPROFILE;
if (machineHome) {
const contents = await readFile(
resolve(machineHome, 'source/global-env-vars.txt'),
'utf8',
).catch(() => '');
for (const line of contents.split(/\r?\n/)) {
const match = line.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
if (match) process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');
}
}
process.env.NODE_ENV = options.nodeEnv;
// Even a dry-run with auto-index enabled in the environment must not write.
process.env.MONGODB_AUTO_INDEX = 'false';
const app = await NestFactory.createApplicationContext(MigrationModule, {
logger: ['warn', 'error'],
});
try {
const connection = await app.get(ConnectionService).getConnection();
if (!connection.db) throw new Error('MongoDB connection has no db handle.');
const result = await migrateAuctionCancellationIndex(
connection.db,
options,
);
console.log(
JSON.stringify(
{
env: options.nodeEnv,
db: resolveMongoDbName(options.nodeEnv),
...result,
},
null,
2,
),
);
} finally {
await app.close();
}
}
if (require.main === module) {
void main().catch((error: unknown) => {
console.error(error instanceof Error ? error.message : error);
process.exitCode = 1;
});
}
@@ -12,6 +12,7 @@ import {
competitionModelDefinitions,
} from '../../competition/competition.schema';
import { ModelDefinition } from './types';
import { AUCTION_CANCELLATION_CLEANUP_INDEX } from './auction-cancellation-index';
export type Position = 'TOP' | 'JGL' | 'MID' | 'ADC' | 'SUP';
export const POSITIONS: Position[] = ['TOP', 'JGL', 'MID', 'ADC', 'SUP'];
@@ -700,6 +701,12 @@ export interface Match extends MongooseTimestamps {
teams: MatchTeam[];
result?: MatchResult;
resultPollState?: MatchResultPollState;
/** 마지막으로 취소된 Redis 경매 lifecycle. 취소 재전송을 멱등 처리한다. */
lastCancelledAuctionLifecycleId?: string;
auctionCancellationCleanupPending?: boolean;
auctionCancellationCleanupRetryAt?: Date;
auctionCancellationCleanupAttempts?: number;
auctionCancellationCleanupError?: string;
doubleElimination?: DoubleEliminationBracket;
legacyImported?: boolean;
legacyImportSource?: {
@@ -804,12 +811,41 @@ export interface AuctionBid {
}
export interface AuctionBidHistoryEntry {
bidRequestId?: string;
revision?: number;
bidRoundId?: string;
playerUserId?: mongoose.Types.ObjectId;
bidderTeamIndex: number;
bidPointAmount: number;
bidderUserId: mongoose.Types.ObjectId;
bidPlacedAtTimestampMS: number;
}
export interface AuctionBidRecord extends AuctionBidHistoryEntry {
_id: string;
matchId: mongoose.Types.ObjectId;
auctionLifecycleId: string;
revision: number;
bidRequestId: string;
}
export const AuctionBidRecordSchema = new Schema<AuctionBidRecord>(
{
_id: { type: String, required: true },
matchId: { type: Schema.Types.ObjectId, required: true },
auctionLifecycleId: { type: String, required: true },
revision: { type: Number, required: true },
bidRequestId: { type: String, required: true },
bidRoundId: { type: String },
playerUserId: { type: Schema.Types.ObjectId },
bidderUserId: { type: Schema.Types.ObjectId, required: true },
bidderTeamIndex: { type: Number, required: true },
bidPointAmount: { type: Number, required: true },
bidPlacedAtTimestampMS: { type: Number, required: true },
},
{ collection: 'auctionBidRecords', versionKey: false },
);
export interface AuctionChatMessage {
id: string;
userId?: mongoose.Types.ObjectId;
@@ -832,6 +868,8 @@ export interface Auction extends MongooseTimestamps {
matchSideEffectStatus: AuctionMatchSideEffectStatus;
matchSideEffectRevision: number;
matchSideEffectError?: string;
matchSideEffectTransitionId?: string;
startupCompleted?: boolean;
teams: AuctionTeam[];
order: AuctionPlayerSnapshot[];
passedQueue: AuctionPlayerSnapshot[];
@@ -858,6 +896,7 @@ export interface Auction extends MongooseTimestamps {
currentHighestBidPointAmount?: number;
currentHighestBidderTeamIndex?: number;
currentBidRoundDeadlineTimestampMS?: number;
currentBidRoundId?: string;
maximumBidRoundDurationSeconds?: number;
bidRoundExtensionSeconds?: number;
initialTeamBidPointAmount?: number;
@@ -1971,6 +2010,11 @@ export const MatchSchema = new Schema<Match>(
type: MatchResultPollStateSubSchema,
required: false,
},
lastCancelledAuctionLifecycleId: { type: String, required: false },
auctionCancellationCleanupPending: { type: Boolean },
auctionCancellationCleanupRetryAt: { type: Date },
auctionCancellationCleanupAttempts: { type: Number },
auctionCancellationCleanupError: { type: String },
doubleElimination: {
type: DoubleEliminationBracketSubSchema,
required: false,
@@ -1995,6 +2039,10 @@ export const MatchSchema = new Schema<Match>(
);
MatchSchema.index({ communityId: 1, scheduledAt: -1 });
MatchSchema.index(
AUCTION_CANCELLATION_CLEANUP_INDEX.key,
AUCTION_CANCELLATION_CLEANUP_INDEX.options,
);
applyCompetitionDomainInvariantIndexes(MatchSchema, 'match');
MatchSchema.index({ communityId: 1, status: 1, scheduledAt: -1 });
MatchSchema.index(
@@ -2174,6 +2222,8 @@ const AuctionBidSubSchema = new Schema<AuctionBid>(
const AuctionBidHistoryEntrySubSchema = new Schema<AuctionBidHistoryEntry>(
{
bidRoundId: { type: String },
playerUserId: { type: Schema.Types.ObjectId, ref: 'user' },
bidderTeamIndex: { type: Number, required: true, min: 1 },
bidPointAmount: { type: Number, required: true },
bidderUserId: { type: Schema.Types.ObjectId, ref: 'user', required: true },
@@ -2195,6 +2245,9 @@ const AuctionChatMessageSubSchema = new Schema<AuctionChatMessage>(
export const AuctionSchema = new Schema<Auction>(
{
startupCompleted: { type: Boolean },
matchSideEffectTransitionId: { type: String },
currentBidRoundId: { type: String },
matchId: {
type: Schema.Types.ObjectId,
ref: 'match',
@@ -2520,6 +2573,11 @@ export const tournamentModelDefinitions: Array<
},
{ name: 'match', schema: MatchSchema, schemaName: 'match' },
{ name: 'auction', schema: AuctionSchema, schemaName: 'auction' },
{
name: 'auctionBidRecord',
schema: AuctionBidRecordSchema,
schemaName: 'auctionBidRecords',
},
{
name: 'auctionHistory',
schema: AuctionHistoryEntrySchema,
@@ -2553,6 +2611,7 @@ export interface TournamentModelTypes {
legacyImportJob: LegacyImportJob;
match: Match;
auction: Auction;
auctionBidRecord: AuctionBidRecord;
auctionHistory: AuctionHistoryEntry;
auctionSideEffectApplication: AuctionSideEffectApplication;
matchStats: MatchStats;
@@ -90,6 +90,33 @@ function createMatch(overrides: MatchOverrides): MatchDocument {
} as unknown as MatchDocument;
}
it('releases reserved codes inside the caller transaction during auction cancellation', async () => {
const session = createSession();
const match = createMatch({
tournamentCodeIds: [new mongoose.Types.ObjectId()],
});
const matchQuery = mongoQuery(match);
const codeQuery = mongoQuery({ modifiedCount: 1 });
const createSessionSpy = jest.fn();
const service = new TournamentCodeService(
{
createSession: createSessionSpy,
getModel: async (name: string) =>
name === 'match'
? { findById: () => matchQuery }
: { updateMany: () => codeQuery },
} as never,
createAuctionRedisService() as never,
);
await service.releaseFromMatch(match, session as never);
expect(createSessionSpy).not.toHaveBeenCalled();
expect(matchQuery.session).not.toHaveBeenCalled();
expect(codeQuery.session).toHaveBeenCalledWith(session);
expect(match.save).toHaveBeenCalledWith({ session });
expect(session.commitTransaction).not.toHaveBeenCalled();
expect(match.tournamentCodeIds).toEqual([]);
});
describe('TournamentCodeService round-robin scheduling', () => {
it('keeps pair slot calculation unchanged', () => {
const service = createService();
@@ -329,23 +329,31 @@ export class TournamentCodeService {
public async releaseFromMatch(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
session?: mongoose.ClientSession,
): Promise<void> {
const matchId = this.resolveMatchId(matchOrId);
// Auction cancellation owns the surrounding transaction. Committing a
// separate release would survive its rollback and conflict with Match.save.
if (session) return this.releaseFromMatchInTransaction(matchOrId, session);
await this.withLockedMatchTransaction(matchId, (session) =>
this.releaseFromMatchInTransaction(matchId, session),
);
}
private async releaseFromMatchInTransaction(
matchId: mongoose.Types.ObjectId,
matchOrId: MatchDocument | mongoose.Types.ObjectId,
session: mongoose.ClientSession,
): Promise<void> {
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
const match = await this.execQuery<MatchDocument | null>(
this.withQuerySession(MatchModel.findById(matchId), session),
);
// Preserve the caller's document and __v when participating in its session.
const match =
matchOrId instanceof mongoose.Types.ObjectId
? await this.execQuery<MatchDocument | null>(
this.withQuerySession(MatchModel.findById(matchOrId), session),
)
: matchOrId;
if (!match) return;
if (match.tournamentCodeIds.length === 0) return;
@@ -240,10 +240,15 @@ export class TournamentResultService {
public async cancelPolling(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
session?: mongoose.ClientSession,
): Promise<void> {
const match = await this.loadMatch(matchOrId);
const match =
matchOrId instanceof mongoose.Types.ObjectId
? await this.loadMatch(matchOrId, session)
: matchOrId;
match.resultPollState = undefined;
await match.save();
if (session) await match.save({ session });
else await match.save();
}
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
@@ -908,12 +913,14 @@ export class TournamentResultService {
private async loadMatch(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
session?: mongoose.ClientSession,
): Promise<MatchDocument> {
const MatchModel = await this.connectionService.getModel('match');
const match =
const query =
matchOrId instanceof mongoose.Types.ObjectId
? await MatchModel.findById(matchOrId)
: await MatchModel.findById(matchOrId._id);
? MatchModel.findById(matchOrId)
: MatchModel.findById(matchOrId._id);
const match = await (session ? query.session(session) : query);
if (!match) throw new NotFoundException('Match not found.');
return match;
}
@@ -446,6 +446,13 @@ async function main(): Promise<void> {
const bidResult = await auctionService.placeBid({
amount: 10,
bidRequestId: randomUUID(),
clientObservedBidRoundId: String(
(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
)
)?.currentBidRoundId,
),
matchId: resources.matchId,
teamIndex: 1,
});
@@ -537,6 +544,13 @@ async function main(): Promise<void> {
await auctionService.reset({
auctionActionRequestId: randomUUID(),
clientObservedAuctionLifecycleId: String(
(
await auctionRedisStateReaderService.readAuctionStateView(
resources.matchId.toString(),
)
)?.auctionLifecycleId,
),
communityId: resources.communityId,
matchId: resources.matchId,
});
@@ -0,0 +1,180 @@
import { createClient } from 'redis';
import mongoose from 'mongoose';
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
import {
AUCTION_BID_ARCHIVE_QUEUE_KEY as queue,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY as payloads,
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY as deadQueue,
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY as deadPayloads,
} from '../src/modules/auction/redis/auction-redis.keys';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new AuctionRedisService(client);
const matchId = new mongoose.Types.ObjectId().toString();
function event(revision: number) {
return {
_id: `${matchId}:life:${String(revision).padStart(16, '0')}`,
matchId,
auctionLifecycleId: 'life',
revision,
bidRequestId: `bid-${revision}`,
bidderUserId: new mongoose.Types.ObjectId().toString(),
bidderTeamIndex: 1,
bidPointAmount: revision,
bidPlacedAtTimestampMS: Date.now(),
};
}
function fixture() {
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
matchedCount: 0,
upsertedCount: operations.length,
}));
const service = new AuctionBidArchiveService(
{ getModel: async () => ({ bulkWrite }) } as never,
redis,
);
return { service, bulkWrite };
}
async function enqueue(id: string, raw: string | null) {
await client.zAdd(queue, { score: 1, value: id });
if (raw !== null) await client.hSet(payloads, id, raw);
}
beforeAll(async () => {
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL)
throw new Error('Use the isolated Redis test runner.');
client.on('error', () => {});
await client.connect();
});
beforeEach(async () => {
await client.flushDb();
});
afterEach(() => jest.restoreAllMocks());
afterAll(async () => {
if (client.isOpen) await client.quit();
});
it('isolates malformed, missing and invalid payloads while saving valid bids and retaining the originals', async () => {
const { service, bulkWrite } = fixture();
const good = event(10);
const invalid = [
['broken', '{broken'],
['empty', ''],
['null', 'null'],
['array', '[]'],
['missing', null],
[event(1)._id, JSON.stringify({ ...event(1), revision: -1 })],
[event(2)._id, JSON.stringify({ ...event(2), bidderUserId: 'bad' })],
[event(3)._id, JSON.stringify({ ...event(3), bidderUserId: undefined })],
[event(4)._id, JSON.stringify({ ...event(4), bidPointAmount: '4' })],
] as Array<[string, string | null]>;
for (const [id, raw] of invalid) await enqueue(id, raw);
await enqueue(good._id, JSON.stringify(good));
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(bulkWrite.mock.calls[0][0]).toHaveLength(1);
expect(await client.zCard(queue)).toBe(0);
expect(await client.hLen(payloads)).toBe(0);
expect(await client.zCard(deadQueue)).toBe(invalid.length);
for (const [id, raw] of invalid) {
expect(JSON.parse((await client.hGet(deadPayloads, id))!)).toMatchObject({
id,
rawPayload: raw,
error: expect.any(String),
quarantinedAtTimestampMS: expect.any(Number),
});
}
expect(await client.ttl(deadPayloads)).toBe(-1);
await expect(service.persistPendingBids()).resolves.toBe(0);
});
it('retains valid bids during a Mongo outage, after moving malformed entries out of the pending queue', async () => {
const { service, bulkWrite } = fixture();
const good = event(1);
await enqueue('broken', '{broken');
await enqueue(good._id, JSON.stringify(good));
bulkWrite.mockRejectedValueOnce(new Error('Mongo offline'));
await expect(service.persistPendingBids()).rejects.toThrow('Mongo offline');
expect(await client.zRange(queue, 0, -1)).toEqual([good._id]);
expect(await client.hGet(payloads, good._id)).toBe(JSON.stringify(good));
expect(await client.zCard(deadQueue)).toBe(1);
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(await client.zCard(queue)).toBe(0);
});
it('does not quarantine a payload repaired concurrently after HMGET', async () => {
const { service } = fixture();
const good = event(1);
await enqueue(good._id, '{broken');
const send = client.sendCommand.bind(client);
jest.spyOn(client, 'sendCommand').mockImplementation(async (command) => {
const result = await send(command);
if (command[0] === 'HMGET')
await client.hSet(payloads, good._id, JSON.stringify(good));
return result;
});
await expect(service.persistPendingBids()).resolves.toBe(0);
expect(await client.zCard(deadQueue)).toBe(0);
expect(await client.zRange(queue, 0, -1)).toEqual([good._id]);
await expect(service.persistPendingBids()).resolves.toBe(1);
});
it('preserves pending payloads if quarantine cannot be written', async () => {
const { service, bulkWrite } = fixture();
const good = event(1);
await enqueue('broken', '{broken');
await enqueue(good._id, JSON.stringify(good));
await client.set(deadPayloads, 'wrong-type');
await expect(service.persistPendingBids()).rejects.toThrow('WRONGTYPE');
expect(await client.zCard(queue)).toBe(2);
expect(await client.hGet(payloads, 'broken')).toBe('{broken');
expect(bulkWrite).not.toHaveBeenCalled();
await client.del(deadPayloads);
await expect(service.persistPendingBids()).resolves.toBe(1);
});
it('does not acknowledge a payload changed while Mongo was saving', async () => {
const { service, bulkWrite } = fixture();
const good = event(1);
await enqueue(good._id, JSON.stringify(good));
bulkWrite.mockImplementationOnce(async () => {
await client.hSet(payloads, good._id, '{replaced');
return { matchedCount: 0, upsertedCount: 1 };
});
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(await client.hGet(payloads, good._id)).toBe('{replaced');
expect(await client.zCard(queue)).toBe(1);
});
it.each([
['bidRoundId', null],
['bidRoundId', 1],
['bidRoundId', {}],
['bidRoundId', []],
['bidRoundId', ' '],
['playerUserId', ''],
])('continues quarantining invalid %s=%j', async (field, value) => {
const { service, bulkWrite } = fixture();
const bad = { ...event(1), [field]: value };
await enqueue(bad._id, JSON.stringify(bad));
await expect(service.persistPendingBids()).resolves.toBe(0);
expect(bulkWrite).not.toHaveBeenCalled();
expect(await client.zCard(deadQueue)).toBe(1);
});
it('archives a pre-upgrade bid with an empty round ID instead of quarantining it', async () => {
const { service, bulkWrite } = fixture();
const legacy = { ...event(1), bidRoundId: '' };
await enqueue(legacy._id, JSON.stringify(legacy));
await expect(service.persistPendingBids()).resolves.toBe(1);
expect(await client.zCard(deadQueue)).toBe(0);
expect(await client.zCard(queue)).toBe(0);
const [[operations]] = bulkWrite.mock.calls as unknown as [
[Array<{ updateOne: { update: { $setOnInsert: object } } }>],
];
expect(operations[0].updateOne.update.$setOnInsert).not.toHaveProperty(
'bidRoundId',
);
});
@@ -0,0 +1,769 @@
import mongoose from 'mongoose';
import { AuctionService } from '../src/modules/auction/auction.service';
import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler';
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
import { migrateAuctionCancellationIndex } from '../src/modules/global/database/migrations/20261109-auction-cancellation-index';
import { AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY } from '../src/modules/auction/redis/auction-redis.keys';
import { TournamentCodeService } from '../src/modules/tournament-code/tournament-code.service';
import { TournamentResultService } from '../src/modules/tournament-result/tournament-result.service';
import {
MatchSchema,
AuctionSchema,
AuctionHistoryEntrySchema,
AuctionSideEffectApplicationSchema,
TournamentCodeSchema,
AuctionBidRecordSchema,
} from '../src/modules/global/database/schema';
let connection: mongoose.Connection;
let models: Record<string, mongoose.Model<any>>;
const redis = {
acquireTokenLock: async () => true,
releaseTokenLock: async () => true,
getCurrentRedisTimestampMS: async () => Date.now(),
raw: () => ({
sendCommand: async (command: string[]) =>
command[0] === 'SCAN' ? ['0', []] : 1,
}),
};
beforeAll(async () => {
if (
process.env.MONGO_TEST_ISOLATED !== '1' ||
!process.env.MONGO_TEST_URL?.startsWith('mongodb://127.0.0.1:')
)
throw new Error('Use test:mongodb with an isolated local replica set.');
connection = await mongoose
.createConnection(process.env.MONGO_TEST_URL, {
autoIndex: false,
autoCreate: false,
})
.asPromise();
models = {
match: connection.model('match', MatchSchema),
auction: connection.model('auction', AuctionSchema),
auctionHistory: connection.model(
'auctionHistory',
AuctionHistoryEntrySchema,
),
auctionSideEffectApplication: connection.model(
'auctionSideEffectApplication',
AuctionSideEffectApplicationSchema,
),
tournamentCode: connection.model('tournamentCode', TournamentCodeSchema),
auctionBidRecord: connection.model(
'auctionBidRecord',
AuctionBidRecordSchema,
),
};
await Promise.all(
Object.values(models).map((model) => model.createCollection()),
);
});
beforeEach(async () => {
await Promise.all(Object.values(models).map((model) => model.deleteMany({})));
});
afterEach(() => jest.restoreAllMocks());
afterAll(async () => {
await connection?.close();
});
async function fixture(reserved: boolean) {
const matchId = new mongoose.Types.ObjectId(),
communityId = new mongoose.Types.ObjectId(),
auctionId = new mongoose.Types.ObjectId(),
codeId = new mongoose.Types.ObjectId();
await models.match.create({
_id: matchId,
communityId,
title: 'Isolated auction test',
scheduledAt: new Date(),
status: 'team-built',
teamBuiltAt: new Date(),
teams: [],
participants: [],
tournamentCodeIds: reserved ? [codeId] : [],
gameSchedule: [],
__v: 0,
});
await models.auction.collection.insertOne({
_id: auctionId,
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: 'life',
phase: 'complete',
revision: 10,
matchSideEffectRevision: 0,
matchSideEffectStatus: 'none',
});
if (reserved)
await models.tournamentCode.collection.insertOne({
_id: codeId,
code: 'ISOLATED-TEST',
state: 'reserved',
matchId,
});
const database = {
getModel: async (name: string) => models[name],
createSession: () => connection.startSession(),
};
const codes = new TournamentCodeService(database as never, redis as never);
const results = new TournamentResultService(
database as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
);
const service = new AuctionService(
database as never,
{} as never,
codes,
results,
redis as never,
{} as never,
{} as never,
);
return {
matchId,
communityId,
auctionId,
codeId,
service,
database,
codes,
results,
};
}
it.each([false, true])(
'cancels the auction atomically with reserved codes=%s',
async (reserved) => {
const f = await fixture(reserved);
await f.service.reset({
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
});
const match = await models.match.findById(f.matchId).lean();
expect(match).toMatchObject({
status: 'recruiting',
teams: [],
tournamentCodeIds: [],
gameSchedule: [],
});
expect(match?.teamBuiltAt).toBeUndefined();
expect(await models.auction.findById(f.auctionId)).toBeNull();
if (reserved)
expect(
(await models.tournamentCode.findById(f.codeId).lean())?.state,
).toBe('released');
},
);
it('rolls back code release and Match changes if cancellation later fails', async () => {
const f = await fixture(true);
jest.spyOn(models.auctionHistory, 'deleteMany').mockImplementation(() => {
throw new Error('forced history failure');
});
await expect(
f.service.reset({
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
}),
).rejects.toThrow('forced history failure');
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
'reserved',
);
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
status: 'team-built',
__v: 0,
tournamentCodeIds: [f.codeId],
});
expect(await models.auction.findById(f.auctionId)).not.toBeNull();
});
it('retries cancellation cleanup from Mongo after Redis fails, without another cancellation request', async () => {
const f = await fixture(true);
const cleanup = jest
.spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle')
.mockRejectedValueOnce(new Error('Redis offline'))
.mockResolvedValue(undefined);
await f.service.reset({
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
});
expect(cleanup).not.toHaveBeenCalled();
await f.service.retryPendingCancellations();
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
auctionCancellationCleanupPending: true,
auctionCancellationCleanupAttempts: 1,
});
await models.match.updateOne(
{ _id: f.matchId },
{ $set: { auctionCancellationCleanupRetryAt: new Date(0) } },
);
await f.service.retryPendingCancellations();
expect(cleanup).toHaveBeenCalledTimes(2);
expect(
(await models.match.findById(f.matchId).lean())
?.auctionCancellationCleanupPending,
).toBe(false);
});
it('keeps all 1005 Mongo bid records and idempotently retries when Redis acknowledgement fails', async () => {
const f = await fixture(false);
const events = Array.from({ length: 1005 }, (_, i) => ({
_id: `${f.matchId.toString()}:life:${String(i + 1).padStart(16, '0')}`,
matchId: f.matchId.toString(),
auctionLifecycleId: 'life',
revision: i + 1,
bidRequestId: `bid:${i}`,
bidRoundId: 'round',
bidderTeamIndex: 1,
bidPointAmount: i + 1,
bidderUserId: new mongoose.Types.ObjectId().toString(),
playerUserId: new mongoose.Types.ObjectId().toString(),
bidPlacedAtTimestampMS: Date.now(),
}));
let failAck = true;
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return events.map((e) => e._id);
if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e));
if (failAck) throw new Error('Redis acknowledgement failed');
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
await expect(archive.persistPendingBids(2000)).rejects.toThrow(
'acknowledgement failed',
);
expect(await models.auctionBidRecord.countDocuments()).toBe(1005);
failAck = false;
await expect(archive.persistPendingBids(2000)).resolves.toBe(1005);
expect(await models.auctionBidRecord.countDocuments()).toBe(1005);
const page = await archive.list(f.matchId, 1000, 100);
expect(page.items.map((item) => item.revision)).toEqual([
1001, 1002, 1003, 1004, 1005,
]);
expect(page.hasMore).toBe(false);
});
it('migrates the cleanup index idempotently and examines only due pending matches', async () => {
const f = await fixture(false);
const documents = Array.from({ length: 10000 }, (_, i) => ({
_id: new mongoose.Types.ObjectId(),
lastCancelledAuctionLifecycleId: `life-${i}`,
...(i < 5
? {
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: new Date(0),
}
: i === 5
? {
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: new Date(Date.now() + 3600000),
}
: {}),
}));
await models.match.collection.insertMany(documents);
const find = jest.spyOn(models.match, 'find');
const cleanup = jest
.spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle')
.mockResolvedValue(undefined);
await f.service.retryPendingCancellations();
expect(cleanup).toHaveBeenCalledTimes(5);
const query = find.mock.calls[0][0] as unknown as Record<string, unknown>;
// Restore only the five due jobs so explain measures the actual worker query.
await models.match.updateMany(
{ _id: { $in: documents.slice(0, 5).map((d) => d._id) } },
{
$set: {
auctionCancellationCleanupPending: true,
auctionCancellationCleanupRetryAt: new Date(0),
},
},
);
const explain = () =>
models.match.collection
.find(query)
.sort({ auctionCancellationCleanupRetryAt: 1 })
.limit(20)
.explain('executionStats');
expect(
(await explain()).executionStats.totalDocsExamined,
).toBeGreaterThanOrEqual(10000);
expect(await migrateAuctionCancellationIndex(connection.db!)).toMatchObject({
dryRun: true,
alreadyPresent: false,
});
expect(
(await models.match.collection.listIndexes().toArray()).some(
(i) => i.name === 'auction_cancellation_cleanup_due',
),
).toBe(false);
await migrateAuctionCancellationIndex(connection.db!, { dryRun: false });
expect(
await migrateAuctionCancellationIndex(connection.db!, { dryRun: false }),
).toMatchObject({ alreadyPresent: true });
const plan = await explain();
expect(plan.executionStats.nReturned).toBe(5);
expect(plan.executionStats.totalDocsExamined).toBe(5);
expect(JSON.stringify(plan.queryPlanner.winningPlan)).toContain('IXSCAN');
});
function archiveEvent(matchId: mongoose.Types.ObjectId, revision: number) {
return {
_id: `${matchId.toString()}:life:${String(revision).padStart(16, '0')}`,
matchId: matchId.toString(),
auctionLifecycleId: 'life',
revision,
bidRequestId: `bid-${revision}`,
bidRoundId: 'round',
bidderUserId: new mongoose.Types.ObjectId().toString(),
playerUserId: new mongoose.Types.ObjectId().toString(),
bidderTeamIndex: 1,
bidPointAmount: revision,
bidPlacedAtTimestampMS: Date.now(),
};
}
it.each(['', undefined, 'round'])(
'persists and lists bids with optional round ID %s without duplicating retries',
async (bidRoundId) => {
const f = await fixture(false);
const event = { ...archiveEvent(f.matchId, 1), bidRoundId };
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return [event._id];
if (command[0] === 'HMGET') return [JSON.stringify(event)];
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
await expect(archive.persistPendingBids()).resolves.toBe(1);
await expect(archive.persistPendingBids()).resolves.toBe(1);
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
const page = await archive.list(f.matchId);
expect(page.items).toHaveLength(1);
expect(String(page.items[0].playerUserId)).toBe(event.playerUserId);
if (bidRoundId) expect(page.items[0].bidRoundId).toBe(bidRoundId);
else expect(page.items[0]).not.toHaveProperty('bidRoundId');
expect(
sendCommand.mock.calls
.filter(([command]) => command[0] === 'EVAL')
.every(([command]) => command[2] === '2'),
).toBe(true);
},
);
it.each([
'json',
'revision',
'object-id',
'required-field',
'identity',
'missing-payload',
])(
'quarantines an invalid %s without losing the valid Mongo bid',
async (kind) => {
const f = await fixture(false);
const bad = archiveEvent(f.matchId, 1),
good = archiveEvent(f.matchId, 2);
const invalid: Record<string, unknown> = { ...bad };
if (kind === 'revision') invalid.revision = -1;
if (kind === 'object-id') invalid.bidderUserId = 'not-an-object-id';
if (kind === 'required-field') delete invalid.bidderUserId;
if (kind === 'identity')
invalid.matchId = new mongoose.Types.ObjectId().toString();
const raw =
kind === 'json'
? '{broken'
: kind === 'missing-payload'
? null
: JSON.stringify(invalid);
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return [bad._id, good._id];
if (command[0] === 'HMGET') return [raw, JSON.stringify(good)];
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
await expect(archive.persistPendingBids()).resolves.toBe(1);
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
expect(
await models.auctionBidRecord.findById(good._id).lean(),
).toMatchObject({ revision: 2 });
expect(await models.auctionBidRecord.findById(bad._id)).toBeNull();
const quarantine = sendCommand.mock.calls
.map((c) => c[0])
.find((c) => c[0] === 'EVAL' && c[2] === '4')!;
expect(JSON.parse(quarantine[11])).toMatchObject({
id: bad._id,
rawPayload: raw,
error: expect.any(String),
});
const ack = sendCommand.mock.calls
.map((c) => c[0])
.find((c) => c[0] === 'EVAL' && c[2] === '2')!;
expect(ack.slice(5)).toEqual([good._id, JSON.stringify(good)]);
},
);
it.each(['partial-write-error', 'partial-acknowledgement'])(
'retains the batch after %s and retries Mongo upserts without duplicates',
async (failure) => {
const f = await fixture(false);
const events = [archiveEvent(f.matchId, 1), archiveEvent(f.matchId, 2)];
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGE') return events.map((e) => e._id);
if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e));
return 1;
});
const archive = new AuctionBidArchiveService(
f.database as never,
{ raw: () => ({ sendCommand }) } as never,
);
const write = models.auctionBidRecord.bulkWrite.bind(
models.auctionBidRecord,
);
jest
.spyOn(models.auctionBidRecord, 'bulkWrite')
.mockImplementationOnce(async (operations, options) => {
const result = await write(operations.slice(0, 1), options);
if (failure === 'partial-write-error')
throw new Error('Lost Mongo acknowledgement');
return result;
});
await expect(archive.persistPendingBids()).rejects.toThrow();
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
expect(
sendCommand.mock.calls.filter(([c]) => c[0] === 'EVAL'),
).toHaveLength(0);
await expect(archive.persistPendingBids()).resolves.toBe(2);
expect(await models.auctionBidRecord.countDocuments()).toBe(2);
},
);
it.each(['Mongo', 'Redis mirroring'])(
'reconciles a terminal status after %s recovers',
async (outage) => {
const f = await fixture(false);
const payload = {
auctionId: f.auctionId.toString(),
matchId: f.matchId.toString(),
auctionLifecycleId: 'life',
auctionStateTransitionId: 'complete-10',
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
afterAuctionMongoProjection: { matchSideEffectRevision: 1 },
};
await models.auction.updateOne(
{ _id: f.auctionId },
{
$set: {
matchSideEffectTransitionId: 'complete-10',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
},
);
let unavailable = outage === 'Mongo';
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGEBYSCORE') return ['dead-letter'];
if (command[0] === 'HGETALL')
return { payloadJson: JSON.stringify(payload) };
if (command[0] === 'HGET') return 'retry limit exceeded';
return 1;
});
const mirror = jest.fn().mockResolvedValue(true),
broadcast = jest.fn();
if (outage === 'Redis mirroring')
mirror.mockRejectedValueOnce(new Error('Redis unavailable'));
const scheduler = new AuctionDurabilityScheduler(
{
getModel: async (name: string) => {
if (unavailable) throw new Error('Mongo offline');
return models[name];
},
} as never,
{
raw: () => ({ sendCommand }),
getCurrentRedisTimestampMS: async () => 1000,
} as never,
undefined,
undefined,
{ mirrorMatchSideEffectStatus: mirror } as never,
{ broadcastMatchSideEffect: broadcast } as never,
);
await scheduler.reconcileDeadLetterStatuses();
expect(sendCommand).toHaveBeenCalledWith([
'ZADD',
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
'6000',
'dead-letter',
]);
unavailable = false;
await scheduler.reconcileDeadLetterStatuses();
expect(
(await models.auction.findById(f.auctionId).lean())
?.matchSideEffectStatus,
).toBe('failed');
expect(mirror).toHaveBeenCalledWith(
expect.objectContaining({
auctionStateTransitionId: 'complete-10',
matchSideEffectStatus: 'failed',
}),
);
expect(broadcast).toHaveBeenCalledTimes(1);
// Replaying the old notification must not overwrite a new generation.
await models.auction.updateOne(
{ _id: f.auctionId },
{
$set: {
matchSideEffectTransitionId: 'new',
matchSideEffectStatus: 'processing',
matchSideEffectRevision: 20,
},
},
);
await scheduler.reconcileDeadLetterStatuses();
expect(
(await models.auction.findById(f.auctionId).lean())
?.matchSideEffectStatus,
).toBe('processing');
expect(broadcast).toHaveBeenCalledTimes(1);
},
);
it('cleans up reserved codes and Match metadata after undoing completion', async () => {
const f = await fixture(true);
const scheduler = new AuctionDurabilityScheduler(
f.database as never,
redis as never,
f.codes,
f.results,
undefined,
undefined,
{
readAuctionStateView: async () => ({
auctionLifecycleId: 'life',
phase: 'bidding',
matchSideEffectTransitionId: 'undo',
}),
} as never,
);
await scheduler.applyDurabilityPayload({
auctionId: f.auctionId,
matchId: f.matchId,
auctionLifecycleId: 'life',
revision: 11,
auctionHistorySequence: 11,
auctionActionType: 'undo',
auctionStateTransitionId: 'undo',
snapshotBefore: {},
historyMutation: { type: 'append' },
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
afterAuctionMongoProjection: {
phase: 'bidding',
revision: 11,
matchSideEffectTransitionId: 'undo',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
});
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
status: 'team-built',
tournamentCodeIds: [],
gameSchedule: [],
teams: [],
});
expect(
(await models.match.findById(f.matchId).lean())?.teamBuiltAt,
).toBeUndefined();
expect(
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
).toBe('completed');
});
it('reads an uncommitted Match in the provided session when cancelling polling by id', async () => {
const f = await fixture(false);
const session = await connection.startSession();
try {
session.startTransaction();
const [created] = await models.match.create(
[
{
communityId: f.communityId,
title: 'Uncommitted',
scheduledAt: new Date(),
},
],
{ session },
);
expect(await models.match.findById(created._id)).toBeNull();
await f.results.cancelPolling(
created._id as mongoose.Types.ObjectId,
session,
);
expect(
await models.match.findById(created._id).session(session),
).not.toBeNull();
await session.abortTransaction();
} finally {
await session.endSession();
}
});
it('rolls back failed Undo cleanup and successfully retries with fresh Match versions', async () => {
const f = await fixture(true);
const scheduler = new AuctionDurabilityScheduler(
f.database as never,
redis as never,
f.codes,
f.results,
undefined,
undefined,
{
readAuctionStateView: async () => ({
auctionLifecycleId: 'life',
phase: 'bidding',
matchSideEffectTransitionId: 'undo-retry',
}),
} as never,
);
const payload = {
auctionId: f.auctionId,
matchId: f.matchId,
auctionLifecycleId: 'life',
revision: 11,
auctionHistorySequence: 11,
auctionActionType: 'undo',
auctionStateTransitionId: 'undo-retry',
snapshotBefore: {},
historyMutation: { type: 'append' },
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
afterAuctionMongoProjection: {
phase: 'bidding',
revision: 11,
matchSideEffectTransitionId: 'undo-retry',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
};
jest
.spyOn(f.results, 'cancelPolling')
.mockRejectedValueOnce(new Error('temporary poll cleanup failure'));
await expect(scheduler.applyDurabilityPayload(payload)).rejects.toThrow(
'temporary poll cleanup failure',
);
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
'reserved',
);
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
__v: 0,
tournamentCodeIds: [f.codeId],
});
await scheduler.applyDurabilityPayload(payload);
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
'released',
);
expect(
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
).toBe('completed');
});
it('answers a retransmitted cancellation of the same lifecycle instead of 404', async () => {
const f = await fixture(true);
const request = {
matchId: f.matchId,
communityId: f.communityId,
clientObservedAuctionLifecycleId: 'life',
};
await f.service.reset(request);
await expect(f.service.reset(request)).resolves.toBeUndefined();
expect(
(await models.match.findById(f.matchId).lean())
?.lastCancelledAuctionLifecycleId,
).toBe('life');
await expect(
f.service.reset({ ...request, clientObservedAuctionLifecycleId: 'other' }),
).rejects.toThrow('Auction not found.');
});
it.each([
'new-lifecycle',
'new-transition',
'completed',
'revision-ahead',
'mirror-superseded',
])('fences terminal notifications against %s', async (scenario) => {
const f = await fixture(false);
await models.auction.updateOne(
{ _id: f.auctionId },
{
$set: {
matchSideEffectTransitionId: 'target',
matchSideEffectRevision: 1,
matchSideEffectStatus: 'pending',
},
},
);
const payload = {
auctionId: f.auctionId.toString(),
matchId: f.matchId.toString(),
auctionLifecycleId: 'life',
auctionStateTransitionId: 'target',
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
afterAuctionMongoProjection: { matchSideEffectRevision: 1 },
};
const state = {
auctionLifecycleId: scenario === 'new-lifecycle' ? 'new-life' : 'life',
matchSideEffectTransitionId:
scenario === 'new-transition' ? 'new' : 'target',
matchSideEffectStatus: scenario === 'completed' ? 'completed' : 'pending',
matchSideEffectRevision: 40,
};
const sendCommand = jest.fn(async (command: string[]) => {
if (command[0] === 'ZRANGEBYSCORE') return ['terminal'];
if (command[0] === 'HGETALL')
return { payloadJson: JSON.stringify(payload) };
if (command[0] === 'HGET') return 'exhausted';
return 1;
});
const broadcast = jest.fn();
const mirror = jest.fn().mockResolvedValue(scenario !== 'mirror-superseded');
const scheduler = new AuctionDurabilityScheduler(
f.database as never,
{
raw: () => ({ sendCommand }),
getCurrentRedisTimestampMS: async () => 1000,
} as never,
undefined,
undefined,
{ mirrorMatchSideEffectStatus: mirror } as never,
{ broadcastMatchSideEffect: broadcast } as never,
{ readAuctionStateView: async () => state } as never,
);
await scheduler.reconcileDeadLetterStatuses();
const changed = ['revision-ahead', 'mirror-superseded'].includes(scenario);
expect(
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
).toBe(changed ? 'failed' : 'pending');
if (changed)
expect(mirror).toHaveBeenCalledWith(
expect.objectContaining({ matchSideEffectRevision: 41 }),
);
expect(broadcast).toHaveBeenCalledTimes(
scenario === 'revision-ahead' ? 1 : 0,
);
});
@@ -0,0 +1,598 @@
import { createClient } from 'redis';
import mongoose from 'mongoose';
import { AuctionService } from '../src/modules/auction/auction.service';
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service';
import { AuctionRedisStateReaderService } from '../src/modules/auction/redis/auction-redis-state-reader.service';
import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys';
import {
AuctionLifecycleLock,
AuctionLifecycleBusyError,
} from '../src/modules/auction/auction-lifecycle-lock';
import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler';
import { AuctionSideEffectService } from '../src/modules/auction/auction-side-effect.service';
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
import {
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
AUCTION_BID_ARCHIVE_QUEUE_KEY,
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
} from '../src/modules/auction/redis/auction-redis.keys';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new AuctionRedisService(client);
const commands = new AuctionRedisCommandService(redis);
const reader = new AuctionRedisStateReaderService(redis);
beforeAll(async () => {
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) {
throw new Error('Use test:redis to start an isolated local Redis server.');
}
client.on('error', () => {});
await client.connect();
});
beforeEach(async () => {
await client.flushDb();
});
afterAll(async () => {
if (client.isOpen) await client.quit();
});
it.each(['count', 'elapsed'])(
'dead-letters permanently pending side effects by %s while retaining the payload',
async (limit) => {
const key = 'review:pending';
const now = Date.now();
await client.hSet(key, {
payloadJson: '{}',
pendingAttempts: limit === 'count' ? '119' : '1',
pendingSinceTimestampMS: String(limit === 'elapsed' ? now - 600000 : now),
});
await client.zAdd(AUCTION_DURABILITY_RETRY_QUEUE_KEY, {
score: now,
value: key,
});
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
deferDurabilityPayload(
key: string,
time: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.deferDurabilityPayload(
key,
now,
new Error('state unavailable'),
),
).toBe('dead-lettered');
expect(
await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key),
).toBeNull();
expect(
await client.zScore(AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, key),
).toBe(now);
expect(await client.hGet(key, 'payloadJson')).toBe('{}');
expect(await client.hGet(key, 'attempts')).toBeNull();
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
now,
);
},
);
it('defers a temporary coordination wait without consuming failed-work attempts or recreating removed payloads', async () => {
const key = 'review:pending';
const now = Date.now();
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
deferDurabilityPayload(
key: string,
time: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.deferDurabilityPayload(key, now, new Error('busy')),
).toBe('gone');
expect(await client.exists(key)).toBe(0);
await client.hSet(key, 'payloadJson', '{}');
expect(
await internal.deferDurabilityPayload(key, now, new Error('busy')),
).toBe('requeued');
expect(await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key)).toBe(
now + 5000,
);
expect(await client.hGet(key, 'attempts')).toBeNull();
});
async function fixture() {
const matchId = new mongoose.Types.ObjectId();
const id = matchId.toString();
const lifecycle = `life:${id}`;
const playerA = new mongoose.Types.ObjectId().toString();
const playerB = new mongoose.Types.ObjectId().toString();
const captain = new mongoose.Types.ObjectId().toString();
const initialRound = `initial:${id}`;
const structure = {
auctionPlayerOrder: [{ userId: playerA }, { userId: playerB }],
auctionTeams: [
{
teamIndex: 1,
captainUserId: captain,
remainingBidPointAmount: 100000,
members: [],
},
],
currentAuctionPlayerOrderIndex: 0,
currentAuctionRound: 1,
passedAuctionPlayerQueue: [],
maximumBidRoundDurationSeconds: 30,
bidRoundExtensionSeconds: 5,
initialTeamBidPointAmount: 100000,
topTierAuctionPriorityThreshold: 'NONE',
manualTopTierAuctionPriorityUserIds: [],
};
await commands.hydrateState(id, {
matchId: id,
auctionLifecycleId: lifecycle,
revision: 0,
auctionHistorySequence: 1,
phase: 'bidding',
maximumBidRoundDurationSeconds: 30,
structureStateJson: JSON.stringify(structure),
liveStatePatch: {
currentAuctionPlayerOrderIndex: 0,
currentBidRoundPlayerUserId: playerA,
currentBidRoundId: initialRound,
maximumBidRoundDurationSeconds: 30,
bidRoundExtensionSeconds: 5,
'team:1:captainUserId': captain,
'team:1:remainingBidPointAmount': 100000,
'team:1:assignedAuctionTeamMemberCount': 1,
},
});
const auction = {
_id: new mongoose.Types.ObjectId(),
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: lifecycle,
phase: 'bidding',
revision: 1,
};
const service = new AuctionService(
{ getModel: async () => ({ findOne: async () => auction }) } as never,
{} as never,
{} as never,
{} as never,
redis,
commands,
reader,
);
const bid = (
requestId: string,
amount: number,
player = playerA,
round = initialRound,
) => ({
matchId: id,
bidderTeamIndex: 1,
bidderUserId: captain,
bidPointAmount: amount,
bidRequestId: requestId,
clientObservedAuctionLifecycleId: lifecycle,
clientObservedCurrentBidRoundPlayerUserId: player,
clientObservedBidRoundId: round,
maximumAuctionTeamMemberCount: 5,
minimumBidPointReservePerRemainingAuctionTeamMemberSlot: 10,
maximumStoredAuctionBidHistoryCount: 1000,
maximumStoredBidRequestResultCount: 500,
minimumBidRequestResultReplayWindowMS: 300000,
});
const undo = async (snapshot: Record<string, unknown>) => {
const internal = service as unknown as {
undoRedisAuction(params: unknown): Promise<unknown>;
};
return internal.undoRedisAuction({
auction,
snapshot,
targetHistorySequence: 1,
undoingCompletedAuction: false,
});
};
return {
id,
matchId,
lifecycle,
playerA,
playerB,
initialRound,
auction,
service,
bid,
undo,
keys: buildAuctionRedisKeys(id),
};
}
it('waits briefly for a busy recovery lock and returns the recovered state', async () => {
const f = await fixture();
f.auction.revision = 10;
const key = `auction:${f.id}:lifecycleMutationLock`;
await redis.acquireTokenLock(key, 'other-reader', 30000);
const released = new Promise((resolve, reject) => {
setTimeout(() => {
void redis.releaseTokenLock(key, 'other-reader').then(resolve, reject);
}, 100);
});
const [view] = await Promise.all([
f.service.getStateByMatchId(f.matchId),
released,
]);
expect(view).toMatchObject({ auctionLifecycleId: f.lifecycle, revision: 11 });
});
it('bounds recovery waits and reports a retriable state error without releasing another owner lock', async () => {
const f = await fixture();
f.auction.revision = 10;
const key = `auction:${f.id}:lifecycleMutationLock`;
await redis.acquireTokenLock(key, 'slow-owner', 30000);
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toMatchObject({
response: { code: 'AUCTION_STATE_RECOVERING', retryAfterMS: 500 },
});
expect(await client.get(key)).toBe('slow-owner');
});
it('keeps accepting bids after 1000 history entries and preserves accepted request replay across players', async () => {
const f = await fixture();
const first = await commands.placeBid(
f.id,
f.bid('accepted-before-next', 10),
);
expect(first).toMatchObject({ ok: true });
await client.rPush(
f.keys.bidHistoryKey,
Array.from({ length: 999 }, () => JSON.stringify({ bidPointAmount: 10 })),
);
await client.hSet(f.keys.liveStateKey, 'phase', 'sold');
await f.service.next(f.matchId);
await f.service.openBidding(f.matchId);
const state = (await reader.readAuctionStateView(f.id))!;
const result = await commands.placeBid(
f.id,
f.bid('next-player', 10, f.playerB, String(state.currentBidRoundId)),
);
expect(result).toMatchObject({ ok: true });
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
expect(
await commands.placeBid(f.id, f.bid('accepted-before-next', 10)),
).toMatchObject({ ...first, isReplay: true });
});
it('500 rejected requests do not block a valid bid, and rejection retention stays bounded', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'currentHighestBidPointAmount', '10');
for (let i = 0; i < 501; i++) {
expect(
await commands.placeBid(f.id, f.bid(`too-low:${i}`, 5)),
).toMatchObject({ rejected: true, code: 'TOO_LOW' });
}
expect(await client.lLen(f.keys.rejectedBidRequestResultOrderKey)).toBe(500);
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(0);
expect(await commands.placeBid(f.id, f.bid('valid', 20))).toMatchObject({
ok: true,
});
expect(await commands.placeBid(f.id, f.bid('valid', 30))).toMatchObject({
rejected: true,
code: 'DUPLICATE_REQUEST_CONFLICT',
});
});
it('still protects 500 accepted results within their replay window', async () => {
const f = await fixture();
for (let i = 1; i <= 500; i++)
expect(
await commands.placeBid(f.id, f.bid(`accepted:${i}`, i)),
).toMatchObject({ ok: true });
expect(await commands.placeBid(f.id, f.bid('overflow', 501))).toMatchObject({
rejected: true,
code: 'BID_REQUEST_RESULT_LIMIT',
});
expect(await commands.placeBid(f.id, f.bid('accepted:1', 1))).toMatchObject({
ok: true,
isReplay: true,
});
});
it('migrates rejected results from the former shared quota without losing accepted replays', async () => {
const f = await fixture();
await commands.placeBid(f.id, f.bid('accepted', 10));
await client.hDel(f.keys.bidRequestResultsKey, '_acceptedOnlyOrderV1');
for (let i = 0; i < 499; i++) {
const id = `legacy-rejected:${i}`;
await client.hSet(f.keys.bidRequestResultsKey, {
[`${id}:status`]: 'rejected',
[`${id}:createdAtTimestampMS`]: String(Date.now()),
});
await client.rPush(f.keys.bidRequestResultOrderKey, id);
}
expect(
await commands.placeBid(f.id, f.bid('next-accepted', 20)),
).toMatchObject({ ok: true });
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(2);
expect(await commands.placeBid(f.id, f.bid('accepted', 10))).toMatchObject({
ok: true,
isReplay: true,
});
});
it('encodes opaque round identities safely in bid history', async () => {
const f = await fixture();
const round = 'round:with"quotes\\and-slashes';
await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', round);
expect(
await commands.placeBid(f.id, f.bid('quoted-round', 10, f.playerA, round)),
).toMatchObject({ ok: true, auctionBidHistoryEntry: { bidRoundId: round } });
expect((await reader.readAuctionStateView(f.id))?.auctionBidHistory).toEqual([
expect.objectContaining({ bidRoundId: round }),
]);
});
it('rejects delayed pre-undo bids and omitted round identities while accepting the restored round', async () => {
const f = await fixture();
const snapshot = (await reader.readAuctionStateView(f.id))!;
await f.undo(snapshot);
const restored = (await reader.readAuctionStateView(f.id))!;
expect(restored.currentBidRoundId).not.toBe(f.initialRound);
expect(await commands.placeBid(f.id, f.bid('old-round', 100))).toMatchObject({
code: 'STALE_BID_ROUND',
});
expect(
await commands.placeBid(f.id, {
...f.bid('missing-round', 100),
clientObservedBidRoundId: undefined,
}),
).toMatchObject({ code: 'BID_CLIENT_UPDATE_REQUIRED' });
expect(
await commands.placeBid(
f.id,
f.bid('new-round', 100, f.playerA, String(restored.currentBidRoundId)),
),
).toMatchObject({
ok: true,
auctionBidHistoryEntry: {
playerUserId: f.playerA,
bidRoundId: restored.currentBidRoundId,
},
});
});
it('removes completion TTLs when resuming an auction', async () => {
const f = await fixture();
const snapshot = (await reader.readAuctionStateView(f.id))!;
await client.expire(f.keys.liveStateKey, 604800);
await client.expire(f.keys.structureStateKey, 604800);
await f.undo(snapshot);
expect(await client.ttl(f.keys.liveStateKey)).toBe(-1);
expect(await client.ttl(f.keys.structureStateKey)).toBe(-1);
});
it('fences direct foreign-lifecycle hydration and repairs an orphan using current Mongo authority', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'revision', '150');
expect(
await commands.hydrateState(f.id, {
matchId: f.id,
auctionLifecycleId: 'replacement',
revision: 0,
auctionHistorySequence: 1,
phase: 'presenting',
maximumBidRoundDurationSeconds: 30,
structureStateJson: '{}',
}),
).toMatchObject({ rejected: true, code: 'LIFECYCLE_CONFLICT' });
const current = {
...f.auction,
auctionLifecycleId: 'replacement',
phase: 'presenting',
revision: 1,
auctionTeams: [],
auctionPlayerOrder: [],
};
jest.spyOn(f.service, 'getByMatchId').mockResolvedValue(current as never);
expect(await f.service.getStateByMatchId(f.matchId)).toMatchObject({
auctionLifecycleId: 'replacement',
phase: 'presenting',
});
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
'replacement',
);
});
it('does not let an old Mongo read overwrite the newly started lifecycle', async () => {
const f = await fixture();
const stale = { ...f.auction, auctionLifecycleId: 'obsolete', revision: 500 };
jest
.spyOn(f.service, 'getByMatchId')
.mockResolvedValueOnce(stale as never)
.mockResolvedValue(f.auction as never);
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toThrow(
'lifecycle has changed',
);
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
f.lifecycle,
);
});
it('serializes lifecycle mutations without deleting another owner lock', async () => {
const lock = new AuctionLifecycleLock(redis);
await lock.run('match', async () => {
await expect(lock.run('match', async () => {})).rejects.toThrow(
AuctionLifecycleBusyError,
);
});
await expect(lock.run('match', async () => 'done')).resolves.toBe('done');
});
it('persists the exact Redis deadline when opening bidding and restoring a bidding snapshot', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, 'phase', 'presenting');
await client.hDel(f.keys.liveStateKey, 'currentBidRoundDeadlineTimestampMS');
await f.service.openBidding(f.matchId);
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const verify = async () => {
const live = (await reader.readAuctionStateView(f.id))!;
const queued = await client.zRange(
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
0,
-1,
);
const payloads = await Promise.all(
queued.map((key) => scheduler.readDurabilityPayload(key)),
);
const payload = payloads.find(
(entry) => entry?.revision === live.revision,
)!;
expect(
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS,
).toBe(live.currentBidRoundDeadlineTimestampMS);
expect(
new Date(
String(payload.afterAuctionMongoProjection.currentDeadline),
).getTime(),
).toBe(live.currentBidRoundDeadlineTimestampMS);
expect(
payload.afterAuctionMongoProjection.passedAuctionPlayerQueue,
).toEqual([]);
return live;
};
const snapshot = await verify();
await f.undo({ ...snapshot, currentBidRoundDeadlineTimestampMS: 1 });
await verify();
});
it('archives every accepted bid beyond the recent 1000, with no duplicate archive event for a replay', async () => {
const f = await fixture();
for (let amount = 1; amount <= 1005; amount++) {
expect(
await commands.placeBid(f.id, {
...f.bid(`archive:${amount}`, amount),
maximumStoredBidRequestResultCount: 2000,
}),
).toMatchObject({ ok: true });
}
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
expect(
await commands.placeBid(f.id, {
...f.bid('archive:1', 1),
maximumStoredBidRequestResultCount: 2000,
}),
).toMatchObject({ ok: true, isReplay: true });
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
const [first] = await client.zRange(AUCTION_BID_ARCHIVE_QUEUE_KEY, 0, 0);
expect(
JSON.parse((await client.hGet(AUCTION_BID_ARCHIVE_PAYLOADS_KEY, first))!),
).toMatchObject({
bidPointAmount: 1,
bidRequestId: 'archive:1',
playerUserId: f.playerA,
auctionLifecycleId: f.lifecycle,
});
});
it.each(['missing', 'empty'])(
'archives an accepted pre-upgrade bid with a %s Redis round ID',
async (kind) => {
const f = await fixture();
if (kind === 'missing')
await client.hDel(f.keys.liveStateKey, 'currentBidRoundId');
else await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', '');
expect(
await commands.placeBid(f.id, {
...f.bid('legacy-round', 10),
clientObservedBidRoundId: undefined,
}),
).toMatchObject({
ok: true,
auctionBidHistoryEntry: { bidRoundId: '', playerUserId: f.playerA },
});
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
matchedCount: 0,
upsertedCount: operations.length,
}));
const archive = new AuctionBidArchiveService(
{ getModel: async () => ({ bulkWrite }) } as never,
redis,
);
await expect(archive.persistPendingBids()).resolves.toBe(1);
const write = bulkWrite.mock.calls[0][0][0] as {
updateOne: { update: { $setOnInsert: Record<string, unknown> } };
};
expect(write.updateOne.update.$setOnInsert).not.toHaveProperty(
'bidRoundId',
);
expect(String(write.updateOne.update.$setOnInsert.playerUserId)).toBe(
f.playerA,
);
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(0);
},
);
it('does not mirror an old generation failure over a newer Redis side effect', async () => {
const f = await fixture();
await client.hSet(f.keys.liveStateKey, {
matchSideEffectTransitionId: 'new',
matchSideEffectRevision: '2',
matchSideEffectStatus: 'pending',
});
const sideEffects = new AuctionSideEffectService(redis);
expect(
await sideEffects.mirrorMatchSideEffectStatus({
matchId: f.id,
auctionLifecycleId: f.lifecycle,
auctionStateTransitionId: 'old',
matchSideEffectRevision: 100,
matchSideEffectStatus: 'failed',
}),
).toBe(false);
expect(await client.hGet(f.keys.liveStateKey, 'matchSideEffectStatus')).toBe(
'pending',
);
});
it('queues a terminal notification for ordinary failures and backfills old dead letters', async () => {
const key = 'review:ordinary-failure';
await client.hSet(key, { payloadJson: '{}', attempts: '9' });
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
const internal = scheduler as unknown as {
recordDurabilityPayloadFailure(
key: string,
now: number,
error: unknown,
): Promise<string>;
};
expect(
await internal.recordDurabilityPayloadFailure(
key,
1000,
new Error('failed before worker'),
),
).toBe('dead-lettered');
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
1000,
);
// Simulate an older deployment which did not create notification jobs.
await client.zRem(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key);
await scheduler.reconcileDeadLetterStatuses();
expect(
await client.hGet(key, 'deadLetterStatusReconciledAtTimestampMS'),
).not.toBeNull();
expect(
await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key),
).toBeNull();
});
+7
View File
@@ -0,0 +1,7 @@
{
"rootDir": "..",
"testEnvironment": "node",
"testRegex": "test/.*\\.mongo-spec\\.ts$",
"transform": { "^.+\\.ts$": "ts-jest" },
"testTimeout": 15000
}
+134
View File
@@ -0,0 +1,134 @@
import { spawn } from 'node:child_process';
import { createRequire } from 'node:module';
import { mkdtemp, mkdir, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, relative, resolve } from 'node:path';
import { createServer } from 'node:net';
import { setTimeout as delay } from 'node:timers/promises';
import mongoose from 'mongoose';
const require = createRequire(import.meta.url);
const listener = createServer();
await new Promise((accept) => listener.listen(0, '127.0.0.1', accept));
const port = listener.address().port;
await new Promise((accept) => listener.close(accept));
const directory = await mkdtemp(join(tmpdir(), 'lolympus-mongo-test-'));
await mkdir(join(directory, 'data'));
const mongo = spawn(
process.env.MONGODB_TEST_SERVER || 'mongod',
[
'--bind_ip',
'127.0.0.1',
'--port',
String(port),
'--replSet',
'auction_review',
'--dbpath',
join(directory, 'data'),
'--logpath',
join(directory, 'mongod.log'),
'--wiredTigerCacheSizeGB',
'0.25',
],
{ windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] },
);
let startupError;
mongo.on('error', (error) => {
startupError = error;
});
const closed = new Promise((accept) => mongo.once('close', accept));
let output = '';
mongo.stdout.on('data', (chunk) => {
output += chunk;
});
mongo.stderr.on('data', (chunk) => {
output += chunk;
});
const uri = `mongodb://127.0.0.1:${port}/auction_review`;
const client = new mongoose.mongo.MongoClient(`${uri}?directConnection=true`, {
serverSelectionTimeoutMS: 500,
});
try {
let connected = false;
for (let attempt = 0; attempt < 60; attempt++) {
if (startupError) throw startupError;
if (mongo.exitCode !== null) throw new Error(`MongoDB exited: ${output}`);
try {
await client.connect();
connected = true;
break;
} catch {
await delay(100);
}
}
if (!connected) throw new Error(`MongoDB did not start: ${output}`);
await client
.db('admin')
.command({
replSetInitiate: {
_id: 'auction_review',
members: [{ _id: 0, host: `127.0.0.1:${port}` }],
},
});
let primary = false;
for (let attempt = 0; attempt < 100; attempt++) {
if ((await client.db('admin').command({ hello: 1 })).isWritablePrimary) {
primary = true;
break;
}
await delay(100);
}
if (!primary)
throw new Error('MongoDB replica set did not elect its primary.');
const build = await client.db('admin').command({ buildInfo: 1 });
console.log(
`Integration tests using isolated MongoDB ${build.version}, Mongoose ${mongoose.version}, loopback port ${port}`,
);
const jest = spawn(
process.execPath,
[
require.resolve('jest/bin/jest'),
'--config',
'test/jest-mongo.json',
'--runInBand',
...process.argv.slice(2),
],
{
cwd: resolve(import.meta.dirname, '..'),
windowsHide: true,
stdio: 'inherit',
env: {
...process.env,
NODE_ENV: 'test',
MONGO_TEST_ISOLATED: '1',
MONGO_TEST_URL: `${uri}?replicaSet=auction_review`,
},
},
);
process.exitCode = await new Promise((accept, reject) => {
jest.once('error', reject);
jest.once('exit', (code) => accept(code ?? 1));
});
} finally {
await client
.db('admin')
.command({ shutdown: 1, force: true })
.catch(() => {});
await client.close().catch(() => {});
if (mongo.pid && mongo.exitCode === null) mongo.kill();
await closed;
// Only remove the unique temporary directory created by this runner.
const target = relative(resolve(tmpdir()), resolve(directory));
if (
!target.startsWith('lolympus-mongo-test-') ||
target.includes('..') ||
/[/\\]/.test(target)
)
throw new Error('Unsafe MongoDB test directory.');
await rm(directory, {
recursive: true,
force: true,
maxRetries: 10,
retryDelay: 100,
});
}
+83
View File
@@ -0,0 +1,83 @@
import type { ApiAuction } from './types';
export interface AuctionResyncReply {
ok?: boolean;
state?: ApiAuction;
retryAfterMS?: number;
}
function waitForRetry(
milliseconds: number,
signal: AbortSignal,
): Promise<void> {
return new Promise((resolve) => {
const finish = () => {
clearTimeout(timer);
signal.removeEventListener('abort', finish);
resolve();
};
const timer = setTimeout(finish, milliseconds);
signal.addEventListener('abort', finish, { once: true });
if (signal.aborted) finish();
});
}
/** Coalesces simultaneous gaps and discards replies from an old connection. */
export function createAuctionResyncRequester(
fetchState: () => Promise<AuctionResyncReply>,
applyState: (state: ApiAuction) => void,
wait = waitForRetry,
) {
let pending: Promise<void> | undefined;
let controller = new AbortController();
let minimumRevision = -1;
let lifecycleId: string | undefined;
return {
request(revision = -1, lifecycle?: string): Promise<void> {
if (revision >= 0) {
minimumRevision =
lifecycle === lifecycleId
? Math.max(minimumRevision, revision)
: revision;
lifecycleId = lifecycle;
}
if (pending) return pending;
const signal = controller.signal;
pending = (async () => {
for (let attempt = 0; attempt < 3 && !signal.aborted; attempt++) {
let retryAfterMS = 500 * (attempt + 1);
try {
const reply = await fetchState();
if (signal.aborted) return;
if (reply.ok && reply.state) {
applyState(reply.state);
// Another gap may have arrived while this snapshot was in flight.
if (
reply.state.auctionLifecycleId !== lifecycleId ||
(reply.state.revision ?? -1) >= minimumRevision
)
return;
}
retryAfterMS = Math.min(
3000,
Math.max(retryAfterMS, reply.retryAfterMS ?? 0),
);
} catch {
/* A timeout gets a bounded retry, never another room join. */
}
if (attempt < 2 && !signal.aborted) await wait(retryAfterMS, signal);
}
})().finally(() => {
if (!signal.aborted) pending = undefined;
});
return pending;
},
cancel() {
controller.abort();
controller = new AbortController();
pending = undefined;
minimumRevision = -1;
lifecycleId = undefined;
},
};
}
+239
View File
@@ -0,0 +1,239 @@
import type { ApiAuction } from "./types";
export interface BidDeltaPayload {
currentBidRoundId?: string;
auctionBidHistoryEntry?: {
bidderTeamIndex?: number;
bidPointAmount?: number;
bidderUserId?: string;
bidPlacedAtTimestampMS?: number;
bidRoundId?: string;
playerUserId?: string;
};
auctionLifecycleId?: string;
bidRequestId?: string;
currentBidRoundDeadlineTimestampMS?: number;
currentHighestBidderTeamIndex?: number;
currentHighestBidPointAmount?: number;
currentServerTimestampMS?: number;
revision?: number;
}
export function shouldApplyIncomingAuctionState(
current: ApiAuction | null,
incoming: ApiAuction,
): boolean {
if (!current) return true;
if (incoming.shouldForceAuctionStateResync) return true;
if (
current.auctionLifecycleId &&
incoming.auctionLifecycleId &&
current.auctionLifecycleId !== incoming.auctionLifecycleId
) {
return true;
}
if (
typeof current.revision === "number" &&
typeof incoming.revision === "number" &&
incoming.revision < current.revision
) {
return false;
}
return true;
}
/** Full state can repair a delta at the same revision; side effects have a
* separate revision and must not regress when a delayed full state arrives. */
export function mergeAuctionState(
current: ApiAuction | null,
incoming: ApiAuction,
): ApiAuction {
if (
current?.auctionLifecycleId === incoming.auctionLifecycleId &&
current?.matchSideEffectTransitionId ===
incoming.matchSideEffectTransitionId &&
(current?.matchSideEffectRevision ?? -1) >
(incoming.matchSideEffectRevision ?? -1)
) {
return {
...incoming,
matchSideEffectRevision: current!.matchSideEffectRevision,
matchSideEffectStatus: current!.matchSideEffectStatus,
};
}
return incoming;
}
export function shouldResyncForBidDelta(
current: ApiAuction | null,
payload: BidDeltaPayload,
): boolean {
if (!current) return true;
if (
payload.auctionLifecycleId &&
current.auctionLifecycleId &&
payload.auctionLifecycleId !== current.auctionLifecycleId
)
return true;
if (
typeof payload.revision !== "number" ||
typeof current.revision !== "number"
)
return true;
if (payload.revision <= current.revision) return false;
return (
payload.revision !== current.revision + 1 ||
current.phase !== "bidding" ||
Boolean(
current.currentBidRoundId &&
payload.currentBidRoundId !== current.currentBidRoundId,
)
);
}
export function applyBidDelta(
current: ApiAuction | null,
payload: BidDeltaPayload,
): ApiAuction | null {
if (!current) return current;
if (shouldResyncForBidDelta(current, payload)) return current;
if (
payload.auctionLifecycleId &&
current.auctionLifecycleId &&
payload.auctionLifecycleId !== current.auctionLifecycleId
) {
return current;
}
if (
typeof current.revision === "number" &&
typeof payload.revision === "number" &&
payload.revision <= current.revision
) {
return current;
}
const bidPointAmount = Number(payload.currentHighestBidPointAmount);
const bidderTeamIndex = Number(payload.currentHighestBidderTeamIndex);
if (!Number.isFinite(bidPointAmount) || !Number.isFinite(bidderTeamIndex)) {
return current;
}
const bidHistoryEntry = normalizeBidHistoryEntry(
payload.auctionBidHistoryEntry,
);
const currentBidRoundDeadlineTimestampMS =
typeof payload.currentBidRoundDeadlineTimestampMS === "number"
? payload.currentBidRoundDeadlineTimestampMS
: current.currentBidRoundDeadlineTimestampMS;
return {
...current,
auctionBidHistory: appendUniqueAuctionBidHistoryEntry(
current.auctionBidHistory ?? [],
bidHistoryEntry,
),
bids: bidHistoryEntry
? appendUniqueLegacyBidEntry(current.bids ?? [], bidHistoryEntry)
: current.bids,
currentBid: bidPointAmount,
currentBidderTeamIndex: bidderTeamIndex,
currentDeadline: currentBidRoundDeadlineTimestampMS
? new Date(currentBidRoundDeadlineTimestampMS).toISOString()
: current.currentDeadline,
currentHighestBidPointAmount: bidPointAmount,
currentHighestBidderTeamIndex: bidderTeamIndex,
currentBidRoundDeadlineTimestampMS,
currentServerTimestampMS:
payload.currentServerTimestampMS ?? current.currentServerTimestampMS,
revision: payload.revision ?? current.revision,
currentBidRoundId: payload.currentBidRoundId ?? current.currentBidRoundId,
};
}
function normalizeBidHistoryEntry(
entry: BidDeltaPayload["auctionBidHistoryEntry"],
): NonNullable<ApiAuction["auctionBidHistory"]>[number] | null {
if (!entry) return null;
const bidderTeamIndex = Number(entry.bidderTeamIndex);
const bidPointAmount = Number(entry.bidPointAmount);
const bidPlacedAtTimestampMS = Number(entry.bidPlacedAtTimestampMS);
if (!Number.isFinite(bidderTeamIndex) || !Number.isFinite(bidPointAmount)) {
return null;
}
return {
bidRoundId: entry.bidRoundId,
playerUserId: entry.playerUserId,
bidderTeamIndex,
bidPointAmount,
bidderUserId: entry.bidderUserId ?? "",
bidPlacedAtTimestampMS: Number.isFinite(bidPlacedAtTimestampMS)
? bidPlacedAtTimestampMS
: Date.now(),
};
}
function appendUniqueAuctionBidHistoryEntry(
current: NonNullable<ApiAuction["auctionBidHistory"]>,
entry: NonNullable<ApiAuction["auctionBidHistory"]>[number] | null,
) {
if (!entry) return current;
if (
current.some(
(existing) =>
existing.bidderTeamIndex === entry.bidderTeamIndex &&
existing.bidPointAmount === entry.bidPointAmount &&
existing.bidPlacedAtTimestampMS === entry.bidPlacedAtTimestampMS,
)
) {
return current;
}
return [...current, entry].slice(-1000);
}
function appendUniqueLegacyBidEntry(
current: ApiAuction["bids"],
entry: NonNullable<ApiAuction["auctionBidHistory"]>[number],
) {
const currentBids = current ?? [];
const legacyBid = {
amount: entry.bidPointAmount,
bidderUserId: entry.bidderUserId,
teamIndex: entry.bidderTeamIndex,
timestamp: new Date(entry.bidPlacedAtTimestampMS).toISOString(),
};
if (
currentBids.some(
(existing) =>
existing.teamIndex === legacyBid.teamIndex &&
existing.amount === legacyBid.amount &&
existing.timestamp === legacyBid.timestamp,
)
) {
return currentBids;
}
return [...currentBids, legacyBid].slice(-1000);
}
/** HTTP 폴링 결과를 소켓 내부 상태(입찰 기준가·라운드 ID)에도 반영할지 결정한다.
* lifecycle 교체는 소켓 이벤트로만 받는다. 늦게 도착한 이전 경매의 응답이
* 새 경매를 덮어쓰지 않도록 기존 상태가 있으면 같은 lifecycle만 반영한다. */
export function shouldApplyPolledAuctionState(
current: ApiAuction | null,
incoming: ApiAuction,
matchId: string | undefined,
): boolean {
if (!matchId || incoming.matchId !== matchId) return false;
if (!incoming.auctionLifecycleId || !Number.isFinite(incoming.revision))
return false;
if (!current) return true;
if (
!current.auctionLifecycleId ||
current.auctionLifecycleId !== incoming.auctionLifecycleId
)
return false;
return (
typeof current.revision === "number" &&
typeof incoming.revision === "number" &&
incoming.revision >= current.revision
);
}
+7 -2
View File
@@ -28,7 +28,10 @@ const REASON_TO_USER_MESSAGE: Record<string, string> = {
AUCTION_TRANSITION_REJECTED_BECAUSE_REQUEST_COORDINATION_RETRYABLE:
'잠시 처리 중이라 반영되지 않았어요. 다시 시도해 주세요.',
// 입찰
BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED: '이미 다음 선수로 넘어갔어요.',
BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED:
'경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.',
BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED:
'이미 다음 선수로 넘어갔어요.',
BID_REJECTED_BECAUSE_BID_ROUND_DEADLINE_PASSED: '마감되어 반영되지 않았어요.',
BID_REJECTED_BECAUSE_AUCTION_NOT_IN_BIDDING_PHASE:
'지금은 입찰할 수 없는 단계예요.',
@@ -100,6 +103,8 @@ export function resolveAuctionErrorMessage(message?: string): string {
}
/** 최신 상태로 rebaseline 후 재시도가 권장되는 거부인지. */
export function isAuctionResyncRecommended(ack: AuctionRejectionAckLike): boolean {
export function isAuctionResyncRecommended(
ack: AuctionRejectionAckLike,
): boolean {
return ack.resyncRecommended === true;
}
+4
View File
@@ -197,6 +197,8 @@ export interface ApiAuction {
currentAuctionPlayerOrderIndex: number;
bids?: ApiAuctionBid[];
auctionBidHistory?: Array<{
bidRoundId?: string;
playerUserId?: string;
bidderTeamIndex: number;
bidPointAmount: number;
bidderUserId: string;
@@ -213,6 +215,7 @@ export interface ApiAuction {
currentDeadline?: string;
currentBidRoundDeadlineTimestampMS?: number;
currentBidRoundPlayerUserId?: string;
currentBidRoundId?: string;
currentServerTimestampMS?: number;
initialPoints?: number;
initialTeamBidPointAmount: number;
@@ -224,6 +227,7 @@ export interface ApiAuction {
auctionLifecycleId?: string;
matchSideEffectRevision?: number;
matchSideEffectStatus?: 'none' | 'pending' | 'processing' | 'completed' | 'failed';
matchSideEffectTransitionId?: string;
canPlaceBidInCurrentBidRound?: boolean;
isAuctionStateTransitionInProgress?: boolean;
shouldForceAuctionStateResync?: boolean;
+174 -211
View File
@@ -3,6 +3,18 @@ import { io, type Socket } from 'socket.io-client';
import { API_BASE_URL, useAuthUserId } from '../config';
import { acquireAccessToken } from '../lib/auth/token-provider';
import type { ApiAuction, ApiAuctionChatMessage, Position } from './types';
import {
createAuctionResyncRequester,
type AuctionResyncReply,
} from './auction-resync';
import {
applyBidDelta,
shouldApplyIncomingAuctionState,
shouldResyncForBidDelta,
shouldApplyPolledAuctionState,
mergeAuctionState,
type BidDeltaPayload,
} from './auction-state';
import { resolveAuctionActionRejection } from './auction-action-retry';
import {
isAuctionResyncRecommended,
@@ -21,6 +33,7 @@ type ConnectionStatus =
| 'error';
interface BidPayload {
clientObservedBidRoundId?: string;
bidderTeamIndex: number;
bidPointAmount: number;
bidRequestId?: string;
@@ -52,33 +65,19 @@ interface ChatHistoryPayload {
interface ResetPayload {
matchId?: string;
auctionLifecycleId?: string;
}
type MatchSideEffectStatus = NonNullable<ApiAuction['matchSideEffectStatus']>;
interface MatchSideEffectPayload {
auctionLifecycleId?: string;
auctionStateTransitionId?: string;
currentServerTimestampMS?: number;
matchSideEffectRevision?: number;
matchSideEffectStatus?: MatchSideEffectStatus;
}
interface BidDeltaPayload {
auctionBidHistoryEntry?: {
bidderTeamIndex?: number;
bidPointAmount?: number;
bidderUserId?: string;
bidPlacedAtTimestampMS?: number;
};
auctionLifecycleId?: string;
bidRequestId?: string;
currentBidRoundDeadlineTimestampMS?: number;
currentHighestBidderTeamIndex?: number;
currentHighestBidPointAmount?: number;
currentServerTimestampMS?: number;
revision?: number;
}
interface BidServerTimingPayload {
fallbackFullStatePath?: boolean;
gatewayTotalDurationMS?: number;
@@ -145,6 +144,9 @@ export function useAuctionSocket(matchId: string | undefined) {
? JSON.stringify([matchId, authUserId ?? null])
: null;
const socketRef = useRef<Socket | null>(null);
const resyncRequesterRef = useRef<ReturnType<
typeof createAuctionResyncRequester
> | null>(null);
const pendingBidTimeoutsRef = useRef<
Record<string, ReturnType<typeof setTimeout>>
>({});
@@ -160,6 +162,7 @@ export function useAuctionSocket(matchId: string | undefined) {
const timeSyncSequenceRef = useRef(0);
const hasSyncedServerTimeRef = useRef(false);
const latestAuctionRef = useRef<ApiAuction | null>(null);
const cancelledLifecycleIdsRef = useRef(new Set<string>());
const latestConfirmedBidPointAmountRef = useRef(0);
const [auction, setAuction] = useState<ApiAuction | null>(null);
const [chatMessages, setChatMessages] = useState<ApiAuctionChatMessage[]>([]);
@@ -187,6 +190,11 @@ export function useAuctionSocket(matchId: string | undefined) {
const commitAuctionState = useCallback(
(nextAuction: ApiAuction, source = 'socket-state'): boolean => {
if (
nextAuction.auctionLifecycleId &&
cancelledLifecycleIdsRef.current.has(nextAuction.auctionLifecycleId)
)
return false;
const currentAuction = latestAuctionRef.current;
const applied = shouldApplyIncomingAuctionState(
currentAuction,
@@ -198,13 +206,12 @@ export function useAuctionSocket(matchId: string | undefined) {
: -1;
const incomingRevision =
typeof nextAuction?.revision === 'number' ? nextAuction.revision : -1;
// 단조성을 깨고 적용되는 경우(=revision이 같거나 뒤로 가는데도 반영됨).
// force-resync/lifecycle 교체로만 발생하며, "2~3 버전 롤백"의 직접 증거가 된다.
// 같은 revision의 전체 상태는 delta 누락 복구이므로 역행으로 기록하지 않는다.
const isBackwardApply =
applied &&
currentRevision >= 0 &&
incomingRevision >= 0 &&
incomingRevision <= currentRevision;
incomingRevision < currentRevision;
addNewRelicPageAction('AuctionClientStateApply', {
matchId: matchId ?? 'none',
source,
@@ -219,6 +226,7 @@ export function useAuctionSocket(matchId: string | undefined) {
return false;
}
nextAuction = mergeAuctionState(currentAuction, nextAuction);
latestAuctionRef.current = nextAuction;
latestConfirmedBidPointAmountRef.current =
resolveCurrentBidPointAmount(nextAuction);
@@ -230,6 +238,13 @@ export function useAuctionSocket(matchId: string | undefined) {
const commitBidDelta = useCallback((payload: BidDeltaPayload): boolean => {
const currentAuction = latestAuctionRef.current;
if (shouldResyncForBidDelta(currentAuction, payload)) {
void resyncRequesterRef.current?.request(
payload.revision,
payload.auctionLifecycleId,
);
return false;
}
const nextAuction = applyBidDelta(currentAuction, payload);
if (!nextAuction || nextAuction === currentAuction) {
return false;
@@ -242,6 +257,29 @@ export function useAuctionSocket(matchId: string | undefined) {
return true;
}, []);
// HTTP 폴링은 화면만 갱신하므로, 소켓이 전이를 놓친 경우 입찰 기준가와
// 라운드 ID가 낡은 채 남는다. 같은 경매의 더 새 상태면 소켓 상태에도 반영한다.
const syncPolledAuctionState = useCallback(
(polledAuction: ApiAuction | null | undefined) => {
if (!polledAuction) return;
if (
polledAuction.auctionLifecycleId &&
cancelledLifecycleIdsRef.current.has(polledAuction.auctionLifecycleId)
)
return;
if (
!shouldApplyPolledAuctionState(
latestAuctionRef.current,
polledAuction,
matchId,
)
)
return;
commitAuctionState(polledAuction, 'http-poll');
},
[commitAuctionState, matchId],
);
const setError = useCallback(
(message: string | null) => {
setErrorState({ key: connectionKey ?? '', message });
@@ -336,7 +374,10 @@ export function useAuctionSocket(matchId: string | undefined) {
: undefined);
const clientObservedRemainingBidRoundTimeMS =
typeof currentDeadlineTimestampMS === 'number'
? Math.max(0, currentDeadlineTimestampMS - (Date.now() + serverTimeOffsetMs))
? Math.max(
0,
currentDeadlineTimestampMS - (Date.now() + serverTimeOffsetMs),
)
: undefined;
const serverDeadlineTimestampMS =
ack.bidDiagnostics?.currentBidRoundDeadlineTimestampMS ??
@@ -399,6 +440,7 @@ export function useAuctionSocket(matchId: string | undefined) {
}
const activeConnectionKey = connectionKey;
const cancelledLifecycleIds = cancelledLifecycleIdsRef.current;
let disposed = false;
void (async () => {
@@ -418,6 +460,33 @@ export function useAuctionSocket(matchId: string | undefined) {
});
socketRef.current = socket;
const receiveAuctionState = (nextAuction: ApiAuction, source: string) => {
const previousRevision = latestAuctionRef.current?.revision ?? -1;
const didApplyState = commitAuctionState(nextAuction, source);
if (didApplyState && (nextAuction.revision ?? 0) > previousRevision) {
clearPendingActionRequests();
}
applyFallbackServerTimestamp(nextAuction.currentServerTimestampMS);
if (nextAuction.chatMessages) {
setChatMessages(mergeChatMessages([], nextAuction.chatMessages));
}
if (didApplyState) {
setError(null);
}
};
const resyncRequester = createAuctionResyncRequester(
async () => {
if (!socket.connected)
throw new Error('Auction socket is disconnected.');
return (await socket
.timeout(3000)
.emitWithAck('resync', {})) as AuctionResyncReply;
},
(state) => {
receiveAuctionState(state, 'socket-resync');
},
);
resyncRequesterRef.current = resyncRequester;
socket.on('connect', () => {
setConnectionState({
@@ -437,6 +506,7 @@ export function useAuctionSocket(matchId: string | undefined) {
});
socket.on('disconnect', () => {
resyncRequester.cancel();
clearTimeSyncTimers();
setConnectionState({
key: activeConnectionKey,
@@ -444,18 +514,12 @@ export function useAuctionSocket(matchId: string | undefined) {
});
});
socket.on('state-pending', () => {
void resyncRequester.request();
});
socket.on('state', (nextAuction: ApiAuction) => {
const didApplyState = commitAuctionState(nextAuction, 'socket-state');
if (didApplyState) {
clearPendingActionRequests();
}
applyFallbackServerTimestamp(nextAuction.currentServerTimestampMS);
if (nextAuction.chatMessages) {
setChatMessages(mergeChatMessages([], nextAuction.chatMessages));
}
if (didApplyState) {
setError(null);
}
receiveAuctionState(nextAuction, 'socket-state');
});
socket.on('chat-history', (payload: ChatHistoryPayload) => {
@@ -509,6 +573,13 @@ export function useAuctionSocket(matchId: string | undefined) {
const currentAuction = latestAuctionRef.current;
if (!currentAuction) return;
if (
payload.auctionStateTransitionId &&
currentAuction.matchSideEffectTransitionId &&
payload.auctionStateTransitionId !==
currentAuction.matchSideEffectTransitionId
)
return;
if (
payload.auctionLifecycleId &&
currentAuction.auctionLifecycleId &&
@@ -540,6 +611,18 @@ export function useAuctionSocket(matchId: string | undefined) {
socket.on('reset', (payload: ResetPayload) => {
if (payload.matchId && payload.matchId !== matchId) return;
const lifecycle =
payload.auctionLifecycleId ??
latestAuctionRef.current?.auctionLifecycleId;
if (lifecycle) cancelledLifecycleIdsRef.current.add(lifecycle);
if (
payload.auctionLifecycleId &&
latestAuctionRef.current?.auctionLifecycleId &&
payload.auctionLifecycleId !==
latestAuctionRef.current.auctionLifecycleId
)
return;
resyncRequester.cancel();
latestAuctionRef.current = null;
latestConfirmedBidPointAmountRef.current = 0;
setAuction(null);
@@ -556,6 +639,8 @@ export function useAuctionSocket(matchId: string | undefined) {
return () => {
disposed = true;
resyncRequesterRef.current?.cancel();
resyncRequesterRef.current = null;
const socket = socketRef.current;
socket?.removeAllListeners();
socket?.disconnect();
@@ -570,6 +655,7 @@ export function useAuctionSocket(matchId: string | undefined) {
pendingBidTimeoutsRef.current = {};
pendingActionTimeoutsRef.current = {};
latestAuctionRef.current = null;
cancelledLifecycleIds.clear();
latestConfirmedBidPointAmountRef.current = 0;
hasSyncedServerTimeRef.current = false;
setPendingBidRequests({});
@@ -621,12 +707,11 @@ export function useAuctionSocket(matchId: string | undefined) {
}
// 거부 등으로 클라 상태가 낡았을 때 서버 최신 state를 다시 받아 rebaseline한다.
// join 핸들러가 getStateByMatchId → broadcastState로 fresh state를 내려주고,
// revision 가드상 더 높은 revision이라 정상 적용된다(force-resync 아님).
// 요청한 소켓만 최신 상태를 받으며 진행 중인 동기화 요청은 합친다.
function resyncAuctionState() {
const socket = socketRef.current;
if (!socket || !socket.connected || !matchId) return;
socket.emit('join', { matchId });
void resyncRequesterRef.current?.request();
}
// retryAttempt: 거부 후 자동 재시도 횟수(루프 방지를 위해 1회로 제한).
@@ -655,10 +740,13 @@ export function useAuctionSocket(matchId: string | undefined) {
...current,
[auctionActionRequestId]: { action: eventName },
}));
pendingActionTimeoutsRef.current[auctionActionRequestId] = setTimeout(() => {
clearPendingActionRequest(auctionActionRequestId);
setError('경매 진행 응답이 지연되어 최신 상태를 다시 확인해주세요.');
}, 10_000);
pendingActionTimeoutsRef.current[auctionActionRequestId] = setTimeout(
() => {
clearPendingActionRequest(auctionActionRequestId);
setError('경매 진행 응답이 지연되어 최신 상태를 다시 확인해주세요.');
},
10_000,
);
addNewRelicPageAction('AuctionClientActionSent', {
matchId: matchId ?? 'none',
action: eventName,
@@ -731,7 +819,10 @@ export function useAuctionSocket(matchId: string | undefined) {
sentPlayerUserId:
payload.clientObservedCurrentBidRoundPlayerUserId ?? 'none',
sentPhase: payload.clientObservedPhase ?? 'none',
...describeAuctionForTelemetry('observed', latestAuctionRef.current),
...describeAuctionForTelemetry(
'observed',
latestAuctionRef.current,
),
...describeAuctionForTelemetry('server', ack?.state ?? null),
});
if (canAutoRetry) {
@@ -772,7 +863,9 @@ export function useAuctionSocket(matchId: string | undefined) {
const currentAuction = auction;
const visibleAuction =
currentAuction && currentAuction.matchId === matchId ? currentAuction : null;
currentAuction && currentAuction.matchId === matchId
? currentAuction
: null;
const visibleConnectionStatus: ConnectionStatus =
connectionKey === null
? 'idle'
@@ -797,6 +890,7 @@ export function useAuctionSocket(matchId: string | undefined) {
bidLatencyDiagnostics,
serverRoundTripMs: matchId ? serverRoundTripMs : null,
serverTimeOffsetMs: matchId ? serverTimeOffsetMs : 0,
syncPolledAuctionState,
placeBid(payload: BidPayload) {
const socket = socketRef.current;
if (!socket || !socket.connected) {
@@ -807,6 +901,12 @@ export function useAuctionSocket(matchId: string | undefined) {
return;
}
if (payload.bidPointAmount <= latestConfirmedBidPointAmountRef.current) {
// 조용히 무시하면 화면과 내부 기준가가 어긋났을 때 입찰 버튼이 반응하지
// 않는 것처럼 보인다. 안내하고 최신 상태를 다시 받는다.
setError(
'현재 최고 입찰가보다 높은 금액만 입찰할 수 있어요. 최신 경매 상태를 다시 불러올게요.',
);
resyncAuctionState();
return;
}
@@ -815,6 +915,8 @@ export function useAuctionSocket(matchId: string | undefined) {
const bidPayload: BidPayload = {
...payload,
bidRequestId,
clientObservedBidRoundId:
payload.clientObservedBidRoundId ?? currentAuction?.currentBidRoundId,
clientObservedAuctionLifecycleId:
payload.clientObservedAuctionLifecycleId ??
currentAuction?.auctionLifecycleId,
@@ -835,36 +937,32 @@ export function useAuctionSocket(matchId: string | undefined) {
clearPendingBidRequest(bidRequestId);
setError('입찰 응답이 지연되어 최신 상태를 다시 확인해주세요.');
}, 10_000);
socket.emit(
'bid',
bidPayload,
(ack?: BidAckPayload) => {
captureBidLatencyDiagnostics(ack);
if (ack?.rejected) {
clearPendingBidRequest(bidRequestId);
// 입찰 거부도 최신 state로 rebaseline(예: 0초 입찰 EXPIRED → 만료된
// 라운드 화면에서 벗어나게). 단 입찰은 자동 재시도하지 않는다(의도치
// 않은 금액 재입찰 방지).
if (ack.state) {
commitAuctionState(ack.state);
} else if (isAuctionResyncRecommended(ack)) {
resyncAuctionState();
}
setError(resolveAuctionRejectionMessage(ack));
return;
socket.emit('bid', bidPayload, (ack?: BidAckPayload) => {
captureBidLatencyDiagnostics(ack);
if (ack?.rejected) {
clearPendingBidRequest(bidRequestId);
// 입찰 거부도 최신 state로 rebaseline(예: 0초 입찰 EXPIRED → 만료된
// 라운드 화면에서 벗어나게). 단 입찰은 자동 재시도하지 않는다(의도치
// 않은 금액 재입찰 방지).
if (ack.state) {
commitAuctionState(ack.state);
} else if (isAuctionResyncRecommended(ack)) {
resyncAuctionState();
}
if (ack?.bidDelta) {
clearPendingBidRequest(bidRequestId);
commitBidDelta(ack.bidDelta);
return;
}
const ackState = ack?.state;
if (ackState) {
clearPendingBidRequest(bidRequestId);
commitAuctionState(ackState);
}
},
);
setError(resolveAuctionRejectionMessage(ack));
return;
}
if (ack?.bidDelta) {
clearPendingBidRequest(bidRequestId);
commitBidDelta(ack.bidDelta);
return;
}
const ackState = ack?.state;
if (ackState) {
clearPendingBidRequest(bidRequestId);
commitAuctionState(ackState);
}
});
},
openBidding() {
emitAuctionAction('open-bidding', buildAuctionActionPayload());
@@ -905,7 +1003,10 @@ function resolveCurrentBidPointAmount(auction: ApiAuction): number {
}
// [AUCTION-DEBUG] 아래 텔레메트리 헬퍼들과 호출부는 조사 종료 후 제거 대상.
type AuctionTelemetryAttributes = Record<string, string | number | boolean | null>;
type AuctionTelemetryAttributes = Record<
string,
string | number | boolean | null
>;
function readAuctionHistorySequence(auction: ApiAuction | null | undefined) {
const value = (auction as { currentAuctionHistorySequence?: unknown } | null)
@@ -914,7 +1015,9 @@ function readAuctionHistorySequence(auction: ApiAuction | null | undefined) {
}
function summarizeAuctionTeams(auction: ApiAuction | null | undefined) {
const teams = Array.isArray(auction?.auctionTeams) ? auction.auctionTeams : [];
const teams = Array.isArray(auction?.auctionTeams)
? auction.auctionTeams
: [];
let assignedMemberCount = 0;
for (const team of teams) {
const members = (team as { members?: unknown[] }).members;
@@ -956,143 +1059,3 @@ function describeAuctionForTelemetry(
[`${prefix}ForceResync`]: Boolean(auction?.shouldForceAuctionStateResync),
};
}
function shouldApplyIncomingAuctionState(
current: ApiAuction | null,
incoming: ApiAuction,
): boolean {
if (!current) return true;
if (incoming.shouldForceAuctionStateResync) return true;
if (
current.auctionLifecycleId &&
incoming.auctionLifecycleId &&
current.auctionLifecycleId !== incoming.auctionLifecycleId
) {
return true;
}
if (
typeof current.revision === 'number' &&
typeof incoming.revision === 'number' &&
incoming.revision <= current.revision
) {
return false;
}
return true;
}
function applyBidDelta(
current: ApiAuction | null,
payload: BidDeltaPayload,
): ApiAuction | null {
if (!current) return current;
if (
payload.auctionLifecycleId &&
current.auctionLifecycleId &&
payload.auctionLifecycleId !== current.auctionLifecycleId
) {
return current;
}
if (
typeof current.revision === 'number' &&
typeof payload.revision === 'number' &&
payload.revision <= current.revision
) {
return current;
}
const bidPointAmount = Number(payload.currentHighestBidPointAmount);
const bidderTeamIndex = Number(payload.currentHighestBidderTeamIndex);
if (!Number.isFinite(bidPointAmount) || !Number.isFinite(bidderTeamIndex)) {
return current;
}
const bidHistoryEntry = normalizeBidHistoryEntry(payload.auctionBidHistoryEntry);
const currentBidRoundDeadlineTimestampMS =
typeof payload.currentBidRoundDeadlineTimestampMS === 'number'
? payload.currentBidRoundDeadlineTimestampMS
: current.currentBidRoundDeadlineTimestampMS;
return {
...current,
auctionBidHistory: appendUniqueAuctionBidHistoryEntry(
current.auctionBidHistory ?? [],
bidHistoryEntry,
),
bids: bidHistoryEntry
? appendUniqueLegacyBidEntry(current.bids ?? [], bidHistoryEntry)
: current.bids,
currentBid: bidPointAmount,
currentBidderTeamIndex: bidderTeamIndex,
currentDeadline: currentBidRoundDeadlineTimestampMS
? new Date(currentBidRoundDeadlineTimestampMS).toISOString()
: current.currentDeadline,
currentHighestBidPointAmount: bidPointAmount,
currentHighestBidderTeamIndex: bidderTeamIndex,
currentBidRoundDeadlineTimestampMS,
currentServerTimestampMS:
payload.currentServerTimestampMS ?? current.currentServerTimestampMS,
revision: payload.revision ?? current.revision,
};
}
function normalizeBidHistoryEntry(
entry: BidDeltaPayload['auctionBidHistoryEntry'],
): NonNullable<ApiAuction['auctionBidHistory']>[number] | null {
if (!entry) return null;
const bidderTeamIndex = Number(entry.bidderTeamIndex);
const bidPointAmount = Number(entry.bidPointAmount);
const bidPlacedAtTimestampMS = Number(entry.bidPlacedAtTimestampMS);
if (!Number.isFinite(bidderTeamIndex) || !Number.isFinite(bidPointAmount)) {
return null;
}
return {
bidderTeamIndex,
bidPointAmount,
bidderUserId: entry.bidderUserId ?? '',
bidPlacedAtTimestampMS: Number.isFinite(bidPlacedAtTimestampMS)
? bidPlacedAtTimestampMS
: Date.now(),
};
}
function appendUniqueAuctionBidHistoryEntry(
current: NonNullable<ApiAuction['auctionBidHistory']>,
entry: NonNullable<ApiAuction['auctionBidHistory']>[number] | null,
) {
if (!entry) return current;
if (
current.some(
(existing) =>
existing.bidderTeamIndex === entry.bidderTeamIndex &&
existing.bidPointAmount === entry.bidPointAmount &&
existing.bidPlacedAtTimestampMS === entry.bidPlacedAtTimestampMS,
)
) {
return current;
}
return [...current, entry];
}
function appendUniqueLegacyBidEntry(
current: ApiAuction['bids'],
entry: NonNullable<ApiAuction['auctionBidHistory']>[number],
) {
const currentBids = current ?? [];
const legacyBid = {
amount: entry.bidPointAmount,
bidderUserId: entry.bidderUserId,
teamIndex: entry.bidderTeamIndex,
timestamp: new Date(entry.bidPlacedAtTimestampMS).toISOString(),
};
if (
currentBids.some(
(existing) =>
existing.teamIndex === legacyBid.teamIndex &&
existing.amount === legacyBid.amount &&
existing.timestamp === legacyBid.timestamp,
)
) {
return currentBids;
}
return [...currentBids, legacyBid];
}
+10 -2
View File
@@ -40,6 +40,7 @@ import {
useDevCaptainTeamIndex,
} from "../config";
import { ApiError } from "../lib/api";
import { mergeAuctionState } from "../api/auction-state";
import {
championDisplayName,
championIconUrl,
@@ -152,7 +153,10 @@ function pickLatestAuction(
const socketRevision = getAuctionRevision(socketAuction);
const queryRevision = getAuctionRevision(queryAuction);
if (socketRevision !== null && queryRevision !== null) {
return queryRevision > socketRevision ? queryAuction : socketAuction;
// Polling can repair a partial socket state at the same revision.
return queryRevision >= socketRevision
? mergeAuctionState(socketAuction, queryAuction)
: mergeAuctionState(queryAuction, socketAuction);
}
}
return getAuctionUpdatedAt(queryAuction) > getAuctionUpdatedAt(socketAuction)
@@ -1901,6 +1905,10 @@ export default function AuctionPage() {
cancelConfirmInputRef.current?.focus();
}, [showCancelModal]);
const { syncPolledAuctionState } = socket;
useEffect(() => {
syncPolledAuctionState(auctionQuery.data);
}, [auctionQuery.data, syncPolledAuctionState]);
const liveAuction = pickLatestAuction(socket.auction, auctionQuery.data);
const auctionState = useMemo(
() =>
@@ -2517,7 +2525,7 @@ export default function AuctionPage() {
</h2>
<p className="mt-2 text-sm leading-6 text-base-content/60">
정말 경매를 취소하고 모집 상태로 되돌리시겠습니까? 경매
데이터, 입찰 기록, 팀 배정 정보가 삭제됩니다.
진행 데이터와 팀 배정 정보가 초기화됩니다.
</p>
</div>
</div>
@@ -7,6 +7,26 @@ import {
const rosterMessage = "팀 배정 상태를 확인해야 해 경매를 완료하지 못했습니다.";
test("asks an outdated client to reload instead of claiming the player advanced", () => {
const message =
"경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.";
assert.equal(
resolveAuctionRejectionMessage({
auctionRejectionReason: "BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED",
message,
}),
message,
);
// An older bundle without the new reason mapping still displays the server message.
assert.equal(
resolveAuctionRejectionMessage({
auctionRejectionReason: "UNRECOGNIZED_NEW_CLIENT_REASON",
message,
}),
message,
);
});
test("shows the common error message when roster validation blocks completion", () => {
assert.equal(
resolveAuctionRejectionMessage({
+95
View File
@@ -0,0 +1,95 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
createAuctionResyncRequester,
type AuctionResyncReply,
} from '../src/api/auction-resync.ts';
import type { ApiAuction } from '../src/api/types.ts';
const state = (revision: number) =>
({ revision, auctionLifecycleId: 'life', phase: 'bidding' }) as ApiAuction;
const noWait = async () => {};
test('coalesces simultaneous gaps into one snapshot request', async () => {
let finish!: (reply: AuctionResyncReply) => void;
let calls = 0;
const received: number[] = [];
const sync = createAuctionResyncRequester(
() => {
calls++;
return new Promise((resolve) => {
finish = resolve;
});
},
(value) => received.push(value.revision!),
noWait,
);
const requests = Array.from({ length: 100 }, () => sync.request(12, 'life'));
assert.equal(calls, 1);
finish({ ok: true, state: state(12) });
await Promise.all(requests);
assert.deepEqual(received, [12]);
});
test('fetches again when a newer gap arrived while the snapshot was in flight', async () => {
let finish!: (reply: AuctionResyncReply) => void;
let calls = 0;
const received: number[] = [];
const sync = createAuctionResyncRequester(
() =>
++calls === 1
? new Promise((resolve) => {
finish = resolve;
})
: Promise.resolve({ ok: true, state: state(14) }),
(value) => received.push(value.revision!),
noWait,
);
const pending = sync.request(12, 'life');
void sync.request(14, 'life');
void sync.request(); // A generic refresh cannot lower the required revision.
finish({ ok: true, state: state(12) });
await pending;
assert.deepEqual(received, [12, 14]);
assert.equal(calls, 2);
});
test('bounds retries and honors retry delay without rejoining the room', async () => {
let calls = 0;
const waits: number[] = [];
const sync = createAuctionResyncRequester(
async () => {
calls++;
return { ok: false, retryAfterMS: 1500 };
},
() => assert.fail('must not apply'),
async (ms) => {
waits.push(ms);
},
);
await sync.request();
assert.equal(calls, 3);
assert.deepEqual(waits, [1500, 1500]);
});
test('drops an old connection reply after cancellation and permits a new request', async () => {
let finish!: (reply: AuctionResyncReply) => void;
let calls = 0;
const received: number[] = [];
const sync = createAuctionResyncRequester(
() =>
++calls === 1
? new Promise((resolve) => {
finish = resolve;
})
: Promise.resolve({ ok: true, state: state(20) }),
(value) => received.push(value.revision!),
noWait,
);
const stale = sync.request();
sync.cancel();
await sync.request();
finish({ ok: true, state: state(10) });
await stale;
assert.deepEqual(received, [20]);
});
+188
View File
@@ -0,0 +1,188 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
applyBidDelta,
mergeAuctionState,
shouldApplyIncomingAuctionState,
shouldApplyPolledAuctionState,
shouldResyncForBidDelta,
} from "../src/api/auction-state.ts";
import type { ApiAuction } from "../src/api/types.ts";
const current = {
auctionLifecycleId: "life",
revision: 10,
phase: "bidding",
currentBidRoundId: "round-A",
currentBidRoundPlayerUserId: "A",
currentHighestBidPointAmount: 10,
auctionBidHistory: [],
bids: [],
} as unknown as ApiAuction;
const bid = {
auctionLifecycleId: "life",
revision: 11,
currentBidRoundId: "round-A",
currentHighestBidPointAmount: 20,
currentHighestBidderTeamIndex: 1,
auctionBidHistoryEntry: {
bidPointAmount: 20,
bidderTeamIndex: 1,
bidderUserId: "captain",
bidPlacedAtTimestampMS: 1000,
playerUserId: "A",
bidRoundId: "round-A",
},
};
test("a contiguous same-round bid updates price and preserves player identity", () => {
const next = applyBidDelta(current, bid)!;
assert.equal(shouldResyncForBidDelta(current, bid), false);
assert.equal(next.currentHighestBidPointAmount, 20);
assert.equal(next.currentBidRoundPlayerUserId, "A");
assert.equal(next.auctionBidHistory?.[0].bidRoundId, "round-A");
});
test("a revision gap resyncs instead of attaching a new player bid to the old structure", () => {
const missedTransition = {
...bid,
revision: 13,
currentBidRoundId: "round-B",
};
assert.equal(shouldResyncForBidDelta(current, missedTransition), true);
assert.equal(applyBidDelta(current, missedTransition), current);
});
test("a delta cannot open bidding or change the round by itself", () => {
assert.equal(
shouldResyncForBidDelta({ ...current, phase: "presenting" }, bid),
true,
);
assert.equal(
shouldResyncForBidDelta(current, { ...bid, currentBidRoundId: "round-B" }),
true,
);
});
test("a full snapshot can repair the same revision but cannot regress auction revision", () => {
assert.equal(
shouldApplyIncomingAuctionState(current, {
...current,
currentBidRoundPlayerUserId: "B",
}),
true,
);
assert.equal(
shouldApplyIncomingAuctionState(current, { ...current, revision: 9 }),
false,
);
});
test("full state repair preserves a newer independently versioned side effect", () => {
const live = {
...current,
matchSideEffectRevision: 3,
matchSideEffectStatus: "completed" as const,
};
const incoming = {
...current,
matchSideEffectRevision: 1,
matchSideEffectStatus: "pending" as const,
};
assert.equal(
mergeAuctionState(live, incoming).matchSideEffectStatus,
"completed",
);
assert.equal(
mergeAuctionState(live, { ...incoming, auctionLifecycleId: "new-life" })
.matchSideEffectStatus,
"pending",
);
});
test("replayed old bids do not resync or change the current state", () => {
const replay = { ...bid, revision: 9 };
assert.equal(shouldResyncForBidDelta(current, replay), false);
assert.equal(applyBidDelta(current, replay), current);
});
test("polled state repairs a socket that missed a transition", () => {
const polled = {
...current,
matchId: "match",
revision: 12,
currentBidRoundId: "round-B",
currentBidRoundPlayerUserId: "B",
currentHighestBidPointAmount: 0,
} as unknown as ApiAuction;
const socketState = { ...current, matchId: "match" } as ApiAuction;
assert.equal(
shouldApplyPolledAuctionState(socketState, polled, "match"),
true,
);
});
test("polled state never regresses or replaces the socket lifecycle", () => {
const socketState = { ...current, matchId: "match" } as ApiAuction;
const same = { ...socketState } as ApiAuction;
const older = { ...socketState, revision: 9 } as ApiAuction;
const otherLifecycle = {
...socketState,
auctionLifecycleId: "old-life",
revision: 99,
} as ApiAuction;
const otherMatch = { ...socketState, matchId: "other", revision: 11 };
assert.equal(shouldApplyPolledAuctionState(socketState, same, "match"), true);
assert.equal(
shouldApplyPolledAuctionState(socketState, older, "match"),
false,
);
assert.equal(
shouldApplyPolledAuctionState(socketState, otherLifecycle, "match"),
false,
);
assert.equal(
shouldApplyPolledAuctionState(socketState, otherMatch, "match"),
false,
);
assert.equal(shouldApplyPolledAuctionState(null, same, "match"), true);
});
test("polling initializes a missing socket baseline but rejects another match or missing identity", () => {
const incoming = {
...current,
matchId: "match",
currentBidRoundId: "round-B",
currentHighestBidPointAmount: 0,
} as ApiAuction;
assert.equal(shouldApplyPolledAuctionState(null, incoming, "match"), true);
assert.equal(
shouldApplyPolledAuctionState(null, incoming, "another-match"),
false,
);
assert.equal(
shouldApplyPolledAuctionState(
null,
{ ...incoming, auctionLifecycleId: undefined },
"match",
),
false,
);
});
test("a prior side-effect generation cannot replace the status of a new transition", () => {
const old = {
...current,
matchSideEffectTransitionId: "old",
matchSideEffectRevision: 100,
matchSideEffectStatus: "failed" as const,
};
const next = {
...current,
revision: 12,
matchSideEffectTransitionId: "new",
matchSideEffectRevision: 3,
matchSideEffectStatus: "pending" as const,
};
assert.equal(mergeAuctionState(old, next).matchSideEffectStatus, "pending");
});