Compare commits
2
Commits
ef141a13d0
...
f649490cd2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f649490cd2 | ||
|
|
f2eec479d9 |
No files matched your search
@@ -62,8 +62,61 @@ $ pnpm run test:e2e
|
||||
|
||||
# test coverage
|
||||
$ pnpm run test:cov
|
||||
|
||||
# auction cancellation/undo transactions against a real local MongoDB replica set
|
||||
$ pnpm run test:mongodb
|
||||
```
|
||||
|
||||
`test:mongodb` requires `mongod` on PATH, or `MONGODB_TEST_SERVER` set to its
|
||||
executable path. The runner starts an isolated replica set on a random loopback
|
||||
port, uses a temporary database directory, and removes it after the tests. It
|
||||
does not use the application's database connection settings.
|
||||
|
||||
## Auction durability
|
||||
|
||||
Accepted Redis bids now enter `auction:bidArchivePayloads` and
|
||||
`auction:bidArchiveQueue` atomically with the bid. The durability scheduler saves
|
||||
them to `auctionBidRecords` using deterministic IDs before removing queue entries.
|
||||
Recent state snapshots remain capped at 1000 bids; the archive is not capped and
|
||||
retains accepted bids across undo/cancellation for audit. These records begin with
|
||||
this deployment; bids already discarded by an older version cannot be recovered.
|
||||
`GET /communities/:communityId/matches/:matchId/auction/bids/history` pages through
|
||||
the current lifecycle's persisted archive using `afterRevision` and `limit` (1–200).
|
||||
Persistence is asynchronous, so an empty last page may gain entries afterward.
|
||||
|
||||
Invalid archive entries are moved atomically to `auction:bidArchiveDeadLetterQueue`
|
||||
and `auction:bidArchiveDeadLetterPayloads`. The payload stores the original text
|
||||
(or `null` for a missing payload), error, ID, and quarantine time. Monitor these
|
||||
keys and review quarantined data before repairing/requeuing it; it is not retried
|
||||
automatically or expired. Other bids continue to persist. Mongo write failures
|
||||
leave valid entries in the pending queue for idempotent retries.
|
||||
|
||||
Older in-flight rounds can produce `bidRoundId: ''`; the archive treats this as
|
||||
an absent optional round ID. Empty player IDs and other invalid field types
|
||||
remain validation errors.
|
||||
|
||||
Cancellation commits its completion marker and pending cleanup fields in Match.
|
||||
The scheduler retries cleanup after restarts, with a delay capped at 60 seconds.
|
||||
Reset notifications include the lifecycle to protect a newly started auction.
|
||||
Dead-letter status notifications use `auction:deadLetterStatusQueue`; failures to
|
||||
persist or mirror the terminal status are retried without rerunning the side effect.
|
||||
Older side-effect dead letters are also reconciled automatically.
|
||||
|
||||
Before deploying the cleanup worker, create its partial Match index using the
|
||||
additive, repeatable migration below. Schema declarations alone do not create
|
||||
indexes when `MONGODB_AUTO_INDEX` is disabled (the default). Choose the target
|
||||
environment explicitly; dry-run does not create indexes or modify documents.
|
||||
|
||||
```bash
|
||||
pnpm --filter api auction:indexes -- --env staging --dry-run
|
||||
pnpm --filter api auction:indexes -- --env staging --apply
|
||||
# Use --env prod for production.
|
||||
```
|
||||
|
||||
The index is `{ auctionCancellationCleanupRetryAt: 1 }`, restricted to
|
||||
`{ auctionCancellationCleanupPending: true }`. The worker only reads due pending
|
||||
entries; it no longer scans legacy cancellation markers without a pending flag.
|
||||
|
||||
## Deployment
|
||||
|
||||
When you're ready to deploy your NestJS application to production, there are some key steps you can take to ensure it runs as efficiently as possible. Check out the [deployment documentation](https://docs.nestjs.com/deployment) for more information.
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
"build": "nest build",
|
||||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
||||
"competition:migrate": "ts-node src/modules/global/database/migrations/run-competition-domain.ts",
|
||||
"auction:indexes": "ts-node src/modules/global/database/migrations/run-auction-cancellation-index.ts",
|
||||
"seed:dev": "ts-node src/seed.ts",
|
||||
"seed:dakao-participant": "ts-node src/seed-dakao-participant.ts",
|
||||
"start:dev": "NODE_ENV=dev NEW_RELIC_HOME=. node --import newrelic/esm-loader.mjs -r newrelic dist/main.js",
|
||||
@@ -24,6 +25,7 @@
|
||||
"test": "jest",
|
||||
"typecheck": "tsc --noEmit --incremental false",
|
||||
"test:redis": "node test/run-redis-tests.mjs",
|
||||
"test:mongodb": "node test/run-mongo-tests.mjs",
|
||||
"test:watch": "jest --watch",
|
||||
"test:cov": "jest --coverage",
|
||||
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
|
||||
const matchId = new mongoose.Types.ObjectId().toString();
|
||||
|
||||
function bid(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
_id: `${matchId}:life:${String(1).padStart(16, '0')}`,
|
||||
matchId,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 1,
|
||||
bidRequestId: 'bid-1',
|
||||
bidderUserId: new mongoose.Types.ObjectId().toString(),
|
||||
playerUserId: new mongoose.Types.ObjectId().toString(),
|
||||
bidderTeamIndex: 1,
|
||||
bidPointAmount: 10,
|
||||
bidPlacedAtTimestampMS: 1_700_000_000_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(payload: Record<string, unknown>) {
|
||||
const raw = JSON.stringify(payload);
|
||||
const commands: string[][] = [];
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
commands.push(command);
|
||||
if (command[0] === 'ZRANGE') return [payload._id];
|
||||
if (command[0] === 'HMGET') return [raw];
|
||||
return 1;
|
||||
});
|
||||
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
|
||||
matchedCount: 0,
|
||||
upsertedCount: operations.length,
|
||||
}));
|
||||
const service = new AuctionBidArchiveService(
|
||||
{ getModel: async () => ({ bulkWrite }) } as never,
|
||||
{ raw: () => ({ sendCommand }) } as never,
|
||||
);
|
||||
return { service, bulkWrite, commands };
|
||||
}
|
||||
|
||||
function savedEvent(bulkWrite: jest.Mock) {
|
||||
const [[operations]] = bulkWrite.mock.calls as unknown as [
|
||||
[
|
||||
Array<{
|
||||
updateOne: { update: { $setOnInsert: Record<string, unknown> } };
|
||||
}>,
|
||||
],
|
||||
];
|
||||
return operations[0].updateOne.update.$setOnInsert;
|
||||
}
|
||||
|
||||
function quarantined(commands: string[][]) {
|
||||
return commands.some(
|
||||
(command) =>
|
||||
command[0] === 'EVAL' &&
|
||||
command[1].includes("redis.call('ZADD', KEYS[4]"),
|
||||
);
|
||||
}
|
||||
|
||||
it('archives a pre-upgrade bid with an empty round ID without the field', async () => {
|
||||
const f = fixture(bid({ bidRoundId: '' }));
|
||||
await expect(f.service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(quarantined(f.commands)).toBe(false);
|
||||
expect(savedEvent(f.bulkWrite)).not.toHaveProperty('bidRoundId');
|
||||
});
|
||||
|
||||
it('keeps a non-empty round ID on the archived bid', async () => {
|
||||
const f = fixture(bid({ bidRoundId: 'round-1' }));
|
||||
await expect(f.service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(savedEvent(f.bulkWrite)).toMatchObject({ bidRoundId: 'round-1' });
|
||||
});
|
||||
|
||||
it.each([' ', null, 1])(
|
||||
'still quarantines an invalid round ID %j',
|
||||
async (bidRoundId) => {
|
||||
const f = fixture(bid({ bidRoundId }));
|
||||
await expect(f.service.persistPendingBids()).resolves.toBe(0);
|
||||
expect(f.bulkWrite).not.toHaveBeenCalled();
|
||||
expect(quarantined(f.commands)).toBe(true);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,242 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Injectable,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import type { AuctionBidRecord } from '../global/database/schema';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { AuctionRedisService } from './redis/auction-redis.service';
|
||||
import {
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY,
|
||||
} from './redis/auction-redis.keys';
|
||||
|
||||
// Compare the payload as well as the ID: a concurrent repair must not be moved
|
||||
// or acknowledged by a worker that read an older value.
|
||||
const QUARANTINE_BID_ARCHIVE_SCRIPT = `
|
||||
local moved = 0
|
||||
for i = 2, #ARGV, 4 do
|
||||
local current = redis.call('HGET', KEYS[1], ARGV[i])
|
||||
local matches = (ARGV[i+1] == '0' and current == false) or (ARGV[i+1] == '1' and current == ARGV[i+2])
|
||||
if matches and redis.call('ZSCORE', KEYS[2], ARGV[i]) then
|
||||
redis.call('HSET', KEYS[3], ARGV[i], ARGV[i+3])
|
||||
redis.call('ZADD', KEYS[4], ARGV[1], ARGV[i])
|
||||
redis.call('HDEL', KEYS[1], ARGV[i])
|
||||
redis.call('ZREM', KEYS[2], ARGV[i])
|
||||
moved = moved + 1
|
||||
end
|
||||
end
|
||||
return moved
|
||||
`;
|
||||
|
||||
@Injectable()
|
||||
export class AuctionBidArchiveService {
|
||||
private readonly logger = new Logger(AuctionBidArchiveService.name);
|
||||
|
||||
constructor(
|
||||
private readonly database: ConnectionService,
|
||||
private readonly redis: AuctionRedisService,
|
||||
) {}
|
||||
|
||||
public async persistPendingBids(limit = 200): Promise<number> {
|
||||
const client = this.redis.raw();
|
||||
const reply = await client.sendCommand([
|
||||
'ZRANGE',
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
'0',
|
||||
String(limit - 1),
|
||||
]);
|
||||
const ids = Array.isArray(reply) ? reply.map(String) : [];
|
||||
if (!ids.length) return 0;
|
||||
const values = await client.sendCommand<Array<string | null>>([
|
||||
'HMGET',
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
...ids,
|
||||
]);
|
||||
const valid: Array<{ event: AuctionBidRecord; raw: string }> = [];
|
||||
const invalid: Array<{ id: string; raw: string | null; error: string }> =
|
||||
[];
|
||||
for (let index = 0; index < ids.length; index++) {
|
||||
const raw = values[index] ?? null;
|
||||
try {
|
||||
valid.push({ event: parseBidArchiveEvent(ids[index], raw), raw: raw! });
|
||||
} catch (error) {
|
||||
invalid.push({
|
||||
id: ids[index],
|
||||
raw,
|
||||
error: String(error).slice(0, 2000),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (invalid.length) {
|
||||
const now = Date.now();
|
||||
const moved = await client.sendCommand<number>([
|
||||
'EVAL',
|
||||
QUARANTINE_BID_ARCHIVE_SCRIPT,
|
||||
'4',
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY,
|
||||
String(now),
|
||||
...invalid.flatMap(({ id, raw, error }) => [
|
||||
id,
|
||||
raw === null ? '0' : '1',
|
||||
raw ?? '',
|
||||
JSON.stringify({
|
||||
id,
|
||||
rawPayload: raw,
|
||||
error,
|
||||
quarantinedAtTimestampMS: now,
|
||||
}),
|
||||
]),
|
||||
]);
|
||||
if (moved > 0)
|
||||
this.logger.warn(`Quarantined ${moved} invalid bid archive events.`);
|
||||
}
|
||||
if (valid.length) {
|
||||
const model = await this.database.getModel('auctionBidRecord');
|
||||
// The deterministic _id is the dedupe key, including after a crash between
|
||||
// Mongo acknowledgement and Redis cleanup. Never trim unpersisted events.
|
||||
const result = await model.bulkWrite(
|
||||
valid.map(({ event }) => ({
|
||||
updateOne: {
|
||||
filter: { _id: event._id },
|
||||
update: { $setOnInsert: event },
|
||||
upsert: true,
|
||||
},
|
||||
})),
|
||||
{ ordered: false, throwOnValidationError: true },
|
||||
);
|
||||
// A partial/ambiguous write leaves valid entries queued. Replaying the
|
||||
// upserts is safe; assuming that a resolved bulkWrite saved all is not.
|
||||
if (result.matchedCount + result.upsertedCount !== valid.length) {
|
||||
throw new Error(
|
||||
'Bid archive batch was not fully acknowledged by MongoDB.',
|
||||
);
|
||||
}
|
||||
await client.sendCommand([
|
||||
'EVAL',
|
||||
"for i = 1, #ARGV, 2 do if redis.call('HGET', KEYS[1], ARGV[i]) == ARGV[i+1] then redis.call('HDEL', KEYS[1], ARGV[i]); redis.call('ZREM', KEYS[2], ARGV[i]) end end return 1",
|
||||
'2',
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
...valid.flatMap(({ event, raw }) => [event._id, raw]),
|
||||
]);
|
||||
}
|
||||
return valid.length;
|
||||
}
|
||||
|
||||
public async list(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
afterRevision = -1,
|
||||
limit = 100,
|
||||
) {
|
||||
if (
|
||||
!Number.isSafeInteger(afterRevision) ||
|
||||
afterRevision < -1 ||
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > 200
|
||||
) {
|
||||
throw new BadRequestException('Invalid bid history cursor or limit.');
|
||||
}
|
||||
const AuctionModel = await this.database.getModel('auction');
|
||||
const auction = await AuctionModel.findOne({ matchId })
|
||||
.select({ auctionLifecycleId: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
if (!auction) throw new NotFoundException('Auction not found.');
|
||||
const prefix = `${matchId.toString()}:${auction.auctionLifecycleId}:`;
|
||||
const model = await this.database.getModel('auctionBidRecord');
|
||||
// The built-in _id index supports ordered pagination without an additional
|
||||
// production index or an unbounded array inside an Auction document.
|
||||
const items = await model
|
||||
.find({
|
||||
_id: {
|
||||
$gt:
|
||||
afterRevision < 0
|
||||
? prefix
|
||||
: prefix + String(afterRevision).padStart(16, '0'),
|
||||
$lt: prefix + '\uffff',
|
||||
},
|
||||
})
|
||||
.sort({ _id: 1 })
|
||||
.limit(limit + 1)
|
||||
.lean()
|
||||
.exec();
|
||||
const hasMore = items.length > limit;
|
||||
if (hasMore) items.pop();
|
||||
return {
|
||||
items,
|
||||
hasMore,
|
||||
nextRevision: items.at(-1)?.revision ?? afterRevision,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function parseBidArchiveEvent(
|
||||
id: string,
|
||||
raw: string | null,
|
||||
): AuctionBidRecord {
|
||||
if (raw === null) throw new Error('Missing bid archive payload.');
|
||||
const value: unknown = JSON.parse(raw);
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error('Bid archive payload must be an object.');
|
||||
}
|
||||
const event = value as Record<string, unknown>;
|
||||
const string = (field: string): string => {
|
||||
const value = event[field];
|
||||
if (typeof value !== 'string' || !value.trim()) {
|
||||
throw new Error(`Invalid bid archive ${field}.`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const integer = (field: string, minimum = 0): number => {
|
||||
const value = event[field];
|
||||
if (
|
||||
typeof value !== 'number' ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < minimum
|
||||
) {
|
||||
throw new Error(`Invalid bid archive ${field}.`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const objectId = (field: string): mongoose.Types.ObjectId => {
|
||||
const value = string(field);
|
||||
if (!mongoose.isObjectIdOrHexString(value)) {
|
||||
throw new Error(`Invalid bid archive ${field}.`);
|
||||
}
|
||||
return new mongoose.Types.ObjectId(value);
|
||||
};
|
||||
const matchId = objectId('matchId');
|
||||
const auctionLifecycleId = string('auctionLifecycleId');
|
||||
const revision = integer('revision');
|
||||
const expectedId = `${matchId.toString()}:${auctionLifecycleId}:${String(revision).padStart(16, '0')}`;
|
||||
if (event._id !== id || id !== expectedId) {
|
||||
throw new Error('Bid archive identity does not match its payload.');
|
||||
}
|
||||
return {
|
||||
_id: id,
|
||||
matchId,
|
||||
auctionLifecycleId,
|
||||
revision,
|
||||
bidRequestId: string('bidRequestId'),
|
||||
bidderUserId: objectId('bidderUserId'),
|
||||
bidderTeamIndex: integer('bidderTeamIndex', 1),
|
||||
bidPointAmount: integer('bidPointAmount'),
|
||||
bidPlacedAtTimestampMS: integer('bidPlacedAtTimestampMS'),
|
||||
...(event.playerUserId !== undefined
|
||||
? { playerUserId: objectId('playerUserId') }
|
||||
: {}),
|
||||
// Pre-upgrade rounds have no Redis round ID; the bid Lua records ''.
|
||||
...(event.bidRoundId !== undefined && event.bidRoundId !== ''
|
||||
? { bidRoundId: string('bidRoundId') }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { NotFoundException } from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionService } from './auction.service';
|
||||
|
||||
const matchId = new mongoose.Types.ObjectId();
|
||||
const communityId = new mongoose.Types.ObjectId();
|
||||
|
||||
function sessionQuery<T>(value: T) {
|
||||
return { session: async () => value };
|
||||
}
|
||||
|
||||
function fixture(options: {
|
||||
auction: Record<string, unknown> | null;
|
||||
match: Record<string, unknown>;
|
||||
failRedisCleanup?: boolean;
|
||||
}) {
|
||||
const session = {
|
||||
startTransaction: jest.fn(),
|
||||
commitTransaction: jest.fn().mockResolvedValue(undefined),
|
||||
abortTransaction: jest.fn().mockResolvedValue(undefined),
|
||||
endSession: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const deleteMany = jest.fn(() => sessionQuery(undefined));
|
||||
const models: Record<string, unknown> = {
|
||||
auction: { findOne: () => sessionQuery(options.auction) },
|
||||
auctionHistory: { deleteMany },
|
||||
auctionSideEffectApplication: { deleteMany },
|
||||
match: { findById: () => sessionQuery(options.match) },
|
||||
};
|
||||
const redisCommands: string[][] = [];
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
redisCommands.push(command);
|
||||
if (command[0] === 'EVAL' && options.failRedisCleanup)
|
||||
throw new Error('redis unavailable');
|
||||
return command[0] === 'SCAN' ? ['0', []] : 1;
|
||||
});
|
||||
const service = new AuctionService(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => session,
|
||||
} as never,
|
||||
{} as never,
|
||||
{ releaseFromMatch: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
{ cancelPolling: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
{
|
||||
acquireTokenLock: async () => true,
|
||||
releaseTokenLock: async () => true,
|
||||
raw: () => ({ sendCommand }),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
return { service, session, redisCommands };
|
||||
}
|
||||
|
||||
function liveMatch(): Record<string, unknown> {
|
||||
return {
|
||||
communityId,
|
||||
status: 'team-built',
|
||||
teams: [],
|
||||
tournamentCodeIds: [],
|
||||
gameSchedule: [],
|
||||
participants: [],
|
||||
save: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function redisAuction() {
|
||||
return {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: 'life',
|
||||
deleteOne: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
it('records the cancelled lifecycle and still succeeds when Redis cleanup fails', async () => {
|
||||
const match = liveMatch();
|
||||
const f = fixture({
|
||||
auction: redisAuction(),
|
||||
match,
|
||||
failRedisCleanup: true,
|
||||
});
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(f.session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(match.lastCancelledAuctionLifecycleId).toBe('life');
|
||||
expect(match.status).toBe('recruiting');
|
||||
});
|
||||
|
||||
it('answers a retransmitted cancellation and durably schedules Redis cleanup', async () => {
|
||||
const match = { ...liveMatch(), lastCancelledAuctionLifecycleId: 'life' };
|
||||
const f = fixture({ auction: null, match });
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(match).toMatchObject({
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: expect.any(Date),
|
||||
});
|
||||
expect(f.redisCommands).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves a committed cancellation even if its lock renewal fails before returning', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const match = liveMatch();
|
||||
const f = fixture({ auction: redisAuction(), match });
|
||||
f.session.commitTransaction.mockImplementation(async () => {
|
||||
await jest.advanceTimersByTimeAsync(10000);
|
||||
});
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(match.auctionCancellationCleanupPending).toBe(true);
|
||||
expect(f.session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps 404 for a missing auction that was not cancelled with this lifecycle', async () => {
|
||||
for (const [observed, recorded] of [
|
||||
['other', 'life'],
|
||||
[undefined, 'life'],
|
||||
['life', undefined],
|
||||
] as const) {
|
||||
const f = fixture({
|
||||
auction: null,
|
||||
match: { ...liveMatch(), lastCancelledAuctionLifecycleId: recorded },
|
||||
});
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: observed,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(f.session.abortTransaction).toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,607 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionService } from './auction.service';
|
||||
import {
|
||||
AuctionDurabilityScheduler,
|
||||
buildAuctionProjectionGuardedUpdate,
|
||||
type AuctionDurabilityPayload,
|
||||
} from './auction-durability.scheduler';
|
||||
import { AuctionDeadlineScheduler } from './auction-deadline.scheduler';
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
|
||||
const query = <T>(value: T) => ({
|
||||
select() {
|
||||
return this;
|
||||
},
|
||||
session() {
|
||||
return this;
|
||||
},
|
||||
lean() {
|
||||
return this;
|
||||
},
|
||||
exec: async () => value,
|
||||
});
|
||||
const session = () => ({
|
||||
startTransaction: jest.fn(),
|
||||
commitTransaction: jest.fn(),
|
||||
abortTransaction: jest.fn(),
|
||||
endSession: jest.fn(),
|
||||
});
|
||||
const state = {
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 12,
|
||||
phase: 'presenting',
|
||||
currentAuctionHistorySequence: 7,
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentAuctionRound: 1,
|
||||
auctionTeams: [],
|
||||
auctionPlayerOrder: [],
|
||||
passedAuctionPlayerQueue: [],
|
||||
currentHighestBidPointAmount: 0,
|
||||
currentHighestBidderTeamIndex: -1,
|
||||
auctionCompletedAtTimestampMS: 0,
|
||||
currentBidRoundDeadlineTimestampMS: 0,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest
|
||||
.spyOn(AuctionLifecycleLock.prototype, 'run')
|
||||
.mockImplementation(async (_id, work) => work());
|
||||
});
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
function matches(
|
||||
document: Record<string, unknown>,
|
||||
filter: Record<string, unknown>,
|
||||
): boolean {
|
||||
return Object.entries(filter).every(([key, value]) => {
|
||||
if (key === '$or')
|
||||
return (value as Record<string, unknown>[]).some((alternative) =>
|
||||
matches(document, alternative),
|
||||
);
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.entries(value).every(([operator, limit]) => {
|
||||
if (operator === '$exists')
|
||||
return (document[key] !== undefined) === limit;
|
||||
if (operator === '$lt') return Number(document[key]) < Number(limit);
|
||||
if (operator === '$lte') return Number(document[key]) <= Number(limit);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
return document[key] === value;
|
||||
});
|
||||
}
|
||||
function update(
|
||||
document: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) {
|
||||
Object.assign(document, mutation.$set);
|
||||
for (const [key, amount] of Object.entries(mutation.$inc ?? {}))
|
||||
document[key] = Number(document[key] ?? 0) + Number(amount);
|
||||
for (const key of Object.keys(mutation.$unset ?? {})) delete document[key];
|
||||
}
|
||||
|
||||
function sideEffectFixture(withReader = false) {
|
||||
const projection: Record<string, unknown> = {
|
||||
_id: 'auction',
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 9,
|
||||
matchSideEffectRevision: 0,
|
||||
};
|
||||
const application: Record<string, unknown> = {};
|
||||
const payload: AuctionDurabilityPayload = {
|
||||
auctionId: 'auction',
|
||||
matchId: 'match',
|
||||
auctionLifecycleId: 'life',
|
||||
auctionStateTransitionId: 'complete',
|
||||
auctionActionType: 'complete',
|
||||
auctionHistorySequence: 10,
|
||||
revision: 10,
|
||||
snapshotBefore: {},
|
||||
historyMutation: { type: 'append' },
|
||||
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
|
||||
afterAuctionMongoProjection: {
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 10,
|
||||
phase: 'complete',
|
||||
matchSideEffectRevision: 1,
|
||||
matchSideEffectStatus: 'pending',
|
||||
matchSideEffectTransitionId: 'complete',
|
||||
auctionTeams: [{ teamIndex: 1, captainUserId: 'captain', members: [] }],
|
||||
},
|
||||
};
|
||||
const AuctionModel = {
|
||||
findById: () => query({ ...projection }),
|
||||
updateOne: (
|
||||
filter: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) => {
|
||||
const matchedCount = Number(matches(projection, filter));
|
||||
if (matchedCount) update(projection, mutation);
|
||||
return query({ matchedCount });
|
||||
},
|
||||
findOneAndUpdate: (
|
||||
filter: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) => {
|
||||
if (!matches(projection, filter)) return query(null);
|
||||
update(projection, mutation);
|
||||
return query({ ...projection });
|
||||
},
|
||||
};
|
||||
const ApplicationModel = {
|
||||
findOne: () => query({ ...application }),
|
||||
updateOne: (
|
||||
filter: Record<string, unknown>,
|
||||
mutation: Record<string, Record<string, unknown>>,
|
||||
) => {
|
||||
if (mutation.$setOnInsert) {
|
||||
if (!application.state)
|
||||
Object.assign(application, mutation.$setOnInsert);
|
||||
return query({ matchedCount: 1 });
|
||||
}
|
||||
const matchedCount = Number(matches(application, filter));
|
||||
if (matchedCount) update(application, mutation);
|
||||
return query({ matchedCount });
|
||||
},
|
||||
};
|
||||
const match = {
|
||||
participants: [],
|
||||
teams: [],
|
||||
save: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const historyFindOne = jest.fn().mockReturnValue(query(null));
|
||||
const historyUpdateOne = jest
|
||||
.fn()
|
||||
.mockReturnValue(query({ matchedCount: 1 }));
|
||||
const models: Record<string, unknown> = {
|
||||
auction: AuctionModel,
|
||||
auctionHistory: {
|
||||
findOne: historyFindOne,
|
||||
updateOne: historyUpdateOne,
|
||||
},
|
||||
auctionSideEffectApplication: ApplicationModel,
|
||||
match: {
|
||||
findById: () => ({ ...query(match), session: async () => match }),
|
||||
},
|
||||
};
|
||||
const reserveForMatch = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('temporary reservation error'))
|
||||
.mockResolvedValue(match);
|
||||
const scheduleInitialPoll = jest.fn().mockResolvedValue(undefined);
|
||||
const releaseFromMatch = jest.fn().mockResolvedValue(undefined);
|
||||
const cancelPolling = jest.fn().mockResolvedValue(undefined);
|
||||
let queued = true;
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGEBYSCORE') return queued ? ['payload'] : [];
|
||||
if (command[0] === 'HGETALL')
|
||||
return { payloadJson: JSON.stringify(payload) };
|
||||
if (command[0] === 'DEL') queued = false;
|
||||
if (command[0] === 'HINCRBY') return 1;
|
||||
return 1;
|
||||
});
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => session(),
|
||||
} as never,
|
||||
{
|
||||
getCurrentRedisTimestampMS: async () => 10000,
|
||||
raw: () => ({ sendCommand }),
|
||||
} as never,
|
||||
{ reserveForMatch, releaseFromMatch } as never,
|
||||
{ scheduleInitialPoll, cancelPolling } as never,
|
||||
undefined,
|
||||
undefined,
|
||||
withReader
|
||||
? ({
|
||||
readAuctionStateView: async () => ({
|
||||
auctionLifecycleId: 'life',
|
||||
phase: payload.afterAuctionMongoProjection.phase,
|
||||
matchSideEffectTransitionId: 'complete',
|
||||
}),
|
||||
} as never)
|
||||
: undefined,
|
||||
);
|
||||
return {
|
||||
scheduler,
|
||||
payload,
|
||||
projection,
|
||||
application,
|
||||
reserveForMatch,
|
||||
scheduleInitialPoll,
|
||||
sendCommand,
|
||||
match,
|
||||
historyFindOne,
|
||||
historyUpdateOne,
|
||||
releaseFromMatch,
|
||||
cancelPolling,
|
||||
};
|
||||
}
|
||||
|
||||
it('retries failed completion work with the current generation and only then deletes the payload', async () => {
|
||||
const f = sideEffectFixture();
|
||||
expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({
|
||||
failed: 1,
|
||||
});
|
||||
expect(f.projection.matchSideEffectStatus).toBe('failed');
|
||||
expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({
|
||||
applied: 1,
|
||||
});
|
||||
expect(f.reserveForMatch).toHaveBeenCalledTimes(2);
|
||||
expect(f.scheduleInitialPoll).toHaveBeenCalledTimes(1);
|
||||
expect(f.application.state).toBe('completed');
|
||||
expect(f.projection.matchSideEffectStatus).toBe('completed');
|
||||
expect(f.projection.matchSideEffectRevision).toBe(5);
|
||||
expect(f.sendCommand).toHaveBeenCalledWith(['DEL', 'payload']);
|
||||
});
|
||||
|
||||
it('keeps an outbox payload while another worker owns its side-effect lease', async () => {
|
||||
const f = sideEffectFixture();
|
||||
Object.assign(f.application, {
|
||||
auctionId: 'auction',
|
||||
auctionStateTransitionId: 'complete',
|
||||
sideEffectType: 'syncMatchTeamsAfterComplete',
|
||||
state: 'processing',
|
||||
processingOwnerToken: 'other',
|
||||
processingLeaseExpiresAtTimestampMS: 50000,
|
||||
});
|
||||
expect(await f.scheduler.processDueDurabilityPayloads(20)).toEqual({
|
||||
applied: 0,
|
||||
failed: 0,
|
||||
skipped: 1,
|
||||
});
|
||||
expect(f.sendCommand).not.toHaveBeenCalledWith(['DEL', 'payload']);
|
||||
expect(
|
||||
f.sendCommand.mock.calls.some(([command]) => command[0] === 'HINCRBY'),
|
||||
).toBe(false);
|
||||
expect(f.reserveForMatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('recovers side effects when a later ordinary projection was persisted first', async () => {
|
||||
const f = sideEffectFixture(true);
|
||||
f.reserveForMatch.mockReset().mockResolvedValue(undefined);
|
||||
Object.assign(f.projection, { revision: 12 });
|
||||
await f.scheduler.applyDurabilityPayload(f.payload);
|
||||
expect(f.reserveForMatch).toHaveBeenCalledTimes(1);
|
||||
expect(f.projection.revision).toBe(12);
|
||||
expect(f.projection.matchSideEffectTransitionId).toBe('complete');
|
||||
expect(f.projection.matchSideEffectStatus).toBe('completed');
|
||||
});
|
||||
|
||||
it('still cleans up completed Match state when a second undo invalidates the first undo history', async () => {
|
||||
const f = sideEffectFixture(true);
|
||||
Object.assign(f.projection, { revision: 12 });
|
||||
Object.assign(f.payload, {
|
||||
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
|
||||
});
|
||||
f.payload.afterAuctionMongoProjection.phase = 'presenting';
|
||||
Object.assign(f.match, {
|
||||
teams: [{ teamIndex: 1 }],
|
||||
teamBuiltAt: new Date(),
|
||||
tournamentCodeIds: ['code'],
|
||||
gameSchedule: [{}],
|
||||
});
|
||||
f.historyFindOne.mockReturnValue(
|
||||
query({
|
||||
sequence: 12,
|
||||
historyMutation: {
|
||||
type: 'deleteAfterSequenceAndAppendUndo',
|
||||
targetHistorySequence: 2,
|
||||
},
|
||||
}),
|
||||
);
|
||||
await f.scheduler.applyDurabilityPayload(f.payload);
|
||||
expect(f.historyUpdateOne).not.toHaveBeenCalled();
|
||||
expect(f.releaseFromMatch).toHaveBeenCalledTimes(1);
|
||||
expect(f.cancelPolling).toHaveBeenCalledTimes(1);
|
||||
expect(f.match).toMatchObject({
|
||||
teams: [],
|
||||
teamBuiltAt: undefined,
|
||||
tournamentCodeIds: [],
|
||||
gameSchedule: [],
|
||||
});
|
||||
expect(f.projection.matchSideEffectStatus).toBe('completed');
|
||||
});
|
||||
|
||||
it('does not let a superseded side effect overwrite the newer generation', async () => {
|
||||
const f = sideEffectFixture();
|
||||
Object.assign(f.projection, {
|
||||
revision: 12,
|
||||
matchSideEffectTransitionId: 'undo',
|
||||
matchSideEffectRevision: 6,
|
||||
});
|
||||
await f.scheduler.applyDurabilityPayload(f.payload);
|
||||
expect(f.reserveForMatch).not.toHaveBeenCalled();
|
||||
expect(f.projection.matchSideEffectTransitionId).toBe('undo');
|
||||
});
|
||||
|
||||
it('removes obsolete projection fields both from Mongo updates and subsequent undo snapshots', () => {
|
||||
const service = new AuctionService(
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const internal = service as unknown as {
|
||||
buildAuctionMongoProjectionFromState(
|
||||
auction: unknown,
|
||||
state: Record<string, unknown>,
|
||||
): Record<string, unknown>;
|
||||
buildAuctionSnapshotFromRedisState(
|
||||
auction: unknown,
|
||||
state: Record<string, unknown>,
|
||||
): Record<string, unknown>;
|
||||
};
|
||||
const old = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
matchId: new mongoose.Types.ObjectId(),
|
||||
completedAt: new Date(),
|
||||
auctionCompletedAtTimestampMS: 100,
|
||||
currentHighestBidderTeamIndex: 2,
|
||||
currentBidderTeamIndex: 2,
|
||||
};
|
||||
const projection = internal.buildAuctionMongoProjectionFromState(old, state);
|
||||
const mutation = buildAuctionProjectionGuardedUpdate({
|
||||
auctionId: old._id,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 12,
|
||||
afterAuctionMongoProjection: projection,
|
||||
} as AuctionDurabilityPayload).update;
|
||||
const stored = { ...old } as Record<string, unknown>;
|
||||
update(stored, mutation);
|
||||
const snapshot = internal.buildAuctionSnapshotFromRedisState(old, state);
|
||||
for (const record of [stored, snapshot]) {
|
||||
expect(record).not.toHaveProperty('completedAt');
|
||||
expect(record).not.toHaveProperty('currentHighestBidderTeamIndex');
|
||||
expect(record.phase).toBe('presenting');
|
||||
}
|
||||
});
|
||||
|
||||
it('applies delayed undo history cleanup without deleting newer branch entries or rolling back projection', async () => {
|
||||
const deleteMany = jest.fn().mockReturnValue(query({}));
|
||||
const updateOne = jest.fn().mockReturnValue(query({}));
|
||||
const projectionUpdate = jest.fn();
|
||||
const models: Record<string, unknown> = {
|
||||
auction: {
|
||||
findById: () => query({ auctionLifecycleId: 'life', revision: 12 }),
|
||||
updateOne: projectionUpdate,
|
||||
},
|
||||
auctionHistory: { findOne: () => query(null), deleteMany, updateOne },
|
||||
auctionSideEffectApplication: {},
|
||||
};
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => null,
|
||||
} as never,
|
||||
{} as never,
|
||||
);
|
||||
await scheduler.applyDurabilityPayload({
|
||||
auctionId: 'auction',
|
||||
matchId: 'match',
|
||||
auctionStateTransitionId: 'undo',
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 11,
|
||||
auctionHistorySequence: 9,
|
||||
auctionActionType: 'undo',
|
||||
historyMutation: {
|
||||
type: 'deleteAfterSequenceAndAppendUndo',
|
||||
targetHistorySequence: 3,
|
||||
},
|
||||
matchSideEffectMutation: { type: 'none' },
|
||||
afterAuctionMongoProjection: {},
|
||||
snapshotBefore: {},
|
||||
} as AuctionDurabilityPayload);
|
||||
expect(deleteMany).toHaveBeenCalledWith(
|
||||
{ auctionId: 'auction', sequence: { $gt: 3, $lt: 9 } },
|
||||
{ session: null },
|
||||
);
|
||||
expect(updateOne).toHaveBeenCalledTimes(1);
|
||||
expect(projectionUpdate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects a cancellation from a prior lifecycle before deleting anything', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId();
|
||||
const deleteOne = jest.fn();
|
||||
const service = new AuctionService(
|
||||
{
|
||||
createSession: async () => session(),
|
||||
getModel: async () => ({
|
||||
findOne: () => ({
|
||||
session: async () => ({
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: 'new-life',
|
||||
deleteOne,
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
await expect(
|
||||
service.reset({
|
||||
matchId,
|
||||
communityId,
|
||||
clientObservedAuctionLifecycleId: 'old-life',
|
||||
}),
|
||||
).rejects.toThrow('lifecycle has changed');
|
||||
expect(deleteOne).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('resumes startup after Mongo creation instead of returning an unfinished auction', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId();
|
||||
const captain = new mongoose.Types.ObjectId(),
|
||||
player = new mongoose.Types.ObjectId(),
|
||||
lateJoiner = new mongoose.Types.ObjectId();
|
||||
const auction = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
matchId,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 1,
|
||||
phase: 'presenting',
|
||||
startupCompleted: false,
|
||||
startAuctionRequestId: 'same-request',
|
||||
teams: [{ captainUserId: captain, members: [], teamIndex: 1 }],
|
||||
order: [{ userId: player }],
|
||||
};
|
||||
const match = {
|
||||
communityId,
|
||||
status: 'recruiting',
|
||||
participants: [captain, player, lateJoiner].map((userId) => ({
|
||||
userId,
|
||||
excluded: false,
|
||||
})),
|
||||
save: jest.fn(),
|
||||
};
|
||||
const updateOne = jest.fn();
|
||||
const historyUpdate = jest.fn();
|
||||
const models: Record<string, unknown> = {
|
||||
auction: { findOne: async () => auction, updateOne },
|
||||
match: { findById: async () => match },
|
||||
auctionHistory: { updateOne: historyUpdate },
|
||||
};
|
||||
const hydrateState = jest.fn().mockResolvedValue({ hydrated: true });
|
||||
const service = new AuctionService(
|
||||
{ getModel: async (name: string) => models[name] } as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{ hydrateState } as never,
|
||||
{} as never,
|
||||
);
|
||||
await service.start({
|
||||
matchId,
|
||||
communityId,
|
||||
startAuctionRequestId: 'same-request',
|
||||
} as never);
|
||||
expect(hydrateState).toHaveBeenCalledTimes(1);
|
||||
expect(historyUpdate).toHaveBeenCalledTimes(1);
|
||||
expect(match.save).toHaveBeenCalledTimes(1);
|
||||
expect(match.status).toBe('team-built');
|
||||
expect(auction.startupCompleted).toBe(true);
|
||||
expect(match.participants.map((entry) => entry.excluded)).toEqual([
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not open bidding until interrupted startup has been completed', async () => {
|
||||
const service = new AuctionService(
|
||||
{
|
||||
getModel: async () => ({
|
||||
findOne: async () => ({
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
startupCompleted: false,
|
||||
}),
|
||||
}),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
await expect(
|
||||
service.openBidding(new mongoose.Types.ObjectId()),
|
||||
).rejects.toThrow('startup is not complete');
|
||||
});
|
||||
|
||||
it('uses Redis completion state to schedule cleanup despite a delayed Mongo projection', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId();
|
||||
const auction = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
matchId,
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: 'life',
|
||||
phase: 'sold',
|
||||
revision: 9,
|
||||
};
|
||||
const snapshot = { ...state, phase: 'bidding', revision: 5 };
|
||||
const models: Record<string, unknown> = {
|
||||
auction: { findOne: () => ({ session: async () => auction }) },
|
||||
match: { findById: () => ({ session: async () => ({ communityId }) }) },
|
||||
auctionHistory: { findOne: () => query({ snapshotBefore: snapshot }) },
|
||||
};
|
||||
const applyTransition = jest.fn().mockResolvedValue({ ok: true });
|
||||
const commands = {
|
||||
reserveActionRequest: async () => ({
|
||||
reserved: true,
|
||||
actionRequestScopeKey: 'scope',
|
||||
auctionStateTransitionId: 'undo',
|
||||
observedProcessingLeaseVersion: 1,
|
||||
}),
|
||||
applyTransition,
|
||||
};
|
||||
const service = new AuctionService(
|
||||
{
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: async () => session(),
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
commands as never,
|
||||
{
|
||||
readAuctionStateView: async () => ({
|
||||
...state,
|
||||
phase: 'complete',
|
||||
revision: 10,
|
||||
}),
|
||||
} as never,
|
||||
);
|
||||
await service.undoTo({ matchId, communityId, sequence: 2 });
|
||||
const input = applyTransition.mock.calls[0][1] as {
|
||||
durabilityPayloadJson: string;
|
||||
};
|
||||
expect(JSON.parse(input.durabilityPayloadJson)).toMatchObject({
|
||||
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['deadline', 'durability'])(
|
||||
'continues %s ticks when lock release fails',
|
||||
async (kind) => {
|
||||
const redis = {
|
||||
acquireTokenLock: jest.fn().mockResolvedValue(true),
|
||||
releaseTokenLock: jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('redis disconnected')),
|
||||
};
|
||||
const scheduler =
|
||||
kind === 'deadline'
|
||||
? new AuctionDeadlineScheduler(
|
||||
redis as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
)
|
||||
: new AuctionDurabilityScheduler({} as never, redis as never);
|
||||
const work = jest.fn().mockResolvedValue({});
|
||||
Object.assign(
|
||||
scheduler,
|
||||
kind === 'deadline'
|
||||
? { processDueBidRoundDeadlines: work }
|
||||
: { processDueDurabilityPayloads: work },
|
||||
);
|
||||
const internal = scheduler as unknown as { safeTick(): Promise<void> };
|
||||
await internal.safeTick();
|
||||
await internal.safeTick();
|
||||
expect(work).toHaveBeenCalledTimes(2);
|
||||
},
|
||||
);
|
||||
@@ -88,10 +88,13 @@ export class AuctionDeadlineScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
);
|
||||
} finally {
|
||||
if (leaderLockAcquired) {
|
||||
await this.auctionRedisService.releaseTokenLock(
|
||||
BID_ROUND_DEADLINE_LEADER_LOCK_KEY,
|
||||
leaderLockToken,
|
||||
);
|
||||
await this.auctionRedisService
|
||||
.releaseTokenLock(BID_ROUND_DEADLINE_LEADER_LOCK_KEY, leaderLockToken)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Auction deadline lock release failed: ${String(error)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
this.tickInProgress = false;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
import { buildAuctionProjectionUnset } from './auction-projection';
|
||||
import {
|
||||
AuctionDurabilityScheduler,
|
||||
buildAuctionHistoryUpsert,
|
||||
@@ -23,6 +25,12 @@ const payload = {
|
||||
} as const;
|
||||
|
||||
describe('auction durability projection helpers', () => {
|
||||
beforeEach(() => {
|
||||
jest
|
||||
.spyOn(AuctionLifecycleLock.prototype, 'run')
|
||||
.mockImplementation(async (_id, work) => work());
|
||||
});
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
it('builds idempotent history upsert by auction and transition id', () => {
|
||||
expect(buildAuctionHistoryUpsert(payload)).toEqual({
|
||||
updateOne: {
|
||||
@@ -59,6 +67,9 @@ describe('auction durability projection helpers', () => {
|
||||
},
|
||||
update: {
|
||||
$set: { phase: 'sold', revision: 9 },
|
||||
$unset: buildAuctionProjectionUnset(
|
||||
payload.afterAuctionMongoProjection,
|
||||
),
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -210,7 +221,12 @@ describe('auction durability projection helpers', () => {
|
||||
auctionLifecycleId: 'life-1',
|
||||
revision: { $lt: 9 },
|
||||
},
|
||||
{ $set: { phase: 'sold', revision: 9 } },
|
||||
{
|
||||
$set: { phase: 'sold', revision: 9 },
|
||||
$unset: buildAuctionProjectionUnset(
|
||||
payload.afterAuctionMongoProjection,
|
||||
),
|
||||
},
|
||||
{ session },
|
||||
);
|
||||
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
@@ -370,7 +386,7 @@ describe('auction durability projection helpers', () => {
|
||||
expect(historyDeleteMany).toHaveBeenCalledWith(
|
||||
{
|
||||
auctionId: 'auction-1',
|
||||
sequence: { $gt: 2 },
|
||||
sequence: { $gt: 2, $lt: 4 },
|
||||
},
|
||||
{ session },
|
||||
);
|
||||
@@ -811,21 +827,14 @@ describe('auction durability projection helpers', () => {
|
||||
skipped: 0,
|
||||
});
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(5, [
|
||||
'ZREM',
|
||||
'EVAL',
|
||||
expect.stringContaining("redis.call('HEXISTS'"),
|
||||
'4',
|
||||
'payload-1',
|
||||
'auction:auctionDurabilityRetryQueue',
|
||||
'payload-1',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(6, [
|
||||
'HSET',
|
||||
'payload-1',
|
||||
'deadLetteredAtRedisTimestampMS',
|
||||
'1000',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(7, [
|
||||
'ZADD',
|
||||
'auction:auctionDurabilityDeadLetterQueue',
|
||||
'auction:deadLetterStatusQueue',
|
||||
'1000',
|
||||
'payload-1',
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -1030,11 +1039,20 @@ describe('auction durability projection helpers', () => {
|
||||
{
|
||||
_id: 'auction-1',
|
||||
auctionLifecycleId: 'life-1',
|
||||
matchSideEffectRevision: 1,
|
||||
$or: [
|
||||
{ matchSideEffectTransitionId: 'transition-1' },
|
||||
{
|
||||
matchSideEffectTransitionId: { $exists: false },
|
||||
matchSideEffectRevision: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
$inc: { matchSideEffectRevision: 1 },
|
||||
$set: { matchSideEffectStatus: 'processing' },
|
||||
$set: {
|
||||
matchSideEffectStatus: 'processing',
|
||||
matchSideEffectTransitionId: 'transition-1',
|
||||
},
|
||||
$unset: { matchSideEffectError: '' },
|
||||
},
|
||||
expect.objectContaining({ returnDocument: 'after' }),
|
||||
@@ -1045,34 +1063,42 @@ describe('auction durability projection helpers', () => {
|
||||
_id: 'auction-1',
|
||||
auctionLifecycleId: 'life-1',
|
||||
matchSideEffectRevision: 2,
|
||||
matchSideEffectTransitionId: 'transition-1',
|
||||
},
|
||||
{
|
||||
$inc: { matchSideEffectRevision: 1 },
|
||||
$set: { matchSideEffectStatus: 'completed' },
|
||||
$set: {
|
||||
matchSideEffectStatus: 'completed',
|
||||
matchSideEffectTransitionId: 'transition-1',
|
||||
},
|
||||
$unset: { matchSideEffectError: '' },
|
||||
},
|
||||
expect.objectContaining({ returnDocument: 'after' }),
|
||||
);
|
||||
expect(mirrorMatchSideEffectStatus).toHaveBeenCalledWith({
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
matchId: 'match-1',
|
||||
matchSideEffectRevision: 2,
|
||||
matchSideEffectStatus: 'processing',
|
||||
});
|
||||
expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
matchId: 'match-1',
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
});
|
||||
expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(1, 'match-1', {
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
currentServerTimestampMS: 2000,
|
||||
matchSideEffectRevision: 2,
|
||||
matchSideEffectStatus: 'processing',
|
||||
});
|
||||
expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(2, 'match-1', {
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
currentServerTimestampMS: 3000,
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
@@ -1080,7 +1106,7 @@ describe('auction durability projection helpers', () => {
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining("redis.call('EXPIRE'"),
|
||||
'11',
|
||||
'12',
|
||||
'auction:match-1:liveAuctionState',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'auction:match-1:auctionStructureState',
|
||||
@@ -1092,6 +1118,7 @@ describe('auction durability projection helpers', () => {
|
||||
'auction:match-1:auctionActionRequestResultOrder',
|
||||
'auction:match-1:rehydrateLock',
|
||||
'auction:match-1:auctionStateTransitionLock',
|
||||
'auction:match-1:rejectedBidRequestResultOrder',
|
||||
'match-1',
|
||||
'life-1',
|
||||
String(7 * 24 * 60 * 60),
|
||||
@@ -1263,7 +1290,12 @@ describe('auction durability projection helpers', () => {
|
||||
const broadcastMatchSideEffect = jest.fn();
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
{
|
||||
createSession: jest.fn().mockResolvedValue(null),
|
||||
createSession: jest.fn().mockResolvedValueOnce(null).mockResolvedValue({
|
||||
startTransaction: jest.fn(),
|
||||
commitTransaction: jest.fn(),
|
||||
abortTransaction: jest.fn(),
|
||||
endSession: jest.fn(),
|
||||
}),
|
||||
getModel: jest.fn((name: string) => {
|
||||
if (name === 'auctionHistory') {
|
||||
return {
|
||||
@@ -1282,7 +1314,11 @@ describe('auction durability projection helpers', () => {
|
||||
return { updateOne: sideEffectUpdateOne };
|
||||
}
|
||||
if (name === 'match')
|
||||
return { findById: jest.fn().mockReturnValue(match) };
|
||||
return {
|
||||
findById: jest.fn().mockReturnValue({
|
||||
session: jest.fn().mockResolvedValue(match),
|
||||
}),
|
||||
};
|
||||
return {};
|
||||
}),
|
||||
} as never,
|
||||
@@ -1301,8 +1337,8 @@ describe('auction durability projection helpers', () => {
|
||||
|
||||
await scheduler.applyDurabilityPayload(resetPayload);
|
||||
|
||||
expect(releaseFromMatch).toHaveBeenCalledWith(match);
|
||||
expect(cancelPolling).toHaveBeenCalledWith(match);
|
||||
expect(releaseFromMatch).toHaveBeenCalledWith(match, expect.any(Object));
|
||||
expect(cancelPolling).toHaveBeenCalledWith(match, expect.any(Object));
|
||||
expect(match).toMatchObject({
|
||||
gameSchedule: [],
|
||||
result: undefined,
|
||||
@@ -1328,12 +1364,14 @@ describe('auction durability projection helpers', () => {
|
||||
);
|
||||
expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
matchId: 'match-1',
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
});
|
||||
expect(broadcastMatchSideEffect).toHaveBeenLastCalledWith('match-1', {
|
||||
auctionLifecycleId: 'life-1',
|
||||
auctionStateTransitionId: 'transition-1',
|
||||
currentServerTimestampMS: 3000,
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: 'completed',
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
Optional,
|
||||
ConflictException,
|
||||
} from '@nestjs/common';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import type mongoose from 'mongoose';
|
||||
@@ -16,6 +17,10 @@ import type {
|
||||
import { TournamentCodeService } from '../tournament-code/tournament-code.service';
|
||||
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||
import { AuctionEventBusService } from './auction-event-bus.service';
|
||||
import { AuctionService } from './auction.service';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
import { buildAuctionProjectionUnset } from './auction-projection';
|
||||
import {
|
||||
AuctionSideEffectService,
|
||||
buildMatchSideEffectDelta,
|
||||
@@ -26,6 +31,7 @@ import { AuctionRedisService } from './redis/auction-redis.service';
|
||||
import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service';
|
||||
import {
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
buildAuctionRedisKeys,
|
||||
} from './redis/auction-redis.keys';
|
||||
@@ -36,6 +42,8 @@ const AUCTION_DURABILITY_BATCH_LIMIT = 20;
|
||||
const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000;
|
||||
const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000;
|
||||
const AUCTION_DURABILITY_MAX_ATTEMPTS = 10;
|
||||
const AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS = 120;
|
||||
const AUCTION_DURABILITY_MAX_PENDING_DURATION_MS = 10 * 60 * 1000;
|
||||
const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000;
|
||||
const AUCTION_DURABILITY_LEADER_LOCK_KEY =
|
||||
'auction:auctionDurabilityScheduler:leader';
|
||||
@@ -45,6 +53,7 @@ const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000;
|
||||
const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete';
|
||||
const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset';
|
||||
const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects';
|
||||
class SideEffectPendingError extends Error {}
|
||||
|
||||
export interface AuctionDurabilityPayload {
|
||||
actorUserId?: unknown;
|
||||
@@ -67,6 +76,7 @@ export interface AuctionDurabilityPayload {
|
||||
export interface AuctionProjectionIdentity {
|
||||
auctionLifecycleId?: string;
|
||||
revision?: number;
|
||||
matchSideEffectTransitionId?: string;
|
||||
}
|
||||
|
||||
type ParsedAuctionHistoryMutation =
|
||||
@@ -85,7 +95,7 @@ interface CleanupSideEffectMatchDocument {
|
||||
participants: Array<{ excluded?: boolean }>;
|
||||
result?: unknown;
|
||||
resultPollState?: unknown;
|
||||
save(): Promise<unknown>;
|
||||
save(options?: { session: mongoose.ClientSession }): Promise<unknown>;
|
||||
status: string;
|
||||
teamBuiltAt?: Date;
|
||||
teams?: unknown[];
|
||||
@@ -99,6 +109,7 @@ export class AuctionDurabilityScheduler
|
||||
private readonly logger = new Logger(AuctionDurabilityScheduler.name);
|
||||
private tickTimer?: NodeJS.Timeout;
|
||||
private tickInProgress = false;
|
||||
private deadLetterScanCursor = '0';
|
||||
|
||||
constructor(
|
||||
private readonly connectionService: ConnectionService,
|
||||
@@ -113,6 +124,9 @@ export class AuctionDurabilityScheduler
|
||||
private readonly auctionEventBusService?: AuctionEventBusService,
|
||||
@Optional()
|
||||
private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService,
|
||||
@Optional() private readonly auctionService?: AuctionService,
|
||||
@Optional()
|
||||
private readonly auctionBidArchiveService?: AuctionBidArchiveService,
|
||||
) {}
|
||||
|
||||
public onModuleInit(): void {
|
||||
@@ -187,6 +201,23 @@ export class AuctionDurabilityScheduler
|
||||
});
|
||||
await this.clearDurabilityPayload(payloadKey);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof SideEffectPendingError ||
|
||||
error instanceof ConflictException
|
||||
) {
|
||||
const disposition = await this.deferDurabilityPayload(
|
||||
payloadKey,
|
||||
currentRedisTimestampMS,
|
||||
error,
|
||||
);
|
||||
if (disposition === 'dead-lettered') {
|
||||
failed += 1;
|
||||
this.logger.warn(
|
||||
`Auction durability payload exceeded its pending limit: ${payloadKey}`,
|
||||
);
|
||||
} else skipped += 1;
|
||||
continue;
|
||||
}
|
||||
failed += 1;
|
||||
const failureDisposition = await this.recordDurabilityPayloadFailure(
|
||||
payloadKey,
|
||||
@@ -214,6 +245,14 @@ export class AuctionDurabilityScheduler
|
||||
const payloadJson = hash.payloadJson;
|
||||
if (!payloadJson) return null;
|
||||
const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload;
|
||||
const committedDeadline = Number(hash.committedBidRoundDeadlineTimestampMS);
|
||||
if (Number.isFinite(committedDeadline) && committedDeadline > 0) {
|
||||
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS =
|
||||
committedDeadline;
|
||||
payload.afterAuctionMongoProjection.currentDeadline = new Date(
|
||||
committedDeadline,
|
||||
).toISOString();
|
||||
}
|
||||
const committedAtRedisTimestampMS = Number(
|
||||
hash.committedAtRedisTimestampMS,
|
||||
);
|
||||
@@ -240,12 +279,12 @@ export class AuctionDurabilityScheduler
|
||||
const sideEffectMutation = parseMatchSideEffectMutation(
|
||||
payload.matchSideEffectMutation,
|
||||
);
|
||||
let shouldUpdateProjection = true;
|
||||
let shouldApplySideEffects = true;
|
||||
let shouldApplyHistoryMutation = true;
|
||||
|
||||
try {
|
||||
session?.startTransaction();
|
||||
const historyMutation = parseAuctionHistoryMutation(
|
||||
payload.historyMutation,
|
||||
);
|
||||
if (typeof AuctionModel.findById === 'function') {
|
||||
const currentProjection =
|
||||
await readAuctionProjectionIdentityForMutation(
|
||||
@@ -256,37 +295,44 @@ export class AuctionDurabilityScheduler
|
||||
if (
|
||||
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
|
||||
) {
|
||||
// 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다.
|
||||
// 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다.
|
||||
if (
|
||||
historyMutation.type === 'append' &&
|
||||
currentProjection.auctionLifecycleId === payload.auctionLifecycleId
|
||||
currentProjection.auctionLifecycleId !== payload.auctionLifecycleId
|
||||
) {
|
||||
await this.applyStaleAppendHistoryRecovery({
|
||||
if (session) await session.commitTransaction();
|
||||
return;
|
||||
}
|
||||
shouldApplyHistoryMutation =
|
||||
!(await this.isAppendHistoryInvalidatedByLaterDestructiveMutation({
|
||||
AuctionHistoryModel,
|
||||
payload,
|
||||
session,
|
||||
});
|
||||
}
|
||||
if (session) await session.commitTransaction();
|
||||
return;
|
||||
}));
|
||||
// History operations have their own sequence bounds. A newer ordinary
|
||||
// projection must not discard an earlier undo's history cleanup.
|
||||
shouldUpdateProjection = false;
|
||||
shouldApplySideEffects =
|
||||
Boolean(this.auctionRedisStateReaderService) ||
|
||||
currentProjection.matchSideEffectTransitionId ===
|
||||
payload.auctionStateTransitionId;
|
||||
}
|
||||
}
|
||||
await this.applyAuctionHistoryMutation({
|
||||
AuctionHistoryModel,
|
||||
AuctionModel,
|
||||
payload,
|
||||
session,
|
||||
});
|
||||
if (shouldApplyHistoryMutation)
|
||||
await this.applyAuctionHistoryMutation({
|
||||
AuctionHistoryModel,
|
||||
AuctionModel,
|
||||
payload,
|
||||
session,
|
||||
});
|
||||
const projectionOperation = buildAuctionProjectionGuardedUpdate(payload);
|
||||
await execMaybe(
|
||||
AuctionModel.updateOne(
|
||||
projectionOperation.filter as never,
|
||||
projectionOperation.update as never,
|
||||
{ session } as never,
|
||||
),
|
||||
);
|
||||
if (sideEffectMutation) {
|
||||
if (shouldUpdateProjection)
|
||||
await execMaybe(
|
||||
AuctionModel.updateOne(
|
||||
projectionOperation.filter as never,
|
||||
projectionOperation.update as never,
|
||||
{ session } as never,
|
||||
),
|
||||
);
|
||||
if (sideEffectMutation && shouldApplySideEffects) {
|
||||
const sideEffectOperation = buildSideEffectApplicationUpsert(
|
||||
payload,
|
||||
sideEffectMutation,
|
||||
@@ -312,7 +358,7 @@ export class AuctionDurabilityScheduler
|
||||
|
||||
await this.broadcastLatestRedisAuctionStateIfRequested(payload);
|
||||
|
||||
if (sideEffectMutation) {
|
||||
if (sideEffectMutation && shouldApplySideEffects) {
|
||||
await this.applyMatchSideEffect(payload, sideEffectMutation);
|
||||
}
|
||||
}
|
||||
@@ -352,6 +398,64 @@ export class AuctionDurabilityScheduler
|
||||
private async applyMatchSideEffect(
|
||||
payload: AuctionDurabilityPayload,
|
||||
mutation: ParsedMatchSideEffectMutation,
|
||||
): Promise<void> {
|
||||
return new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
String(payload.matchId),
|
||||
async () => {
|
||||
if (this.auctionRedisStateReaderService) {
|
||||
const state =
|
||||
await this.auctionRedisStateReaderService.readAuctionStateView(
|
||||
String(payload.matchId),
|
||||
);
|
||||
if (!state)
|
||||
throw new SideEffectPendingError(
|
||||
'Auction realtime state is unavailable.',
|
||||
);
|
||||
if (
|
||||
state.auctionLifecycleId !== payload.auctionLifecycleId ||
|
||||
(state.matchSideEffectTransitionId &&
|
||||
state.matchSideEffectTransitionId !==
|
||||
payload.auctionStateTransitionId) ||
|
||||
(mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE &&
|
||||
state.phase !== 'complete')
|
||||
)
|
||||
return;
|
||||
// An ordinary later projection may have reached Mongo before this
|
||||
// side-effect-bearing transition. Install its pending generation only
|
||||
// while Redis still names it and Mongo has not advanced beyond it.
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
const pendingRevision = Number(
|
||||
payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0,
|
||||
);
|
||||
await execMaybe(
|
||||
AuctionModel.updateOne(
|
||||
{
|
||||
_id: payload.auctionId,
|
||||
auctionLifecycleId: payload.auctionLifecycleId,
|
||||
$or: [
|
||||
{ matchSideEffectRevision: { $lt: pendingRevision } },
|
||||
{ matchSideEffectRevision: { $exists: false } },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
matchSideEffectTransitionId: payload.auctionStateTransitionId,
|
||||
matchSideEffectRevision: pendingRevision,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
$unset: { matchSideEffectError: '' },
|
||||
} as never,
|
||||
),
|
||||
);
|
||||
}
|
||||
await this.applyMatchSideEffectWhileLocked(payload, mutation);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async applyMatchSideEffectWhileLocked(
|
||||
payload: AuctionDurabilityPayload,
|
||||
mutation: ParsedMatchSideEffectMutation,
|
||||
): Promise<void> {
|
||||
if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) {
|
||||
await this.applySyncMatchTeamsSideEffect(payload, mutation);
|
||||
@@ -407,7 +511,26 @@ export class AuctionDurabilityScheduler
|
||||
}) as never,
|
||||
),
|
||||
);
|
||||
if (!hasMatchedDocument(leaseResult)) return;
|
||||
if (!hasMatchedDocument(leaseResult)) {
|
||||
const existing = await AuctionSideEffectApplicationModel.findOne(
|
||||
identity as never,
|
||||
)
|
||||
.lean()
|
||||
.exec();
|
||||
if (existing?.state === 'completed') {
|
||||
await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
payload,
|
||||
status: 'completed',
|
||||
});
|
||||
await this.expireCompletedAuctionRedisRuntime(payload);
|
||||
return;
|
||||
}
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side effect is still owned by another worker.',
|
||||
);
|
||||
}
|
||||
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
|
||||
{
|
||||
@@ -420,7 +543,11 @@ export class AuctionDurabilityScheduler
|
||||
status: 'processing',
|
||||
},
|
||||
);
|
||||
if (!processingDelta) return;
|
||||
if (!processingDelta) {
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side-effect generation is not ready.',
|
||||
);
|
||||
}
|
||||
const expectedMatchSideEffectRevision =
|
||||
processingDelta.matchSideEffectRevision;
|
||||
|
||||
@@ -489,7 +616,11 @@ export class AuctionDurabilityScheduler
|
||||
await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
completedAtTimestampMS,
|
||||
@@ -518,7 +649,11 @@ export class AuctionDurabilityScheduler
|
||||
await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
lastError: error instanceof Error ? error.message : String(error),
|
||||
@@ -544,33 +679,6 @@ export class AuctionDurabilityScheduler
|
||||
}
|
||||
}
|
||||
|
||||
private async applyStaleAppendHistoryRecovery(params: {
|
||||
AuctionHistoryModel: {
|
||||
findOne?: (...args: never[]) => unknown;
|
||||
updateOne: (...args: never[]) => unknown;
|
||||
};
|
||||
payload: AuctionDurabilityPayload;
|
||||
session: mongoose.ClientSession | null | undefined;
|
||||
}): Promise<void> {
|
||||
const invalidated =
|
||||
await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params);
|
||||
if (invalidated) return;
|
||||
|
||||
const historyOperation = buildAuctionHistoryUpsert(
|
||||
params.payload,
|
||||
).updateOne;
|
||||
await execMaybe(
|
||||
params.AuctionHistoryModel.updateOne(
|
||||
historyOperation.filter as never,
|
||||
historyOperation.update as never,
|
||||
{
|
||||
session: params.session,
|
||||
upsert: true,
|
||||
} as never,
|
||||
) as Promise<unknown> | { exec(): Promise<unknown> },
|
||||
);
|
||||
}
|
||||
|
||||
private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: {
|
||||
AuctionHistoryModel: {
|
||||
findOne?: (...args: never[]) => unknown;
|
||||
@@ -622,9 +730,7 @@ export class AuctionDurabilityScheduler
|
||||
await this.applyMatchCleanupSideEffect({
|
||||
mutation,
|
||||
payload,
|
||||
updateMatch: async (match) => {
|
||||
await this.tournamentCodeService?.releaseFromMatch(match as never);
|
||||
await this.tournamentResultService?.cancelPolling(match as never);
|
||||
updateMatch: (match) => {
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
@@ -635,7 +741,6 @@ export class AuctionDurabilityScheduler
|
||||
match.teamBuiltAt = undefined;
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -647,13 +752,14 @@ export class AuctionDurabilityScheduler
|
||||
await this.applyMatchCleanupSideEffect({
|
||||
mutation,
|
||||
payload,
|
||||
updateMatch: async (match) => {
|
||||
await this.tournamentCodeService?.releaseFromMatch(match as never);
|
||||
await this.tournamentResultService?.cancelPolling(match as never);
|
||||
updateMatch: (match) => {
|
||||
match.status = 'team-built';
|
||||
match.teams = [];
|
||||
match.teamBuiltAt = undefined;
|
||||
match.tournamentCodeIds = [];
|
||||
match.gameSchedule = [];
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -661,7 +767,7 @@ export class AuctionDurabilityScheduler
|
||||
private async applyMatchCleanupSideEffect(params: {
|
||||
mutation: ParsedMatchSideEffectMutation;
|
||||
payload: AuctionDurabilityPayload;
|
||||
updateMatch: (match: CleanupSideEffectMatchDocument) => Promise<void>;
|
||||
updateMatch: (match: CleanupSideEffectMatchDocument) => void;
|
||||
}): Promise<void> {
|
||||
if (!this.tournamentCodeService || !this.tournamentResultService) {
|
||||
throw new Error('Auction side-effect services are not available.');
|
||||
@@ -700,7 +806,25 @@ export class AuctionDurabilityScheduler
|
||||
}) as never,
|
||||
),
|
||||
);
|
||||
if (!hasMatchedDocument(leaseResult)) return;
|
||||
if (!hasMatchedDocument(leaseResult)) {
|
||||
const existing = await AuctionSideEffectApplicationModel.findOne(
|
||||
identity as never,
|
||||
)
|
||||
.lean()
|
||||
.exec();
|
||||
if (existing?.state === 'completed') {
|
||||
await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel,
|
||||
currentServerTimestampMS: currentRedisTimestampMS,
|
||||
payload: params.payload,
|
||||
status: 'completed',
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side effect is still owned by another worker.',
|
||||
);
|
||||
}
|
||||
|
||||
const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus(
|
||||
{
|
||||
@@ -714,29 +838,56 @@ export class AuctionDurabilityScheduler
|
||||
status: 'processing',
|
||||
},
|
||||
);
|
||||
if (!processingDelta) return;
|
||||
if (!processingDelta) {
|
||||
throw new SideEffectPendingError(
|
||||
'Auction side-effect generation is not ready.',
|
||||
);
|
||||
}
|
||||
const expectedMatchSideEffectRevision =
|
||||
processingDelta.matchSideEffectRevision;
|
||||
|
||||
try {
|
||||
const match = (await execMaybe(
|
||||
MatchModel.findById(params.payload.matchId) as never,
|
||||
)) as CleanupSideEffectMatchDocument | null;
|
||||
if (!match) {
|
||||
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
|
||||
}
|
||||
if (match.competitionId) {
|
||||
throw new Error(
|
||||
'Cannot reset or undo Match teams after the immutable Competition has been created.',
|
||||
const session = await this.connectionService.createSession();
|
||||
if (!session)
|
||||
throw new Error('Failed to start Match cleanup transaction.');
|
||||
try {
|
||||
session.startTransaction();
|
||||
const match = (await MatchModel.findById(
|
||||
params.payload.matchId,
|
||||
).session(session)) as CleanupSideEffectMatchDocument | null;
|
||||
if (!match)
|
||||
throw new Error(`Match not found: ${String(params.payload.matchId)}`);
|
||||
if (match.competitionId)
|
||||
throw new Error(
|
||||
'Cannot reset or undo Match teams after the immutable Competition has been created.',
|
||||
);
|
||||
await this.tournamentCodeService.releaseFromMatch(
|
||||
match as never,
|
||||
session,
|
||||
);
|
||||
await this.tournamentResultService.cancelPolling(
|
||||
match as never,
|
||||
session,
|
||||
);
|
||||
params.updateMatch(match);
|
||||
await match.save({ session });
|
||||
await session.commitTransaction();
|
||||
} catch (error) {
|
||||
await session.abortTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
await params.updateMatch(match);
|
||||
|
||||
const completedAtTimestampMS =
|
||||
await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
completedAtTimestampMS,
|
||||
@@ -766,7 +917,11 @@ export class AuctionDurabilityScheduler
|
||||
error instanceof Error ? error.message : String(error);
|
||||
await execMaybe(
|
||||
AuctionSideEffectApplicationModel.updateOne(
|
||||
identity as never,
|
||||
{
|
||||
...identity,
|
||||
processingOwnerToken: ownerToken,
|
||||
state: 'processing',
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
lastError: errorMessage,
|
||||
@@ -828,7 +983,10 @@ export class AuctionDurabilityScheduler
|
||||
params.AuctionHistoryModel.deleteMany(
|
||||
{
|
||||
auctionId: params.payload.auctionId,
|
||||
sequence: { $gt: mutation.targetHistorySequence },
|
||||
sequence: {
|
||||
$gt: mutation.targetHistorySequence,
|
||||
$lt: params.payload.auctionHistorySequence,
|
||||
},
|
||||
} as never,
|
||||
{ session: params.session } as never,
|
||||
) as Promise<unknown> | { exec(): Promise<unknown> },
|
||||
@@ -842,6 +1000,7 @@ export class AuctionDurabilityScheduler
|
||||
params.AuctionHistoryModel.deleteMany(
|
||||
{
|
||||
auctionId: params.payload.auctionId,
|
||||
sequence: { $lt: params.payload.auctionHistorySequence },
|
||||
} as never,
|
||||
{ session: params.session } as never,
|
||||
) as Promise<unknown> | { exec(): Promise<unknown> },
|
||||
@@ -874,16 +1033,43 @@ export class AuctionDurabilityScheduler
|
||||
};
|
||||
currentServerTimestampMS: number;
|
||||
errorMessage?: string;
|
||||
expectedMatchSideEffectRevision: number;
|
||||
expectedMatchSideEffectRevision?: number;
|
||||
payload: AuctionDurabilityPayload;
|
||||
status: AuctionMatchSideEffectStatus;
|
||||
onlyUnfinished?: boolean;
|
||||
}): Promise<MatchSideEffectDelta | null> {
|
||||
const updatedAuction = (await execMaybe(
|
||||
params.AuctionModel.findOneAndUpdate(
|
||||
{
|
||||
_id: params.payload.auctionId,
|
||||
auctionLifecycleId: params.payload.auctionLifecycleId,
|
||||
matchSideEffectRevision: params.expectedMatchSideEffectRevision,
|
||||
...(params.onlyUnfinished
|
||||
? { matchSideEffectStatus: { $ne: 'completed' } }
|
||||
: {}),
|
||||
...(params.status === 'processing'
|
||||
? {
|
||||
$or: [
|
||||
{
|
||||
matchSideEffectTransitionId:
|
||||
params.payload.auctionStateTransitionId,
|
||||
},
|
||||
{
|
||||
matchSideEffectTransitionId: { $exists: false },
|
||||
matchSideEffectRevision:
|
||||
params.expectedMatchSideEffectRevision,
|
||||
},
|
||||
],
|
||||
}
|
||||
: {
|
||||
matchSideEffectTransitionId:
|
||||
params.payload.auctionStateTransitionId,
|
||||
...(params.expectedMatchSideEffectRevision === undefined
|
||||
? {}
|
||||
: {
|
||||
matchSideEffectRevision:
|
||||
params.expectedMatchSideEffectRevision,
|
||||
}),
|
||||
}),
|
||||
} as never,
|
||||
{
|
||||
$inc: { matchSideEffectRevision: 1 },
|
||||
@@ -892,6 +1078,8 @@ export class AuctionDurabilityScheduler
|
||||
? { matchSideEffectError: params.errorMessage }
|
||||
: {}),
|
||||
matchSideEffectStatus: params.status,
|
||||
matchSideEffectTransitionId:
|
||||
params.payload.auctionStateTransitionId,
|
||||
},
|
||||
...(params.errorMessage
|
||||
? {}
|
||||
@@ -915,6 +1103,7 @@ export class AuctionDurabilityScheduler
|
||||
|
||||
const delta = buildMatchSideEffectDelta({
|
||||
auctionLifecycleId: updatedAuction.auctionLifecycleId,
|
||||
auctionStateTransitionId: params.payload.auctionStateTransitionId,
|
||||
currentServerTimestampMS: params.currentServerTimestampMS,
|
||||
matchSideEffectRevision: Number(
|
||||
updatedAuction.matchSideEffectRevision ?? 0,
|
||||
@@ -923,19 +1112,170 @@ export class AuctionDurabilityScheduler
|
||||
updatedAuction.matchSideEffectStatus ?? params.status,
|
||||
});
|
||||
|
||||
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
|
||||
auctionLifecycleId: delta.auctionLifecycleId,
|
||||
matchId: String(params.payload.matchId),
|
||||
matchSideEffectRevision: delta.matchSideEffectRevision,
|
||||
matchSideEffectStatus: delta.matchSideEffectStatus,
|
||||
});
|
||||
this.auctionEventBusService?.broadcastMatchSideEffect(
|
||||
String(params.payload.matchId),
|
||||
delta,
|
||||
);
|
||||
const mirrored =
|
||||
await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({
|
||||
auctionLifecycleId: delta.auctionLifecycleId,
|
||||
auctionStateTransitionId: params.payload.auctionStateTransitionId,
|
||||
matchId: String(params.payload.matchId),
|
||||
matchSideEffectRevision: delta.matchSideEffectRevision,
|
||||
matchSideEffectStatus: delta.matchSideEffectStatus,
|
||||
});
|
||||
if (mirrored !== false) {
|
||||
this.auctionEventBusService?.broadcastMatchSideEffect(
|
||||
String(params.payload.matchId),
|
||||
delta,
|
||||
);
|
||||
}
|
||||
return delta;
|
||||
}
|
||||
|
||||
public async reconcileDeadLetterStatuses(limit = 20): Promise<void> {
|
||||
const client = this.auctionRedisService.raw();
|
||||
const now = await this.auctionRedisService.getCurrentRedisTimestampMS();
|
||||
// Also reconcile entries dead-lettered before status notifications existed.
|
||||
const scanned = await client.sendCommand([
|
||||
'ZSCAN',
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
this.deadLetterScanCursor,
|
||||
'COUNT',
|
||||
String(limit),
|
||||
]);
|
||||
if (Array.isArray(scanned) && Array.isArray(scanned[1])) {
|
||||
this.deadLetterScanCursor = String(scanned[0]);
|
||||
for (let index = 0; index < scanned[1].length; index += 2) {
|
||||
const key = String(scanned[1][index]);
|
||||
if (
|
||||
!(await client.sendCommand([
|
||||
'HGET',
|
||||
key,
|
||||
'deadLetterStatusReconciledAtTimestampMS',
|
||||
]))
|
||||
) {
|
||||
await client.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
'NX',
|
||||
String(now),
|
||||
key,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
const reply = await client.sendCommand([
|
||||
'ZRANGEBYSCORE',
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
'-inf',
|
||||
String(now),
|
||||
'LIMIT',
|
||||
'0',
|
||||
String(limit),
|
||||
]);
|
||||
if (!Array.isArray(reply)) return;
|
||||
for (const key of reply.map(String)) {
|
||||
try {
|
||||
const payload = await this.readDurabilityPayload(key);
|
||||
if (
|
||||
payload &&
|
||||
parseMatchSideEffectMutation(payload.matchSideEffectMutation)
|
||||
) {
|
||||
const state =
|
||||
await this.auctionRedisStateReaderService?.readAuctionStateView(
|
||||
String(payload.matchId),
|
||||
);
|
||||
// A terminal notification for a previous generation must not change
|
||||
// a new completion/undo, even when Mongo is behind Redis.
|
||||
const superseded =
|
||||
state &&
|
||||
(state.auctionLifecycleId !== payload.auctionLifecycleId ||
|
||||
state.matchSideEffectStatus === 'completed' ||
|
||||
(state.matchSideEffectTransitionId &&
|
||||
state.matchSideEffectTransitionId !==
|
||||
payload.auctionStateTransitionId));
|
||||
if (!superseded) {
|
||||
const AuctionModel =
|
||||
await this.connectionService.getModel('auction');
|
||||
if (
|
||||
state?.matchSideEffectTransitionId ===
|
||||
payload.auctionStateTransitionId
|
||||
) {
|
||||
const revision = Number(
|
||||
payload.afterAuctionMongoProjection.matchSideEffectRevision ??
|
||||
0,
|
||||
);
|
||||
await AuctionModel.updateOne(
|
||||
{
|
||||
_id: payload.auctionId,
|
||||
auctionLifecycleId: payload.auctionLifecycleId,
|
||||
$or: [
|
||||
{ matchSideEffectRevision: { $lt: revision } },
|
||||
{ matchSideEffectRevision: { $exists: false } },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
$set: {
|
||||
matchSideEffectTransitionId:
|
||||
payload.auctionStateTransitionId,
|
||||
matchSideEffectRevision: revision,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
},
|
||||
);
|
||||
// Heal a same-generation revision mismatch before sending the
|
||||
// terminal status; otherwise Redis could ignore it as older.
|
||||
await AuctionModel.updateOne(
|
||||
{
|
||||
_id: payload.auctionId,
|
||||
auctionLifecycleId: payload.auctionLifecycleId,
|
||||
matchSideEffectTransitionId: payload.auctionStateTransitionId,
|
||||
matchSideEffectStatus: { $ne: 'completed' },
|
||||
} as never,
|
||||
{
|
||||
$max: {
|
||||
matchSideEffectRevision: Number(
|
||||
state.matchSideEffectRevision ?? revision,
|
||||
),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
const error = await client.sendCommand(['HGET', key, 'lastError']);
|
||||
await this.persistAndBroadcastMatchSideEffectStatus({
|
||||
AuctionModel: AuctionModel as never,
|
||||
payload,
|
||||
currentServerTimestampMS: now,
|
||||
status: 'failed',
|
||||
onlyUnfinished: true,
|
||||
errorMessage:
|
||||
typeof error === 'string'
|
||||
? error
|
||||
: 'Auction side effect exhausted its retry limit.',
|
||||
});
|
||||
}
|
||||
}
|
||||
await client.sendCommand([
|
||||
'EVAL',
|
||||
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 1 then redis.call('HSET', KEYS[1], 'deadLetterStatusReconciledAtTimestampMS', ARGV[1]) end redis.call('ZREM', KEYS[2], KEYS[1]); return 1",
|
||||
'2',
|
||||
key,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(now),
|
||||
]);
|
||||
} catch (error) {
|
||||
// Mongo or Redis mirroring may be unavailable at dead-letter time. Keep
|
||||
// the notification durable until the failure can be shown to viewers.
|
||||
await client.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(now + 5000),
|
||||
key,
|
||||
]);
|
||||
this.logger.warn(
|
||||
`Auction terminal status will retry for ${key}: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async safeTick(): Promise<void> {
|
||||
if (this.tickInProgress) return;
|
||||
this.tickInProgress = true;
|
||||
@@ -948,7 +1288,22 @@ export class AuctionDurabilityScheduler
|
||||
AUCTION_DURABILITY_LEADER_LOCK_TTL_MS,
|
||||
);
|
||||
if (!leaderLockAcquired) return;
|
||||
await this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT);
|
||||
// A failed maintenance task must not starve the other durable queues.
|
||||
for (const work of [
|
||||
() => this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT),
|
||||
() => this.reconcileDeadLetterStatuses(AUCTION_DURABILITY_BATCH_LIMIT),
|
||||
() => this.auctionBidArchiveService?.persistPendingBids(),
|
||||
() =>
|
||||
this.auctionService?.retryPendingCancellations(
|
||||
AUCTION_DURABILITY_BATCH_LIMIT,
|
||||
),
|
||||
]) {
|
||||
try {
|
||||
await work();
|
||||
} catch (error) {
|
||||
this.logger.warn(`Auction maintenance task failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Auction durability tick failed: ${
|
||||
@@ -957,10 +1312,13 @@ export class AuctionDurabilityScheduler
|
||||
);
|
||||
} finally {
|
||||
if (leaderLockAcquired) {
|
||||
await this.auctionRedisService.releaseTokenLock(
|
||||
AUCTION_DURABILITY_LEADER_LOCK_KEY,
|
||||
leaderLockToken,
|
||||
);
|
||||
await this.auctionRedisService
|
||||
.releaseTokenLock(AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Auction durability lock release failed: ${String(error)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
this.tickInProgress = false;
|
||||
}
|
||||
@@ -987,6 +1345,51 @@ export class AuctionDurabilityScheduler
|
||||
]);
|
||||
}
|
||||
|
||||
private async deferDurabilityPayload(
|
||||
payloadKey: string,
|
||||
timestampMS: number,
|
||||
error: unknown,
|
||||
): Promise<'gone' | 'requeued' | 'dead-lettered'> {
|
||||
// Keep coordination waits separate from failed work, but bound them across
|
||||
// restarts too. Updating the payload and queue atomically avoids orphan jobs.
|
||||
const result = await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'EVAL',
|
||||
[
|
||||
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end",
|
||||
"local count = redis.call('HINCRBY', KEYS[1], 'pendingAttempts', 1)",
|
||||
"redis.call('HSETNX', KEYS[1], 'pendingSinceTimestampMS', ARGV[1])",
|
||||
"local since = tonumber(redis.call('HGET', KEYS[1], 'pendingSinceTimestampMS'))",
|
||||
"redis.call('HSET', KEYS[1], 'lastPendingReason', ARGV[5], 'lastPendingAtTimestampMS', ARGV[1])",
|
||||
'if count >= tonumber(ARGV[3]) or tonumber(ARGV[1]) - since >= tonumber(ARGV[4]) then',
|
||||
" redis.call('ZREM', KEYS[2], KEYS[1])",
|
||||
" redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1], 'lastError', ARGV[5])",
|
||||
" redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1])",
|
||||
" redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1])",
|
||||
' return 2',
|
||||
'end',
|
||||
"redis.call('ZADD', KEYS[2], tonumber(ARGV[1]) + tonumber(ARGV[2]), KEYS[1])",
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'4',
|
||||
payloadKey,
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(timestampMS),
|
||||
String(AUCTION_DURABILITY_RETRY_DELAY_MS),
|
||||
String(AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS),
|
||||
String(AUCTION_DURABILITY_MAX_PENDING_DURATION_MS),
|
||||
formatErrorMessageForRedis(error),
|
||||
]);
|
||||
return Number(result) === 2
|
||||
? 'dead-lettered'
|
||||
: Number(result) === 1
|
||||
? 'requeued'
|
||||
: 'gone';
|
||||
}
|
||||
|
||||
private async recordDurabilityPayloadFailure(
|
||||
payloadKey: string,
|
||||
currentRedisTimestampMS: number,
|
||||
@@ -1012,24 +1415,17 @@ export class AuctionDurabilityScheduler
|
||||
Number.isFinite(attempts) &&
|
||||
attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS
|
||||
) {
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'HSET',
|
||||
'EVAL',
|
||||
"if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1]); redis.call('ZREM', KEYS[2], KEYS[1]); redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1]); redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1]); return 1",
|
||||
'4',
|
||||
payloadKey,
|
||||
'deadLetteredAtRedisTimestampMS',
|
||||
String(currentRedisTimestampMS),
|
||||
]);
|
||||
await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand([
|
||||
'ZADD',
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
String(currentRedisTimestampMS),
|
||||
payloadKey,
|
||||
]);
|
||||
return 'dead-lettered';
|
||||
}
|
||||
@@ -1064,7 +1460,7 @@ export class AuctionDurabilityScheduler
|
||||
'end',
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'11',
|
||||
'12',
|
||||
keys.liveStateKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.structureStateKey,
|
||||
@@ -1076,6 +1472,7 @@ export class AuctionDurabilityScheduler
|
||||
keys.actionRequestResultOrderKey,
|
||||
keys.rehydrateLockKey,
|
||||
keys.stateTransitionLockKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
matchId,
|
||||
payload.auctionLifecycleId,
|
||||
String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS),
|
||||
@@ -1159,6 +1556,7 @@ async function readAuctionProjectionIdentityForMutation(
|
||||
? (query as { select: (...args: never[]) => unknown }).select({
|
||||
auctionLifecycleId: 1,
|
||||
revision: 1,
|
||||
matchSideEffectTransitionId: 1,
|
||||
} as never)
|
||||
: query;
|
||||
const scoped =
|
||||
@@ -1178,6 +1576,10 @@ async function readAuctionProjectionIdentityForMutation(
|
||||
if (!projection || typeof projection !== 'object') return {};
|
||||
const record = projection as Record<string, unknown>;
|
||||
return {
|
||||
matchSideEffectTransitionId:
|
||||
typeof record.matchSideEffectTransitionId === 'string'
|
||||
? record.matchSideEffectTransitionId
|
||||
: undefined,
|
||||
auctionLifecycleId:
|
||||
typeof record.auctionLifecycleId === 'string'
|
||||
? record.auctionLifecycleId
|
||||
@@ -1200,6 +1602,7 @@ export function buildAuctionProjectionGuardedUpdate(
|
||||
},
|
||||
update: {
|
||||
$set: payload.afterAuctionMongoProjection,
|
||||
$unset: buildAuctionProjectionUnset(payload.afterAuctionMongoProjection),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -24,4 +24,10 @@ export class AuctionEventBusService {
|
||||
?.to(matchId)
|
||||
.emit('state', serializeRealtimeAuction(state));
|
||||
}
|
||||
|
||||
public broadcastReset(matchId: string, auctionLifecycleId: string): void {
|
||||
this.auctionNamespace
|
||||
?.to(matchId)
|
||||
.emit('reset', { matchId, auctionLifecycleId });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { AuctionLifecycleLock } from './auction-lifecycle-lock';
|
||||
|
||||
afterEach(() => jest.useRealTimers());
|
||||
|
||||
it('renews ownership and handles renewal/release failures without unhandled rejection', async () => {
|
||||
jest.useFakeTimers();
|
||||
const sendCommand = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('redis disconnected'));
|
||||
const releaseTokenLock = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('release failed'));
|
||||
const lock = new AuctionLifecycleLock({
|
||||
acquireTokenLock: jest.fn().mockResolvedValue(true),
|
||||
releaseTokenLock,
|
||||
raw: () => ({ sendCommand }),
|
||||
} as never);
|
||||
const result = lock.run('match', async () => {
|
||||
await jest.advanceTimersByTimeAsync(10000);
|
||||
});
|
||||
await expect(result).rejects.toThrow('redis disconnected');
|
||||
expect(releaseTokenLock).toHaveBeenCalledTimes(1);
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('preserves an operation result if only lock release fails', async () => {
|
||||
const lock = new AuctionLifecycleLock({
|
||||
acquireTokenLock: jest.fn().mockResolvedValue(true),
|
||||
releaseTokenLock: jest.fn().mockRejectedValue(new Error('release failed')),
|
||||
} as never);
|
||||
await expect(lock.run('match', async () => 42)).resolves.toBe(42);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { ConflictException, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { AuctionRedisService } from './redis/auction-redis.service';
|
||||
|
||||
export class AuctionLifecycleBusyError extends ConflictException {
|
||||
constructor() {
|
||||
super({
|
||||
code: 'AUCTION_LIFECYCLE_BUSY',
|
||||
message: '다른 경매 작업을 처리 중입니다. 잠시 후 다시 시도해 주세요.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Serializes startup, cancellation, undo and Match side effects across nodes. */
|
||||
export class AuctionLifecycleLock {
|
||||
private readonly logger = new Logger(AuctionLifecycleLock.name);
|
||||
|
||||
constructor(private readonly redis: AuctionRedisService) {}
|
||||
|
||||
async run<T>(matchId: string, operation: () => Promise<T>): Promise<T> {
|
||||
// Kept outside runtime cleanup: cancelling an auction must not release a
|
||||
// lock still held by its cancellation/recovery operation.
|
||||
const key = `auction:${matchId}:lifecycleMutationLock`;
|
||||
const token = randomUUID();
|
||||
if (!(await this.redis.acquireTokenLock(key, token, 30_000))) {
|
||||
throw new AuctionLifecycleBusyError();
|
||||
}
|
||||
let renewal: Promise<void> | undefined;
|
||||
let ownershipError: Error | undefined;
|
||||
const timer = setInterval(() => {
|
||||
if (renewal) return;
|
||||
renewal = this.redis
|
||||
.raw()
|
||||
.sendCommand<string | null>([
|
||||
'SET',
|
||||
key,
|
||||
token,
|
||||
'IFEQ',
|
||||
token,
|
||||
'PX',
|
||||
'30000',
|
||||
])
|
||||
.then((result) => {
|
||||
if (result !== 'OK')
|
||||
throw new Error('Auction lifecycle lock ownership was lost.');
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
ownershipError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
})
|
||||
.finally(() => {
|
||||
renewal = undefined;
|
||||
});
|
||||
}, 10_000);
|
||||
timer.unref?.();
|
||||
try {
|
||||
const result = await operation();
|
||||
clearInterval(timer);
|
||||
await renewal;
|
||||
if (ownershipError) throw ownershipError;
|
||||
return result;
|
||||
} finally {
|
||||
clearInterval(timer);
|
||||
await renewal;
|
||||
await this.redis.releaseTokenLock(key, token).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Auction lifecycle lock release failed: ${String(error)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
/** Missing optional fields in a complete projection mean deletion, not retention. */
|
||||
export const AUCTION_OPTIONAL_PROJECTION_FIELDS = [
|
||||
'completedAt',
|
||||
'auctionCompletedAtTimestampMS',
|
||||
'currentBidderTeamIndex',
|
||||
'currentHighestBidderTeamIndex',
|
||||
'currentDeadline',
|
||||
'currentBidRoundDeadlineTimestampMS',
|
||||
'currentBidRoundId',
|
||||
] as const;
|
||||
|
||||
export function buildAuctionProjectionUnset(
|
||||
projection: Record<string, unknown>,
|
||||
) {
|
||||
return Object.fromEntries(
|
||||
AUCTION_OPTIONAL_PROJECTION_FIELDS.filter(
|
||||
(field) => projection[field] === undefined,
|
||||
).map((field) => [field, '']),
|
||||
);
|
||||
}
|
||||
@@ -5,6 +5,11 @@ import {
|
||||
} from './auction-rejection';
|
||||
|
||||
describe('resolveAuctionRejectionReason', () => {
|
||||
it('distinguishes an outdated client from a stale round and does not suggest a state-only resync', () => {
|
||||
const reason = resolveAuctionRejectionReason('BID_CLIENT_UPDATE_REQUIRED');
|
||||
expect(reason).toBe('BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED');
|
||||
expect(isAuctionResyncRecommended(reason)).toBe(false);
|
||||
});
|
||||
it('reports roster validation failures without recommending a retry', () => {
|
||||
const reason = resolveAuctionRejectionReason(
|
||||
'AUCTION_ROSTER_INVALID',
|
||||
|
||||
@@ -46,6 +46,8 @@ export const AUCTION_REJECTION_REASON = {
|
||||
'AUCTION_TRANSITION_REJECTED_BECAUSE_REQUEST_COORDINATION_RETRYABLE',
|
||||
|
||||
// === 입찰(bid) ===
|
||||
BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED:
|
||||
'BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED',
|
||||
/** 입찰 라운드가 이미 다음 선수/다음 라운드로 넘어감(내부 `STALE_BID_ROUND`). */
|
||||
BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED:
|
||||
'BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED',
|
||||
@@ -86,6 +88,8 @@ export type AuctionRejectionReason =
|
||||
|
||||
/** 내부 짧은 코드 → 설명적 reason. */
|
||||
const INTERNAL_CODE_TO_REASON: Record<string, AuctionRejectionReason> = {
|
||||
BID_CLIENT_UPDATE_REQUIRED:
|
||||
AUCTION_REJECTION_REASON.BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED,
|
||||
AUCTION_ROSTER_INVALID:
|
||||
AUCTION_REJECTION_REASON.AUCTION_TRANSITION_REJECTED_BECAUSE_ROSTER_INVALID,
|
||||
CONFLICT:
|
||||
|
||||
@@ -96,6 +96,7 @@ describe('auction side effect helpers', () => {
|
||||
'life-1',
|
||||
'2',
|
||||
'completed',
|
||||
'',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -25,6 +25,7 @@ export interface BuildSideEffectLeaseAcquireUpdateParams {
|
||||
|
||||
export interface MatchSideEffectDelta {
|
||||
auctionLifecycleId: string;
|
||||
auctionStateTransitionId?: string;
|
||||
currentServerTimestampMS: number;
|
||||
matchSideEffectRevision: number;
|
||||
matchSideEffectStatus: AuctionMatchSideEffectStatus;
|
||||
@@ -36,6 +37,7 @@ export class AuctionSideEffectService {
|
||||
|
||||
public async mirrorMatchSideEffectStatus(params: {
|
||||
auctionLifecycleId: string;
|
||||
auctionStateTransitionId?: string;
|
||||
matchId: string;
|
||||
matchSideEffectRevision: number;
|
||||
matchSideEffectStatus: AuctionMatchSideEffectStatus;
|
||||
@@ -47,6 +49,8 @@ export class AuctionSideEffectService {
|
||||
'EVAL',
|
||||
[
|
||||
"if redis.call('HGET', KEYS[1], 'auctionLifecycleId') ~= ARGV[1] then return 0 end",
|
||||
"local generation = redis.call('HGET', KEYS[1], 'matchSideEffectTransitionId')",
|
||||
"if ARGV[4] ~= '' and generation and generation ~= ARGV[4] then return 0 end",
|
||||
"local currentRevision = tonumber(redis.call('HGET', KEYS[1], 'matchSideEffectRevision') or '-1')",
|
||||
'local nextRevision = tonumber(ARGV[2])',
|
||||
'if currentRevision >= nextRevision then return 0 end',
|
||||
@@ -58,6 +62,7 @@ export class AuctionSideEffectService {
|
||||
params.auctionLifecycleId,
|
||||
String(params.matchSideEffectRevision),
|
||||
params.matchSideEffectStatus,
|
||||
params.auctionStateTransitionId ?? '',
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
@@ -104,6 +109,9 @@ export function buildMatchSideEffectDelta(
|
||||
): MatchSideEffectDelta {
|
||||
return {
|
||||
auctionLifecycleId: params.auctionLifecycleId,
|
||||
...(params.auctionStateTransitionId
|
||||
? { auctionStateTransitionId: params.auctionStateTransitionId }
|
||||
: {}),
|
||||
currentServerTimestampMS: params.currentServerTimestampMS,
|
||||
matchSideEffectRevision: params.matchSideEffectRevision,
|
||||
matchSideEffectStatus: params.matchSideEffectStatus,
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
Param,
|
||||
Post,
|
||||
UseGuards,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import { AllowPublicAccess } from '../auth/allow-public-access.decorator';
|
||||
@@ -19,6 +20,7 @@ import { ParseObjectIdPipe } from '../common/parse-object-id.pipe';
|
||||
import type { UserDocument } from '../global/database/schema';
|
||||
import { AuctionService } from './auction.service';
|
||||
import { AuctionGateway } from './auction.gateway';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
import {
|
||||
NextAuctionDto,
|
||||
OpenBiddingDto,
|
||||
@@ -35,6 +37,7 @@ export class AuctionController {
|
||||
constructor(
|
||||
private readonly auctionService: AuctionService,
|
||||
private readonly auctionGateway: AuctionGateway,
|
||||
private readonly auctionBidArchiveService: AuctionBidArchiveService,
|
||||
) {}
|
||||
|
||||
@Post()
|
||||
@@ -78,6 +81,20 @@ export class AuctionController {
|
||||
}
|
||||
|
||||
// 공개 — 팀장 2명만 쓸 엔드포인트(트러스트 기반, 10명 동시 관전).
|
||||
@Get('bids/history')
|
||||
@AllowPublicAccess()
|
||||
public async getBidHistory(
|
||||
@Param('matchId', ParseObjectIdPipe) matchId: mongoose.Types.ObjectId,
|
||||
@Query('afterRevision') afterRevision?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
return this.auctionBidArchiveService.list(
|
||||
matchId,
|
||||
afterRevision === undefined ? -1 : Number(afterRevision),
|
||||
limit === undefined ? 100 : Number(limit),
|
||||
);
|
||||
}
|
||||
|
||||
@Post('open-bidding')
|
||||
@AllowPublicAccess()
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@@ -100,6 +117,7 @@ export class AuctionController {
|
||||
teamIndex: body.bidderTeamIndex,
|
||||
amount: body.bidPointAmount,
|
||||
bidRequestId: body.bidRequestId,
|
||||
clientObservedBidRoundId: body.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId: body.clientObservedAuctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId:
|
||||
body.clientObservedCurrentBidRoundPlayerUserId,
|
||||
@@ -213,7 +231,10 @@ export class AuctionController {
|
||||
clientObservedPhase: body.clientObservedPhase,
|
||||
clientObservedRevision: body.clientObservedRevision,
|
||||
});
|
||||
this.auctionGateway.broadcastReset(matchId.toString());
|
||||
this.auctionGateway.broadcastReset(
|
||||
matchId.toString(),
|
||||
body.clientObservedAuctionLifecycleId,
|
||||
);
|
||||
}
|
||||
|
||||
private assertSameCommunity(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ServiceUnavailableException } from '@nestjs/common';
|
||||
|
||||
jest.mock('../auth/jwt.service', () => ({
|
||||
JwtService: class JwtService {},
|
||||
@@ -7,6 +8,76 @@ jest.mock('../auth/jwt.service', () => ({
|
||||
import { AuctionGateway } from './auction.gateway';
|
||||
|
||||
describe('AuctionGateway', () => {
|
||||
it('signals pending initial state so the client can retry privately after recovery', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId().toString();
|
||||
const gateway = new AuctionGateway(
|
||||
{
|
||||
getByMatchId: async () => ({}),
|
||||
getStateByMatchId: async () => {
|
||||
throw new ServiceUnavailableException({
|
||||
code: 'AUCTION_STATE_RECOVERING',
|
||||
message: 'recovering',
|
||||
});
|
||||
},
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() };
|
||||
await gateway.handleJoin(socket as never, { matchId });
|
||||
expect(socket.emit).toHaveBeenCalledTimes(1);
|
||||
expect(socket.emit).toHaveBeenCalledWith('state-pending', {
|
||||
retryAfterMS: 500,
|
||||
});
|
||||
});
|
||||
it('sends join and resync snapshots only to the requesting socket and coalesces in-flight resyncs', async () => {
|
||||
const matchId = new mongoose.Types.ObjectId().toString();
|
||||
const getStateByMatchId = jest.fn().mockResolvedValue({ revision: 3 });
|
||||
const gateway = new AuctionGateway(
|
||||
{
|
||||
getByMatchId: async () => ({ chatMessages: [] }),
|
||||
getStateByMatchId,
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const broadcast = jest.fn();
|
||||
(gateway as unknown as { server: unknown }).server = {
|
||||
to: () => ({ emit: broadcast }),
|
||||
};
|
||||
const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() };
|
||||
await gateway.handleJoin(socket as never, { matchId });
|
||||
expect(socket.emit).toHaveBeenCalledWith('state', { revision: 3 });
|
||||
expect(broadcast).not.toHaveBeenCalled();
|
||||
let finish!: (value: unknown) => void;
|
||||
getStateByMatchId.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const ack = jest.fn(),
|
||||
busyAck = jest.fn();
|
||||
const pending = gateway.handleResync(socket as never, {}, ack);
|
||||
await gateway.handleResync(socket as never, {}, busyAck);
|
||||
expect(busyAck).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ code: 'RESYNC_BUSY' }),
|
||||
);
|
||||
finish({ revision: 4 });
|
||||
await pending;
|
||||
expect(ack).toHaveBeenCalledWith({ ok: true, state: { revision: 4 } });
|
||||
expect(getStateByMatchId).toHaveBeenCalledTimes(2);
|
||||
expect(broadcast).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
function createGateway(params?: {
|
||||
getCurrentRedisTimestampMS?: jest.Mock;
|
||||
placeBidDelta?: jest.Mock;
|
||||
|
||||
@@ -44,6 +44,7 @@ interface JoinAuctionPayload {
|
||||
}
|
||||
|
||||
interface BidPayload {
|
||||
clientObservedBidRoundId?: string;
|
||||
bidderTeamIndex: number;
|
||||
bidPointAmount: number;
|
||||
bidRequestId?: string;
|
||||
@@ -79,6 +80,8 @@ interface BidRateLimitState {
|
||||
}
|
||||
|
||||
interface AuctionSocketData {
|
||||
auctionResyncAtTimestampMS?: number;
|
||||
auctionResyncInProgress?: boolean;
|
||||
auctionBidRateLimit?: BidRateLimitState;
|
||||
identity?: IdentityDocument;
|
||||
matchId?: string;
|
||||
@@ -162,9 +165,55 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
|
||||
new mongoose.Types.ObjectId(payload.matchId),
|
||||
);
|
||||
this.emitChatHistory(socket, auction);
|
||||
this.broadcastState(payload.matchId, auctionState);
|
||||
socket.emit('state', serializeRealtimeAuction(auctionState));
|
||||
} catch (error) {
|
||||
this.emitKnownError(socket, error);
|
||||
const rejection = this.resolveErrorPayload(error);
|
||||
if (rejection.code === 'AUCTION_STATE_RECOVERING') {
|
||||
socket.emit('state-pending', { retryAfterMS: 500 });
|
||||
} else {
|
||||
this.emitKnownError(socket, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SubscribeMessage('resync')
|
||||
public async handleResync(
|
||||
@ConnectedSocket() socket: AuctionSocket,
|
||||
@MessageBody() _payload: unknown,
|
||||
@Ack() ack?: (response: Record<string, unknown>) => void,
|
||||
) {
|
||||
const matchId = this.requireJoinedMatchId(socket);
|
||||
if (!matchId) {
|
||||
ack?.({ ok: false, code: 'JOIN_REQUIRED' });
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const retryAfterMS =
|
||||
500 - (now - (socket.data.auctionResyncAtTimestampMS ?? 0));
|
||||
if (socket.data.auctionResyncInProgress || retryAfterMS > 0) {
|
||||
ack?.({
|
||||
ok: false,
|
||||
code: 'RESYNC_BUSY',
|
||||
retryAfterMS: Math.max(500, retryAfterMS),
|
||||
});
|
||||
return;
|
||||
}
|
||||
socket.data.auctionResyncInProgress = true;
|
||||
socket.data.auctionResyncAtTimestampMS = now;
|
||||
try {
|
||||
const state = await this.auctionService.getStateByMatchId(
|
||||
new mongoose.Types.ObjectId(matchId),
|
||||
);
|
||||
// Only the requesting socket receives the full snapshot.
|
||||
ack?.({ ok: true, state: serializeRealtimeAuction(state) });
|
||||
} catch (error) {
|
||||
ack?.({
|
||||
...this.resolveErrorPayload(error),
|
||||
ok: false,
|
||||
retryAfterMS: 500,
|
||||
});
|
||||
} finally {
|
||||
socket.data.auctionResyncInProgress = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -196,6 +245,7 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
|
||||
teamIndex: payload.bidderTeamIndex,
|
||||
amount: payload.bidPointAmount,
|
||||
bidRequestId,
|
||||
clientObservedBidRoundId: payload.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId:
|
||||
payload.clientObservedAuctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId:
|
||||
@@ -609,8 +659,11 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit {
|
||||
this.server.to(matchId).emit('bid', delta);
|
||||
}
|
||||
|
||||
public broadcastReset(matchId: string) {
|
||||
this.server.to(matchId).emit('reset', { matchId });
|
||||
public broadcastReset(matchId: string, auctionLifecycleId?: string) {
|
||||
this.server.to(matchId).emit('reset', {
|
||||
matchId,
|
||||
...(auctionLifecycleId ? { auctionLifecycleId } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
public broadcastMatchSideEffect(
|
||||
|
||||
@@ -10,6 +10,7 @@ import { AuctionEventBusService } from './auction-event-bus.service';
|
||||
import { AuctionRedisModule } from './redis/auction-redis.module';
|
||||
import { AuctionSideEffectService } from './auction-side-effect.service';
|
||||
import { AuctionService } from './auction.service';
|
||||
import { AuctionBidArchiveService } from './auction-bid-archive.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -21,6 +22,7 @@ import { AuctionService } from './auction.service';
|
||||
controllers: [AuctionController],
|
||||
providers: [
|
||||
AuctionService,
|
||||
AuctionBidArchiveService,
|
||||
AuctionGateway,
|
||||
AuctionEventBusService,
|
||||
AuctionDeadlineScheduler,
|
||||
|
||||
@@ -45,6 +45,10 @@ describe('AuctionService', () => {
|
||||
const allModels = {
|
||||
auctionHistory: auctionHistoryModel,
|
||||
...models,
|
||||
auction: {
|
||||
updateOne: jest.fn().mockResolvedValue({ matchedCount: 1 }),
|
||||
...((models.auction as object) ?? {}),
|
||||
},
|
||||
};
|
||||
const connectionService = {
|
||||
getModel: jest.fn((name: string) => allModels[name]),
|
||||
@@ -697,7 +701,7 @@ describe('AuctionService', () => {
|
||||
expect(auctionRedisService.raw().sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining('currentAuctionLifecycleId'),
|
||||
'11',
|
||||
'12',
|
||||
`auction:${matchId.toString()}:liveAuctionState`,
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
`auction:${matchId.toString()}:auctionStructureState`,
|
||||
@@ -709,6 +713,7 @@ describe('AuctionService', () => {
|
||||
`auction:${matchId.toString()}:auctionActionRequestResultOrder`,
|
||||
`auction:${matchId.toString()}:rehydrateLock`,
|
||||
`auction:${matchId.toString()}:auctionStateTransitionLock`,
|
||||
`auction:${matchId.toString()}:rejectedBidRequestResultOrder`,
|
||||
matchId.toString(),
|
||||
'old-life-1',
|
||||
]);
|
||||
@@ -909,9 +914,9 @@ describe('AuctionService', () => {
|
||||
shouldForceAuctionStateResync: true,
|
||||
});
|
||||
expect(auctionRedisService.acquireTokenLock).toHaveBeenCalledWith(
|
||||
`auction:${matchId.toString()}:rehydrateLock`,
|
||||
`auction:${matchId.toString()}:lifecycleMutationLock`,
|
||||
expect.any(String),
|
||||
5000,
|
||||
30000,
|
||||
);
|
||||
expect(auctionRedisCommandService.hydrateState).toHaveBeenCalledWith(
|
||||
matchId.toString(),
|
||||
@@ -2621,8 +2626,14 @@ describe('AuctionService', () => {
|
||||
auctionRedisCommandService.reserveActionRequest,
|
||||
).not.toHaveBeenCalled();
|
||||
expect(auctionRedisCommandService.applyTransition).not.toHaveBeenCalled();
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match);
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(match).toMatchObject({
|
||||
gameSchedule: [],
|
||||
result: undefined,
|
||||
@@ -2642,13 +2653,12 @@ describe('AuctionService', () => {
|
||||
auctionId,
|
||||
});
|
||||
expect(auction.deleteOne).toHaveBeenCalledWith({ session });
|
||||
expect(redisClient.sendCommand).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
matchId.toString(),
|
||||
'life-1',
|
||||
]),
|
||||
);
|
||||
expect(redisClient.sendCommand).not.toHaveBeenCalled();
|
||||
expect(match).toMatchObject({
|
||||
lastCancelledAuctionLifecycleId: 'life-1',
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: expect.any(Date),
|
||||
});
|
||||
expect(session.commitTransaction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -2732,8 +2742,14 @@ describe('AuctionService', () => {
|
||||
service.undoTo({ matchId, communityId, sequence: 3 }),
|
||||
).resolves.toBe(restored);
|
||||
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match);
|
||||
expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(
|
||||
match,
|
||||
session,
|
||||
);
|
||||
expect(match.status).toBe('team-built');
|
||||
expect(match.result).toBeUndefined();
|
||||
expect(auctionReplaceOne).toHaveBeenCalledWith(
|
||||
|
||||
@@ -5,8 +5,16 @@ import {
|
||||
InternalServerErrorException,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
ServiceUnavailableException,
|
||||
Optional,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import {
|
||||
AuctionLifecycleLock,
|
||||
AuctionLifecycleBusyError,
|
||||
} from './auction-lifecycle-lock';
|
||||
import { AUCTION_OPTIONAL_PROJECTION_FIELDS } from './auction-projection';
|
||||
import { AuctionEventBusService } from './auction-event-bus.service';
|
||||
import { performance } from 'perf_hooks';
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
@@ -58,8 +66,6 @@ const AUCTION_BID_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000;
|
||||
const AUCTION_MAX_STORED_ACTION_REQUEST_RESULT_COUNT = 500;
|
||||
const AUCTION_ACTION_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000;
|
||||
const AUCTION_ACTION_REQUEST_LEASE_DURATION_MS = 15 * 1000;
|
||||
const AUCTION_REHYDRATE_LOCK_TTL_MS = 5 * 1000;
|
||||
const AUCTION_REHYDRATE_LOCK_WAIT_MS = 50;
|
||||
const AUCTION_HISTORY_LIST_LIMIT = 100;
|
||||
const AUCTION_HISTORY_LIST_CACHE_TTL_MS = 2 * 1000;
|
||||
const AUCTION_HISTORY_LIST_CACHE_MAX_ENTRIES = 256;
|
||||
@@ -82,6 +88,7 @@ interface StartAuctionParams {
|
||||
}
|
||||
|
||||
interface PlaceBidParams {
|
||||
clientObservedBidRoundId?: string;
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
teamIndex: number;
|
||||
amount: number;
|
||||
@@ -160,6 +167,7 @@ interface AuctionHistoryListCacheEntry {
|
||||
export type AuctionStateResponse = AuctionDocument | Record<string, unknown>;
|
||||
|
||||
export interface AuctionBidDeltaResponse {
|
||||
currentBidRoundId?: string;
|
||||
auctionBidHistoryEntry: Record<string, unknown>;
|
||||
auctionLifecycleId?: string;
|
||||
bidServerTiming?: AuctionBidServerTiming;
|
||||
@@ -212,9 +220,20 @@ export class AuctionService {
|
||||
private readonly auctionRedisService: AuctionRedisService,
|
||||
private readonly auctionRedisCommandService: AuctionRedisCommandService,
|
||||
private readonly auctionRedisStateReaderService: AuctionRedisStateReaderService,
|
||||
@Optional()
|
||||
private readonly auctionEventBusService?: AuctionEventBusService,
|
||||
) {}
|
||||
|
||||
public async start(params: StartAuctionParams): Promise<AuctionDocument> {
|
||||
return new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
params.matchId.toString(),
|
||||
() => this.startLocked(params),
|
||||
);
|
||||
}
|
||||
|
||||
private async startLocked(
|
||||
params: StartAuctionParams,
|
||||
): Promise<AuctionDocument> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const UserModel = await this.connectionService.getModel('user');
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
@@ -231,7 +250,7 @@ export class AuctionService {
|
||||
const existing = await AuctionModel.findOne({ matchId: params.matchId });
|
||||
if (existing) {
|
||||
if (existing.startAuctionRequestId === params.startAuctionRequestId) {
|
||||
return existing;
|
||||
return this.finishAuctionStartup(existing, match);
|
||||
}
|
||||
if (this.isRestartableResetRedisAuction(existing)) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
@@ -249,6 +268,19 @@ export class AuctionService {
|
||||
}
|
||||
}
|
||||
|
||||
// No Mongo auction owns this match. Remove runtime left by an interrupted
|
||||
// cancellation before creating a new lifecycle (under the lifecycle lock).
|
||||
const orphanedRuntime =
|
||||
await this.auctionRedisStateReaderService.readLiveStateIdentity(
|
||||
params.matchId.toString(),
|
||||
);
|
||||
if (orphanedRuntime?.auctionLifecycleId) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
params.matchId.toString(),
|
||||
orphanedRuntime.auctionLifecycleId,
|
||||
);
|
||||
}
|
||||
|
||||
const activeParticipants = this.applyAuctionCut(match);
|
||||
const desiredAuctionCount = activeParticipants.length;
|
||||
if (desiredAuctionCount < 10) {
|
||||
@@ -381,6 +413,7 @@ export class AuctionService {
|
||||
auctionStartedByUserId: params.actorUserId,
|
||||
matchSideEffectStatus: 'none',
|
||||
matchSideEffectRevision: 0,
|
||||
startupCompleted: false,
|
||||
teams: legacyTeams,
|
||||
order: prioritizedOrder,
|
||||
passedQueue: [],
|
||||
@@ -419,12 +452,12 @@ export class AuctionService {
|
||||
matchId: params.matchId,
|
||||
});
|
||||
if (raceWinner?.startAuctionRequestId === params.startAuctionRequestId) {
|
||||
return raceWinner;
|
||||
return this.finishAuctionStartup(raceWinner, match);
|
||||
}
|
||||
throw new BadRequestException('Auction already exists for this match.');
|
||||
}
|
||||
await this.recordSnapshot(auction, 'start');
|
||||
await this.auctionRedisCommandService.hydrateState(
|
||||
const hydration = await this.auctionRedisCommandService.hydrateState(
|
||||
params.matchId.toString(),
|
||||
{
|
||||
auctionHistorySequence: 1,
|
||||
@@ -437,9 +470,18 @@ export class AuctionService {
|
||||
structureStateJson,
|
||||
},
|
||||
);
|
||||
if ('rejected' in hydration)
|
||||
throw new ConflictException(
|
||||
`Auction initialization failed: ${hydration.code}.`,
|
||||
);
|
||||
|
||||
match.status = 'team-built';
|
||||
await match.save();
|
||||
auction.startupCompleted = true;
|
||||
await AuctionModel.updateOne(
|
||||
{ _id: auction._id },
|
||||
{ $set: { startupCompleted: true } },
|
||||
);
|
||||
this.enqueueBackgroundOrderEnrichment({
|
||||
auctionId: auction._id,
|
||||
targets: enrichTargets,
|
||||
@@ -450,6 +492,61 @@ export class AuctionService {
|
||||
return auction;
|
||||
}
|
||||
|
||||
private async finishAuctionStartup(
|
||||
auction: AuctionDocument,
|
||||
match: MatchDocument,
|
||||
): Promise<AuctionDocument> {
|
||||
// Documents predating the startup marker already completed the old path.
|
||||
if (auction.startupCompleted !== false) return auction;
|
||||
// Retry the original cut; recruiting may have changed since Mongo creation.
|
||||
const roster = new Set([
|
||||
...auction.teams.map((team) => team.captainUserId.toString()),
|
||||
...auction.order.map((player) => player.userId.toString()),
|
||||
]);
|
||||
const participants = new Set(
|
||||
match.participants.map((entry) => entry.userId.toString()),
|
||||
);
|
||||
if ([...roster].some((id) => !participants.has(id))) {
|
||||
throw new ConflictException(
|
||||
'Auction startup roster has changed. Cancel and restart the auction.',
|
||||
);
|
||||
}
|
||||
const History = await this.connectionService.getModel('auctionHistory');
|
||||
await History.updateOne(
|
||||
{ auctionId: auction._id, sequence: 1 },
|
||||
{
|
||||
$setOnInsert: {
|
||||
auctionId: auction._id,
|
||||
matchId: auction.matchId,
|
||||
sequence: 1,
|
||||
action: 'start',
|
||||
snapshotBefore: this.cloneJson(
|
||||
typeof auction.toObject === 'function'
|
||||
? auction.toObject()
|
||||
: auction,
|
||||
),
|
||||
takenAt: auction.startedAt ?? new Date(),
|
||||
},
|
||||
},
|
||||
{ upsert: true },
|
||||
);
|
||||
await this.hydrateRedisStateFromAuctionDocument(auction);
|
||||
if (match.status === 'recruiting') {
|
||||
match.participants.forEach((participant) => {
|
||||
participant.excluded = !roster.has(participant.userId.toString());
|
||||
});
|
||||
match.status = 'team-built';
|
||||
await match.save();
|
||||
}
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
await AuctionModel.updateOne(
|
||||
{ _id: auction._id },
|
||||
{ $set: { startupCompleted: true } },
|
||||
);
|
||||
auction.startupCompleted = true;
|
||||
return auction;
|
||||
}
|
||||
|
||||
public async getByMatchId(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
): Promise<AuctionDocument> {
|
||||
@@ -462,18 +559,46 @@ export class AuctionService {
|
||||
public async getStateByMatchId(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
): Promise<AuctionStateResponse> {
|
||||
const auction = await this.getByMatchId(matchId);
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await this.getStateByMatchIdOnce(matchId);
|
||||
} catch (error) {
|
||||
if (!(error instanceof AuctionLifecycleBusyError)) throw error;
|
||||
if (attempt >= 4)
|
||||
throw new ServiceUnavailableException({
|
||||
code: 'AUCTION_STATE_RECOVERING',
|
||||
retryAfterMS: 500,
|
||||
message: '경매 상태를 복구 중입니다. 잠시 후 다시 시도해 주세요.',
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async getStateByMatchIdOnce(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
): Promise<AuctionStateResponse> {
|
||||
let auction = await this.getByMatchId(matchId);
|
||||
if (!this.isRedisV1Auction(auction)) return auction;
|
||||
|
||||
if (auction.startupCompleted === false) {
|
||||
auction = await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
matchId.toString(),
|
||||
async () => {
|
||||
const current = await this.getByMatchId(matchId);
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const match = await MatchModel.findById(matchId);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
return this.finishAuctionStartup(current, match);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const redisState = await this.readRedisAuctionStateOrRehydrate(auction);
|
||||
if (!redisState) return auction;
|
||||
|
||||
const mongoState =
|
||||
typeof auction.toObject === 'function'
|
||||
? auction.toObject({ depopulate: true })
|
||||
: auction;
|
||||
return {
|
||||
...this.cloneJson(mongoState),
|
||||
...this.buildAuctionSnapshotFromRedisState(auction, redisState),
|
||||
...redisState,
|
||||
chatMessages: auction.chatMessages ?? [],
|
||||
};
|
||||
@@ -485,12 +610,15 @@ export class AuctionService {
|
||||
const matchId = auction.matchId.toString();
|
||||
const mongoRevision = auction.revision ?? 0;
|
||||
const mongoAuctionLifecycleId = auction.auctionLifecycleId;
|
||||
if (!mongoAuctionLifecycleId || auction.phase === 'complete') {
|
||||
if (!mongoAuctionLifecycleId) {
|
||||
return this.auctionRedisStateReaderService.readAuctionStateView(matchId);
|
||||
}
|
||||
|
||||
const liveIdentity =
|
||||
await this.auctionRedisStateReaderService.readLiveStateIdentity(matchId);
|
||||
// Completed runtimes expire after the retention period. Mongo remains the
|
||||
// archive; viewing it must not create an immortal Redis runtime again.
|
||||
if (!liveIdentity && auction.phase === 'complete') return null;
|
||||
const shouldHydrate = shouldHydrateRedisAuctionState({
|
||||
live: liveIdentity,
|
||||
mongo: {
|
||||
@@ -503,29 +631,32 @@ export class AuctionService {
|
||||
return this.auctionRedisStateReaderService.readAuctionStateView(matchId);
|
||||
}
|
||||
|
||||
const keys = buildAuctionRedisKeys(matchId);
|
||||
const lockToken = randomUUID();
|
||||
const lockAcquired = await this.auctionRedisService.acquireTokenLock(
|
||||
keys.rehydrateLockKey,
|
||||
lockToken,
|
||||
AUCTION_REHYDRATE_LOCK_TTL_MS,
|
||||
await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
matchId,
|
||||
async () => {
|
||||
const currentAuction = await this.getByMatchId(auction.matchId);
|
||||
if (currentAuction.auctionLifecycleId !== mongoAuctionLifecycleId) {
|
||||
throw new ConflictException(
|
||||
'Auction lifecycle has changed. Refresh the auction.',
|
||||
);
|
||||
}
|
||||
const currentLive =
|
||||
await this.auctionRedisStateReaderService.readLiveStateIdentity(
|
||||
matchId,
|
||||
);
|
||||
if (
|
||||
currentLive?.auctionLifecycleId &&
|
||||
currentLive.auctionLifecycleId !== mongoAuctionLifecycleId
|
||||
) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
matchId,
|
||||
currentLive.auctionLifecycleId,
|
||||
);
|
||||
}
|
||||
await this.hydrateRedisStateFromAuctionDocument(currentAuction);
|
||||
},
|
||||
);
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
await this.hydrateRedisStateFromAuctionDocument(auction);
|
||||
} finally {
|
||||
await this.auctionRedisService.releaseTokenLock(
|
||||
keys.rehydrateLockKey,
|
||||
lockToken,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, AUCTION_REHYDRATE_LOCK_WAIT_MS),
|
||||
);
|
||||
}
|
||||
|
||||
return this.auctionRedisStateReaderService.readAuctionStateView(matchId, {
|
||||
shouldForceAuctionStateResync: true,
|
||||
});
|
||||
@@ -814,6 +945,7 @@ export class AuctionService {
|
||||
bidderTeamIndex: params.teamIndex,
|
||||
bidderUserId: bidContext.bidderTeamCaptainUserId,
|
||||
bidRequestId: params.bidRequestId ?? randomUUID(),
|
||||
clientObservedBidRoundId: params.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId: auctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId,
|
||||
matchId,
|
||||
@@ -890,6 +1022,7 @@ export class AuctionService {
|
||||
bidderTeamIndex: params.teamIndex,
|
||||
bidderUserId: this.stringFromState(team.captainUserId),
|
||||
bidRequestId: params.bidRequestId ?? randomUUID(),
|
||||
clientObservedBidRoundId: params.clientObservedBidRoundId,
|
||||
clientObservedAuctionLifecycleId: auctionLifecycleId,
|
||||
clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId,
|
||||
matchId,
|
||||
@@ -923,6 +1056,10 @@ export class AuctionService {
|
||||
): AuctionBidDeltaResponse {
|
||||
return {
|
||||
auctionBidHistoryEntry: result.auctionBidHistoryEntry,
|
||||
currentBidRoundId:
|
||||
typeof result.auctionBidHistoryEntry.bidRoundId === 'string'
|
||||
? result.auctionBidHistoryEntry.bidRoundId
|
||||
: undefined,
|
||||
auctionLifecycleId: this.stringFromState(state.auctionLifecycleId),
|
||||
bidRequestId: result.bidRequestId,
|
||||
currentBidRoundDeadlineTimestampMS:
|
||||
@@ -1408,6 +1545,34 @@ export class AuctionService {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
}): Promise<void> {
|
||||
let committed = false;
|
||||
try {
|
||||
await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
params.matchId.toString(),
|
||||
() =>
|
||||
this.resetLocked(params, () => {
|
||||
committed = true;
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!committed) throw error;
|
||||
// A committed cancellation is successful even if lock renewal/teardown
|
||||
// fails afterward. Cleanup is durably queued in the same Mongo transaction.
|
||||
this.logger.warn(
|
||||
`Auction cancellation committed; post-commit coordination failed: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async resetLocked(
|
||||
params: {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
clientObservedAuctionLifecycleId?: string;
|
||||
clientObservedRevision?: number;
|
||||
},
|
||||
onCommitted: () => void,
|
||||
): Promise<void> {
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
const AuctionHistoryModel =
|
||||
await this.connectionService.getModel('auctionHistory');
|
||||
@@ -1419,70 +1584,155 @@ export class AuctionService {
|
||||
if (!session) {
|
||||
throw new InternalServerErrorException('Failed to start DB session.');
|
||||
}
|
||||
let redisRuntimeToClear: {
|
||||
auctionLifecycleId?: string;
|
||||
matchId: string;
|
||||
} | null = null;
|
||||
try {
|
||||
session.startTransaction();
|
||||
const auction = await AuctionModel.findOne({
|
||||
matchId: params.matchId,
|
||||
}).session(session);
|
||||
if (!auction) throw new NotFoundException('Auction not found.');
|
||||
|
||||
const match = await MatchModel.findById(params.matchId).session(session);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
if (!match.communityId.equals(params.communityId)) {
|
||||
throw new BadRequestException('Community mismatch.');
|
||||
}
|
||||
this.assertCompetitionLifecycleMutable(match, 'reset the auction');
|
||||
|
||||
if (this.isRedisV1Auction(auction)) {
|
||||
redisRuntimeToClear = {
|
||||
auctionLifecycleId: auction.auctionLifecycleId,
|
||||
matchId: params.matchId.toString(),
|
||||
};
|
||||
await this.resetMatchAfterAuctionCancellation(match, session);
|
||||
await AuctionSideEffectApplicationModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await auction.deleteOne({ session });
|
||||
if (!auction) {
|
||||
// 이미 커밋된 취소의 재전송이면 성공으로 응답하고 남은 Redis 정리만 재시도한다.
|
||||
const cancelledMatch = params.clientObservedAuctionLifecycleId
|
||||
? await MatchModel.findById(params.matchId).session(session)
|
||||
: null;
|
||||
if (
|
||||
!cancelledMatch ||
|
||||
cancelledMatch.lastCancelledAuctionLifecycleId !==
|
||||
params.clientObservedAuctionLifecycleId
|
||||
) {
|
||||
throw new NotFoundException('Auction not found.');
|
||||
}
|
||||
if (!cancelledMatch.communityId.equals(params.communityId)) {
|
||||
throw new BadRequestException('Community mismatch.');
|
||||
}
|
||||
cancelledMatch.auctionCancellationCleanupPending = true;
|
||||
cancelledMatch.auctionCancellationCleanupRetryAt = new Date();
|
||||
await cancelledMatch.save({ session });
|
||||
} else {
|
||||
await auction.deleteOne({ session });
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
if (match.status === 'team-built') {
|
||||
await this.tournamentCodeService.releaseFromMatch(match);
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
match.gameSchedule = [];
|
||||
match.participants.forEach((participant) => {
|
||||
participant.excluded = false;
|
||||
});
|
||||
match.teamBuiltAt = undefined;
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save({ session });
|
||||
if (
|
||||
this.isRedisV1Auction(auction) &&
|
||||
(!params.clientObservedAuctionLifecycleId ||
|
||||
params.clientObservedAuctionLifecycleId !==
|
||||
auction.auctionLifecycleId)
|
||||
) {
|
||||
throw new ConflictException(
|
||||
'Auction lifecycle has changed. Refresh before cancelling.',
|
||||
);
|
||||
}
|
||||
|
||||
const match = await MatchModel.findById(params.matchId).session(
|
||||
session,
|
||||
);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
if (!match.communityId.equals(params.communityId)) {
|
||||
throw new BadRequestException('Community mismatch.');
|
||||
}
|
||||
this.assertCompetitionLifecycleMutable(match, 'reset the auction');
|
||||
|
||||
if (this.isRedisV1Auction(auction)) {
|
||||
// 같은 트랜잭션에서 취소 완료를 기록해, 재전송을 멱등 처리한다.
|
||||
match.lastCancelledAuctionLifecycleId = auction.auctionLifecycleId;
|
||||
match.auctionCancellationCleanupPending = true;
|
||||
match.auctionCancellationCleanupRetryAt = new Date();
|
||||
match.auctionCancellationCleanupAttempts = 0;
|
||||
match.auctionCancellationCleanupError = undefined;
|
||||
await this.resetMatchAfterAuctionCancellation(match, session);
|
||||
await AuctionSideEffectApplicationModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
await auction.deleteOne({ session });
|
||||
} else {
|
||||
await auction.deleteOne({ session });
|
||||
await AuctionHistoryModel.deleteMany({
|
||||
auctionId: auction._id,
|
||||
}).session(session);
|
||||
if (match.status === 'team-built') {
|
||||
await this.tournamentCodeService.releaseFromMatch(match, session);
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
match.gameSchedule = [];
|
||||
match.participants.forEach((participant) => {
|
||||
participant.excluded = false;
|
||||
});
|
||||
match.teamBuiltAt = undefined;
|
||||
match.result = undefined;
|
||||
match.resultPollState = undefined;
|
||||
await match.save({ session });
|
||||
}
|
||||
}
|
||||
}
|
||||
await session.commitTransaction();
|
||||
onCommitted();
|
||||
} catch (err) {
|
||||
await session.abortTransaction();
|
||||
throw err;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
}
|
||||
|
||||
if (redisRuntimeToClear) {
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
redisRuntimeToClear.matchId,
|
||||
redisRuntimeToClear.auctionLifecycleId,
|
||||
);
|
||||
public async retryPendingCancellations(batchLimit = 20): Promise<void> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const pending = await MatchModel.find({
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: { $lte: new Date() },
|
||||
})
|
||||
.sort({ auctionCancellationCleanupRetryAt: 1 })
|
||||
.limit(batchLimit)
|
||||
.lean()
|
||||
.exec();
|
||||
for (const match of pending) {
|
||||
const lifecycle = match.lastCancelledAuctionLifecycleId;
|
||||
if (!lifecycle) continue;
|
||||
const filter = {
|
||||
_id: match._id,
|
||||
lastCancelledAuctionLifecycleId: lifecycle,
|
||||
};
|
||||
try {
|
||||
await new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
match._id.toString(),
|
||||
async () => {
|
||||
// Compare-and-delete in Redis also protects a newly started lifecycle.
|
||||
await this.clearRedisAuctionRuntimeForLifecycle(
|
||||
match._id.toString(),
|
||||
lifecycle,
|
||||
);
|
||||
const AuctionModel =
|
||||
await this.connectionService.getModel('auction');
|
||||
if (!(await AuctionModel.exists({ matchId: match._id }))) {
|
||||
this.auctionEventBusService?.broadcastReset(
|
||||
match._id.toString(),
|
||||
lifecycle,
|
||||
);
|
||||
}
|
||||
await MatchModel.updateOne(filter, {
|
||||
$set: { auctionCancellationCleanupPending: false },
|
||||
$unset: {
|
||||
auctionCancellationCleanupRetryAt: '',
|
||||
auctionCancellationCleanupError: '',
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
const attempts = (match.auctionCancellationCleanupAttempts ?? 0) + 1;
|
||||
await MatchModel.updateOne(filter, {
|
||||
$set: {
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: new Date(
|
||||
Date.now() + Math.min(60000, 1000 * 2 ** Math.min(attempts, 6)),
|
||||
),
|
||||
auctionCancellationCleanupError: String(error).slice(0, 2000),
|
||||
},
|
||||
$inc: { auctionCancellationCleanupAttempts: 1 },
|
||||
});
|
||||
this.logger.warn(
|
||||
`Auction cancellation cleanup will retry for ${match._id.toString()}: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1497,6 +1747,19 @@ export class AuctionService {
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
sequence: number;
|
||||
}): Promise<AuctionStateResponse> {
|
||||
return new AuctionLifecycleLock(this.auctionRedisService).run(
|
||||
params.matchId.toString(),
|
||||
() => this.undoToLocked(params),
|
||||
);
|
||||
}
|
||||
|
||||
private async undoToLocked(
|
||||
params: AuctionStructureActionRequestParams & {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
communityId: mongoose.Types.ObjectId;
|
||||
sequence: number;
|
||||
},
|
||||
): Promise<AuctionStateResponse> {
|
||||
const AuctionModel = await this.connectionService.getModel('auction');
|
||||
const AuctionHistoryModel =
|
||||
await this.connectionService.getModel('auctionHistory');
|
||||
@@ -1508,6 +1771,7 @@ export class AuctionService {
|
||||
}
|
||||
|
||||
let redisUndoContext: {
|
||||
state?: Record<string, unknown>;
|
||||
actionRequestParams?: AuctionStructureActionRequestParams;
|
||||
auction: AuctionDocument;
|
||||
snapshot: Record<string, unknown>;
|
||||
@@ -1543,8 +1807,12 @@ export class AuctionService {
|
||||
throw new BadRequestException('Auction history snapshot is invalid.');
|
||||
}
|
||||
|
||||
const undoingCompletedAuction =
|
||||
Boolean(auction.completedAt) && !('completedAt' in snapshot);
|
||||
const liveState = this.isRedisV1Auction(auction)
|
||||
? await this.requireRedisAuctionState(params.matchId.toString())
|
||||
: undefined;
|
||||
const undoingCompletedAuction = liveState
|
||||
? liveState.phase === 'complete' && snapshot.phase !== 'complete'
|
||||
: Boolean(auction.completedAt) && !('completedAt' in snapshot);
|
||||
if (undoingCompletedAuction) {
|
||||
this.assertCompetitionLifecycleMutable(
|
||||
match,
|
||||
@@ -1553,6 +1821,7 @@ export class AuctionService {
|
||||
}
|
||||
if (this.isRedisV1Auction(auction)) {
|
||||
redisUndoContext = {
|
||||
state: liveState,
|
||||
auction,
|
||||
actionRequestParams: params,
|
||||
snapshot: snapshot,
|
||||
@@ -1562,8 +1831,8 @@ export class AuctionService {
|
||||
await session.commitTransaction();
|
||||
} else {
|
||||
if (undoingCompletedAuction) {
|
||||
await this.tournamentCodeService.releaseFromMatch(match);
|
||||
await this.tournamentResultService.cancelPolling(match);
|
||||
await this.tournamentCodeService.releaseFromMatch(match, session);
|
||||
await this.tournamentResultService.cancelPolling(match, session);
|
||||
match.status = 'team-built';
|
||||
match.result = undefined;
|
||||
await match.save({ session });
|
||||
@@ -1758,6 +2027,14 @@ export class AuctionService {
|
||||
currentAuctionHistorySequence: auctionHistorySequence,
|
||||
lastAppliedAuctionStateTransitionId: auctionStateTransitionId,
|
||||
revision: expectedRevision + 1,
|
||||
currentBidRoundId:
|
||||
params.deadline.mode === 'set-relative'
|
||||
? auctionStateTransitionId
|
||||
: params.state.currentBidRoundId,
|
||||
currentBidRoundDeadlineTimestampMS:
|
||||
params.deadline.mode === 'remove'
|
||||
? 0
|
||||
: params.state.currentBidRoundDeadlineTimestampMS,
|
||||
};
|
||||
const afterAuctionMongoProjection =
|
||||
this.buildAuctionMongoProjectionFromState(params.auction, afterState);
|
||||
@@ -1769,8 +2046,18 @@ export class AuctionService {
|
||||
if (matchSideEffectMutation.type !== 'none') {
|
||||
afterAuctionMongoProjection.matchSideEffectStatus = 'pending';
|
||||
afterAuctionMongoProjection.matchSideEffectRevision =
|
||||
(params.auction.matchSideEffectRevision ?? 0) + 1;
|
||||
Math.max(
|
||||
params.auction.matchSideEffectRevision ?? 0,
|
||||
this.numberFromState(params.state.matchSideEffectRevision),
|
||||
) + 1;
|
||||
afterAuctionMongoProjection.matchSideEffectError = undefined;
|
||||
afterAuctionMongoProjection.matchSideEffectTransitionId =
|
||||
auctionStateTransitionId;
|
||||
params.liveStatePatch.matchSideEffectTransitionId =
|
||||
auctionStateTransitionId;
|
||||
params.liveStatePatch.matchSideEffectStatus = 'pending';
|
||||
params.liveStatePatch.matchSideEffectRevision =
|
||||
afterAuctionMongoProjection.matchSideEffectRevision as number;
|
||||
}
|
||||
|
||||
const durabilityPayload = {
|
||||
@@ -1925,6 +2212,7 @@ export class AuctionService {
|
||||
}
|
||||
|
||||
private async undoRedisAuction(params: {
|
||||
state?: Record<string, unknown>;
|
||||
actionRequestParams?: AuctionStructureActionRequestParams;
|
||||
auction: AuctionDocument;
|
||||
snapshot: Record<string, unknown>;
|
||||
@@ -1932,7 +2220,8 @@ export class AuctionService {
|
||||
undoingCompletedAuction: boolean;
|
||||
}): Promise<AuctionStateResponse> {
|
||||
const matchId = params.auction.matchId.toString();
|
||||
const state = await this.requireRedisAuctionState(matchId);
|
||||
const state =
|
||||
params.state ?? (await this.requireRedisAuctionState(matchId));
|
||||
const actionRequest = await this.reserveRedisStructureActionRequest({
|
||||
action: 'undo',
|
||||
matchId,
|
||||
@@ -2118,8 +2407,11 @@ export class AuctionService {
|
||||
typeof auction.toObject === 'function'
|
||||
? (auction.toObject({ depopulate: true }) as Record<string, unknown>)
|
||||
: (auction as unknown as Record<string, unknown>);
|
||||
const snapshotBase = this.cloneJson(base);
|
||||
for (const field of AUCTION_OPTIONAL_PROJECTION_FIELDS)
|
||||
delete snapshotBase[field];
|
||||
return {
|
||||
...this.cloneJson(base),
|
||||
...snapshotBase,
|
||||
...this.buildAuctionMongoProjectionFromState(auction, state),
|
||||
_id: auction._id,
|
||||
matchId: auction.matchId,
|
||||
@@ -2145,6 +2437,7 @@ export class AuctionService {
|
||||
);
|
||||
const phase = this.stringFromState(state.phase) || 'presenting';
|
||||
const projection: Record<string, unknown> = {
|
||||
currentBidRoundId: state.currentBidRoundId || undefined,
|
||||
auctionBidHistory,
|
||||
auctionCompletedAtTimestampMS:
|
||||
phase === 'complete' &&
|
||||
@@ -2285,6 +2578,8 @@ export class AuctionService {
|
||||
|
||||
private formatRedisBidRejectCode(code: string): string {
|
||||
switch (code) {
|
||||
case 'BID_CLIENT_UPDATE_REQUIRED':
|
||||
return '경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.';
|
||||
case 'PHASE':
|
||||
return 'Auction is not in bidding phase.';
|
||||
case 'STALE_BID_ROUND':
|
||||
@@ -2701,8 +2996,8 @@ export class AuctionService {
|
||||
match: MatchDocument,
|
||||
session: mongoose.ClientSession,
|
||||
): Promise<void> {
|
||||
await this.tournamentCodeService.releaseFromMatch(match);
|
||||
await this.tournamentResultService.cancelPolling(match);
|
||||
await this.tournamentCodeService.releaseFromMatch(match, session);
|
||||
await this.tournamentResultService.cancelPolling(match, session);
|
||||
match.status = 'recruiting';
|
||||
match.teams = [];
|
||||
match.tournamentCodeIds = [];
|
||||
@@ -2717,6 +3012,11 @@ export class AuctionService {
|
||||
}
|
||||
|
||||
private assertLiveAuctionOperationSupported(auction: AuctionDocument): void {
|
||||
if (auction.startupCompleted === false) {
|
||||
throw new ConflictException(
|
||||
'Auction startup is not complete. Refresh to resume initialization.',
|
||||
);
|
||||
}
|
||||
if (auction.auctionEngineVersion && !this.isRedisV1Auction(auction)) {
|
||||
throw new BadRequestException(
|
||||
'Legacy auction live operations are not supported.',
|
||||
@@ -2881,7 +3181,7 @@ export class AuctionService {
|
||||
auction.topTierThreshold ??
|
||||
'NONE',
|
||||
};
|
||||
await this.auctionRedisCommandService.hydrateState(
|
||||
const hydration = await this.auctionRedisCommandService.hydrateState(
|
||||
auction.matchId.toString(),
|
||||
{
|
||||
auctionHistorySequence: auction.currentAuctionHistorySequence ?? 0,
|
||||
@@ -2889,6 +3189,10 @@ export class AuctionService {
|
||||
auction.auctionLifecycleId ??
|
||||
`auction:${auction._id.toString()}:rehydrated`,
|
||||
liveStatePatch: {
|
||||
currentBidRoundId: auction.phase === 'bidding' ? randomUUID() : '',
|
||||
matchSideEffectTransitionId: auction.matchSideEffectTransitionId,
|
||||
matchSideEffectRevision: auction.matchSideEffectRevision,
|
||||
matchSideEffectStatus: auction.matchSideEffectStatus,
|
||||
...buildAuctionRedisLiveStatePatchFromStructure(structureInput),
|
||||
currentHighestBidPointAmount:
|
||||
auction.currentHighestBidPointAmount ?? auction.currentBid ?? 0,
|
||||
@@ -2905,6 +3209,10 @@ export class AuctionService {
|
||||
structureStateJson: stringifyAuctionRedisStructureState(structureInput),
|
||||
},
|
||||
);
|
||||
if ('rejected' in hydration)
|
||||
throw new ConflictException(
|
||||
`Auction recovery failed: ${hydration.code}.`,
|
||||
);
|
||||
}
|
||||
|
||||
private async clearRedisAuctionRuntimeForLifecycle(
|
||||
@@ -2925,10 +3233,10 @@ export class AuctionService {
|
||||
' end',
|
||||
'end',
|
||||
"redis.call('ZREM', KEYS[2], ARGV[1])",
|
||||
"redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11])",
|
||||
"redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11], KEYS[12])",
|
||||
'return 1',
|
||||
].join('\n'),
|
||||
'11',
|
||||
'12',
|
||||
keys.liveStateKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.structureStateKey,
|
||||
@@ -2940,6 +3248,7 @@ export class AuctionService {
|
||||
keys.actionRequestResultOrderKey,
|
||||
keys.rehydrateLockKey,
|
||||
keys.stateTransitionLockKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
matchId,
|
||||
expectedAuctionLifecycleId ?? '',
|
||||
]);
|
||||
|
||||
@@ -72,6 +72,10 @@ export class StartAuctionDto {
|
||||
}
|
||||
|
||||
export class PlaceBidDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
clientObservedBidRoundId?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsUUID()
|
||||
bidRequestId?: string;
|
||||
|
||||
@@ -7,7 +7,13 @@ function makeService(
|
||||
) {
|
||||
const sendCommand = jest
|
||||
.fn()
|
||||
.mockImplementation(() => Promise.resolve(commandReplies.shift()));
|
||||
.mockImplementation((command: string[]) =>
|
||||
Promise.resolve(
|
||||
command[0] === 'EVAL'
|
||||
? [currentServerTimestampMS, ...commandReplies.splice(0, 3)]
|
||||
: commandReplies.shift(),
|
||||
),
|
||||
);
|
||||
const auctionRedisService = {
|
||||
getCurrentRedisTimestampMS: jest
|
||||
.fn()
|
||||
@@ -125,19 +131,14 @@ describe('AuctionRedisStateReaderService', () => {
|
||||
});
|
||||
expect(view).not.toHaveProperty('team:2:remainingBidPointAmount');
|
||||
expect(view).not.toHaveProperty('bidRoundExpiryDecision');
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(1, [
|
||||
'HGETALL',
|
||||
expect(sendCommand).toHaveBeenCalledTimes(1);
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining("redis.call('TIME')"),
|
||||
'3',
|
||||
'auction:match-1:liveAuctionState',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(2, [
|
||||
'GET',
|
||||
'auction:match-1:auctionStructureState',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenNthCalledWith(3, [
|
||||
'LRANGE',
|
||||
'auction:match-1:auctionBidHistory',
|
||||
'0',
|
||||
'-1',
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -111,18 +111,11 @@ export class AuctionRedisStateReaderService {
|
||||
liveReply,
|
||||
structureStateJson,
|
||||
bidEntries,
|
||||
] = await Promise.all([
|
||||
this.auctionRedisService.getCurrentRedisTimestampMS(),
|
||||
this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['HGETALL', keys.liveStateKey]),
|
||||
this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['GET', keys.structureStateKey]),
|
||||
this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand(['LRANGE', keys.bidHistoryKey, '0', '-1']),
|
||||
]);
|
||||
] = await this.auctionRedisService
|
||||
.raw()
|
||||
.sendCommand<
|
||||
unknown[]
|
||||
>(['EVAL', ["local now = redis.call('TIME')", 'local timestamp = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)', "return {tostring(timestamp), redis.call('HGETALL', KEYS[1]), redis.call('GET', KEYS[2]), redis.call('LRANGE', KEYS[3], 0, -1)}"].join('\n'), '3', keys.liveStateKey, keys.structureStateKey, keys.bidHistoryKey]);
|
||||
|
||||
const liveState = parseHashReply(liveReply, {
|
||||
includeInternalLiveStateFields:
|
||||
@@ -144,7 +137,7 @@ export class AuctionRedisStateReaderService {
|
||||
auctionBidHistoryEntries: Array.isArray(bidEntries)
|
||||
? bidEntries.map(String)
|
||||
: [],
|
||||
currentServerTimestampMS,
|
||||
currentServerTimestampMS: Number(currentServerTimestampMS),
|
||||
liveState: liveState as Record<string, unknown> & {
|
||||
auctionLifecycleId: string;
|
||||
phase: string;
|
||||
|
||||
@@ -46,6 +46,7 @@ describe('auction apply transition redis script contract', () => {
|
||||
'auction:match-1:bidRequestResults',
|
||||
'auction:match-1:bidRequestResultOrder',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'auction:match-1:rejectedBidRequestResultOrder',
|
||||
]);
|
||||
expect(invocation.values.slice(0, 19)).toEqual([
|
||||
'match-1',
|
||||
|
||||
@@ -79,6 +79,7 @@ export function buildAuctionApplyTransitionScriptInvocation(
|
||||
keys.bidRequestResultsKey,
|
||||
keys.bidRequestResultOrderKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
],
|
||||
script: AUCTION_APPLY_TRANSITION_SCRIPT,
|
||||
values: [
|
||||
@@ -128,6 +129,7 @@ local auctionBidHistoryKey = KEYS[8]
|
||||
local bidRequestResultsKey = KEYS[9]
|
||||
local bidRequestResultOrderKey = KEYS[10]
|
||||
local bidRoundDeadlineQueueKey = KEYS[11]
|
||||
local rejectedBidRequestResultOrderKey = KEYS[12]
|
||||
|
||||
local matchId = ARGV[1]
|
||||
local auctionLifecycleId = ARGV[2]
|
||||
@@ -270,16 +272,27 @@ for i = 1, liveStatePatchPairCount do
|
||||
end
|
||||
redis.call('HSET', unpack(hsetArgs))
|
||||
|
||||
local committedBidRoundDeadlineTimestampMS = ''
|
||||
if deadlineMode == 'set-relative' then
|
||||
local currentBidRoundDeadlineTimestampMS = currentRedisTimestampMS + (deadlineDurationSeconds * 1000)
|
||||
committedBidRoundDeadlineTimestampMS = tostring(currentBidRoundDeadlineTimestampMS)
|
||||
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS', tostring(currentBidRoundDeadlineTimestampMS))
|
||||
redis.call('HSET', liveAuctionStateKey, 'currentBidRoundId', auctionStateTransitionId)
|
||||
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
|
||||
elseif deadlineMode == 'remove' then
|
||||
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
|
||||
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
|
||||
end
|
||||
|
||||
-- A resumed auction must not disappear when its former completion TTL elapses.
|
||||
if redis.call('HGET', liveAuctionStateKey, 'phase') ~= 'complete' then
|
||||
for _, index in ipairs({1, 2, 3, 6, 7, 8, 9, 10, 12}) do
|
||||
redis.call('PERSIST', KEYS[index])
|
||||
end
|
||||
end
|
||||
|
||||
if shouldResetBidRuntime == '1' then
|
||||
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey)
|
||||
redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey, rejectedBidRequestResultOrderKey)
|
||||
end
|
||||
|
||||
redis.call('SADD', appliedAuctionStateTransitionIdsKey, transitionDedupeMember)
|
||||
@@ -288,6 +301,8 @@ redis.call(
|
||||
durabilityPayloadKey,
|
||||
'payloadJson',
|
||||
durabilityPayloadJson,
|
||||
'committedBidRoundDeadlineTimestampMS',
|
||||
committedBidRoundDeadlineTimestampMS,
|
||||
'preparedAtRedisTimestampMS',
|
||||
tostring(currentRedisTimestampMS),
|
||||
'committedAtRedisTimestampMS',
|
||||
|
||||
@@ -30,6 +30,9 @@ describe('auction bid redis script contract', () => {
|
||||
'auction:match-1:bidRequestResults',
|
||||
'auction:match-1:bidRequestResultOrder',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'auction:match-1:rejectedBidRequestResultOrder',
|
||||
'auction:bidArchiveQueue',
|
||||
'auction:bidArchivePayloads',
|
||||
]);
|
||||
expect(invocation.values).toEqual([
|
||||
'match-1',
|
||||
@@ -45,6 +48,7 @@ describe('auction bid redis script contract', () => {
|
||||
'100',
|
||||
'60000',
|
||||
'200',
|
||||
'',
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { AuctionRedisKeys } from './auction-redis.keys';
|
||||
import {
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
} from './auction-redis.keys';
|
||||
import { defineAuctionRedisScript } from './auction-redis.scripts';
|
||||
|
||||
export interface AuctionBidScriptInput {
|
||||
@@ -8,6 +12,7 @@ export interface AuctionBidScriptInput {
|
||||
bidderUserId: string;
|
||||
clientObservedAuctionLifecycleId: string;
|
||||
clientObservedCurrentBidRoundPlayerUserId: string;
|
||||
clientObservedBidRoundId?: string;
|
||||
maximumAuctionTeamMemberCount: number;
|
||||
maximumStoredAuctionBidHistoryCount: number;
|
||||
maximumStoredBidRequestResultCount: number;
|
||||
@@ -40,15 +45,19 @@ export function buildBidRequestPayloadFingerprint(params: {
|
||||
bidPointAmount: number;
|
||||
clientObservedAuctionLifecycleId: string;
|
||||
clientObservedCurrentBidRoundPlayerUserId: string;
|
||||
clientObservedBidRoundId?: string;
|
||||
matchId: string;
|
||||
}): string {
|
||||
return [
|
||||
const parts = [
|
||||
params.matchId,
|
||||
params.clientObservedAuctionLifecycleId,
|
||||
params.clientObservedCurrentBidRoundPlayerUserId,
|
||||
params.bidderTeamIndex,
|
||||
params.bidPointAmount,
|
||||
].join('|');
|
||||
];
|
||||
if (params.clientObservedBidRoundId)
|
||||
parts.push(params.clientObservedBidRoundId);
|
||||
return parts.join('|');
|
||||
}
|
||||
|
||||
export function buildAuctionBidScriptInvocation(
|
||||
@@ -62,6 +71,9 @@ export function buildAuctionBidScriptInvocation(
|
||||
keys.bidRequestResultsKey,
|
||||
keys.bidRequestResultOrderKey,
|
||||
keys.deadlineQueueKey,
|
||||
keys.rejectedBidRequestResultOrderKey,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
],
|
||||
script: AUCTION_BID_SCRIPT,
|
||||
values: [
|
||||
@@ -78,6 +90,7 @@ export function buildAuctionBidScriptInvocation(
|
||||
String(input.maximumStoredBidRequestResultCount),
|
||||
String(input.minimumBidRequestResultReplayWindowMS),
|
||||
String(input.maximumStoredAuctionBidHistoryCount),
|
||||
input.clientObservedBidRoundId ?? '',
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -90,6 +103,9 @@ local auctionBidHistoryKey = KEYS[2]
|
||||
local bidRequestResultsKey = KEYS[3]
|
||||
local bidRequestResultOrderKey = KEYS[4]
|
||||
local bidRoundDeadlineQueueKey = KEYS[5]
|
||||
local rejectedBidRequestResultOrderKey = KEYS[6]
|
||||
local bidArchiveQueueKey = KEYS[7]
|
||||
local bidArchivePayloadsKey = KEYS[8]
|
||||
|
||||
local matchId = ARGV[1]
|
||||
local bidderTeamIndex = ARGV[2]
|
||||
@@ -104,6 +120,7 @@ local bidRequestPayloadFingerprint = ARGV[10]
|
||||
local maximumStoredBidRequestResultCount = tonumber(ARGV[11])
|
||||
local minimumBidRequestResultReplayWindowMS = tonumber(ARGV[12])
|
||||
local maximumStoredAuctionBidHistoryCount = tonumber(ARGV[13])
|
||||
local clientObservedBidRoundId = ARGV[14] or ''
|
||||
|
||||
local redisTime = redis.call('TIME')
|
||||
local currentRedisTimestampMS = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)
|
||||
@@ -168,12 +185,23 @@ if existingFingerprint then
|
||||
return replayExistingRequest()
|
||||
end
|
||||
|
||||
trimOldBidRequestResults()
|
||||
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
|
||||
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
|
||||
-- Upgrade the old shared queue once, preserving every accepted replay record.
|
||||
if redis.call('HSETNX', bidRequestResultsKey, '_acceptedOnlyOrderV1', '1') == 1 then
|
||||
for _, oldRequestId in ipairs(redis.call('LRANGE', bidRequestResultOrderKey, 0, -1)) do
|
||||
if redis.call('HGET', bidRequestResultsKey, oldRequestId .. ':status') == 'rejected' then
|
||||
redis.call('LREM', bidRequestResultOrderKey, 0, oldRequestId)
|
||||
hdelBidRequest(oldRequestId)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local function storeRejectedRequest(rejectCode)
|
||||
-- Rejections have their own bounded retention. They must never consume the
|
||||
-- protected replay capacity of accepted bids.
|
||||
while redis.call('LLEN', rejectedBidRequestResultOrderKey) >= maximumStoredBidRequestResultCount do
|
||||
local oldest = redis.call('LPOP', rejectedBidRequestResultOrderKey)
|
||||
if oldest then hdelBidRequest(oldest) end
|
||||
end
|
||||
local rejectedCurrentBidRoundDeadlineTimestampMS = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or ''
|
||||
redis.call(
|
||||
'HSET',
|
||||
@@ -191,7 +219,7 @@ local function storeRejectedRequest(rejectCode)
|
||||
requestField('createdAtTimestampMS'),
|
||||
tostring(currentRedisTimestampMS)
|
||||
)
|
||||
redis.call('RPUSH', bidRequestResultOrderKey, bidRequestId)
|
||||
redis.call('RPUSH', rejectedBidRequestResultOrderKey, bidRequestId)
|
||||
return {'REJECTED', rejectCode, bidRequestId, tostring(currentRedisTimestampMS), rejectedCurrentBidRoundDeadlineTimestampMS}
|
||||
end
|
||||
|
||||
@@ -202,9 +230,16 @@ end
|
||||
|
||||
local auctionLifecycleId = redis.call('HGET', liveAuctionStateKey, 'auctionLifecycleId')
|
||||
local currentBidRoundPlayerUserId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundPlayerUserId')
|
||||
local currentBidRoundId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundId') or ''
|
||||
if currentBidRoundId ~= '' and clientObservedBidRoundId == '' then
|
||||
return storeRejectedRequest('BID_CLIENT_UPDATE_REQUIRED')
|
||||
end
|
||||
if auctionLifecycleId ~= clientObservedAuctionLifecycleId or currentBidRoundPlayerUserId ~= clientObservedCurrentBidRoundPlayerUserId then
|
||||
return storeRejectedRequest('STALE_BID_ROUND')
|
||||
end
|
||||
if currentBidRoundId ~= clientObservedBidRoundId then
|
||||
return storeRejectedRequest('STALE_BID_ROUND')
|
||||
end
|
||||
|
||||
local currentBidRoundDeadlineTimestampMS = tonumber(redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or '0')
|
||||
if currentBidRoundDeadlineTimestampMS <= currentRedisTimestampMS then
|
||||
@@ -235,8 +270,9 @@ if bidPointAmount + requiredReserveAfterBid > remainingBidPointAmount then
|
||||
return storeRejectedRequest('INSUFFICIENT')
|
||||
end
|
||||
|
||||
if redis.call('LLEN', auctionBidHistoryKey) >= maximumStoredAuctionBidHistoryCount then
|
||||
return storeRejectedRequest('BID_HISTORY_LIMIT')
|
||||
trimOldBidRequestResults()
|
||||
if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then
|
||||
return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)}
|
||||
end
|
||||
|
||||
local maximumBidRoundDurationSeconds = tonumber(redis.call('HGET', liveAuctionStateKey, 'maximumBidRoundDurationSeconds') or '0')
|
||||
@@ -254,14 +290,36 @@ end
|
||||
local nextBidRoundDeadlineTimestampMS = currentRedisTimestampMS + nextRemainingBidRoundTimeMS
|
||||
|
||||
local revision = redis.call('HINCRBY', liveAuctionStateKey, 'revision', 1)
|
||||
local auctionBidHistoryEntryJson =
|
||||
'{"bidderTeamIndex":' .. bidderTeamIndex ..
|
||||
',"bidPointAmount":' .. tostring(bidPointAmount) ..
|
||||
',"bidderUserId":"' .. bidderUserId ..
|
||||
'","bidPlacedAtTimestampMS":' .. tostring(currentRedisTimestampMS) ..
|
||||
'}'
|
||||
local auctionBidHistoryEntryJson = cjson.encode({
|
||||
bidRequestId = bidRequestId,
|
||||
revision = revision,
|
||||
bidderTeamIndex = tonumber(bidderTeamIndex),
|
||||
bidPointAmount = bidPointAmount,
|
||||
bidderUserId = bidderUserId,
|
||||
playerUserId = currentBidRoundPlayerUserId,
|
||||
bidRoundId = currentBidRoundId,
|
||||
bidPlacedAtTimestampMS = currentRedisTimestampMS
|
||||
})
|
||||
|
||||
-- Accepted replay returns above this point. Persist every accepted event before
|
||||
-- trimming the recent view; the durable archive worker removes only saved events.
|
||||
local archiveId = matchId .. ':' .. auctionLifecycleId .. ':' .. string.format('%016d', revision)
|
||||
redis.call('HSET', bidArchivePayloadsKey, archiveId, cjson.encode({
|
||||
_id = archiveId,
|
||||
matchId = matchId,
|
||||
auctionLifecycleId = auctionLifecycleId,
|
||||
revision = revision,
|
||||
bidRequestId = bidRequestId,
|
||||
bidderTeamIndex = tonumber(bidderTeamIndex),
|
||||
bidPointAmount = bidPointAmount,
|
||||
bidderUserId = bidderUserId,
|
||||
playerUserId = currentBidRoundPlayerUserId,
|
||||
bidRoundId = currentBidRoundId,
|
||||
bidPlacedAtTimestampMS = currentRedisTimestampMS
|
||||
}))
|
||||
redis.call('ZADD', bidArchiveQueueKey, currentRedisTimestampMS, archiveId)
|
||||
redis.call('RPUSH', auctionBidHistoryKey, auctionBidHistoryEntryJson)
|
||||
redis.call('LTRIM', auctionBidHistoryKey, -maximumStoredAuctionBidHistoryCount, -1)
|
||||
redis.call(
|
||||
'HSET',
|
||||
liveAuctionStateKey,
|
||||
|
||||
@@ -4,6 +4,14 @@ export const AUCTION_DURABILITY_RETRY_QUEUE_KEY =
|
||||
'auction:auctionDurabilityRetryQueue';
|
||||
export const AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY =
|
||||
'auction:auctionDurabilityDeadLetterQueue';
|
||||
export const AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY =
|
||||
'auction:deadLetterStatusQueue';
|
||||
export const AUCTION_BID_ARCHIVE_QUEUE_KEY = 'auction:bidArchiveQueue';
|
||||
export const AUCTION_BID_ARCHIVE_PAYLOADS_KEY = 'auction:bidArchivePayloads';
|
||||
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY =
|
||||
'auction:bidArchiveDeadLetterQueue';
|
||||
export const AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY =
|
||||
'auction:bidArchiveDeadLetterPayloads';
|
||||
|
||||
export interface AuctionRedisKeys {
|
||||
actionRequestResultOrderKey: string;
|
||||
@@ -12,6 +20,7 @@ export interface AuctionRedisKeys {
|
||||
bidHistoryKey: string;
|
||||
bidRequestResultOrderKey: string;
|
||||
bidRequestResultsKey: string;
|
||||
rejectedBidRequestResultOrderKey: string;
|
||||
deadlineQueueKey: string;
|
||||
durabilityDeadLetterQueueKey: string;
|
||||
durabilityPayloadKey: (auctionStateTransitionId: string) => string;
|
||||
@@ -33,6 +42,7 @@ export function buildAuctionRedisKeys(matchId: string): AuctionRedisKeys {
|
||||
bidHistoryKey: `${prefix}:auctionBidHistory`,
|
||||
bidRequestResultOrderKey: `${prefix}:bidRequestResultOrder`,
|
||||
bidRequestResultsKey: `${prefix}:bidRequestResults`,
|
||||
rejectedBidRequestResultOrderKey: `${prefix}:rejectedBidRequestResultOrder`,
|
||||
deadlineQueueKey: AUCTION_BID_ROUND_DEADLINE_QUEUE_KEY,
|
||||
durabilityDeadLetterQueueKey: AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
durabilityPayloadKey: (auctionStateTransitionId: string) =>
|
||||
|
||||
@@ -24,7 +24,7 @@ describe('auction redis state assembly', () => {
|
||||
mongo: { auctionLifecycleId: 'new-life', revision: 2 },
|
||||
},
|
||||
{
|
||||
expected: false,
|
||||
expected: true,
|
||||
live: { auctionLifecycleId: 'redis-life', revision: 3 },
|
||||
mongo: { auctionLifecycleId: 'mongo-life', revision: 2 },
|
||||
},
|
||||
|
||||
@@ -62,13 +62,11 @@ export function shouldHydrateRedisAuctionState(
|
||||
params: ShouldHydrateRedisAuctionStateParams,
|
||||
): boolean {
|
||||
if (!params.live) return true;
|
||||
if (params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId)
|
||||
return true;
|
||||
const redisRevision = params.live.revision ?? -1;
|
||||
const mongoRevision = params.mongo.revision;
|
||||
if (mongoRevision > redisRevision) return true;
|
||||
return (
|
||||
mongoRevision === redisRevision &&
|
||||
params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId
|
||||
);
|
||||
return mongoRevision > redisRevision;
|
||||
}
|
||||
|
||||
export function assembleAuctionViewFromRedisState(
|
||||
@@ -140,13 +138,17 @@ if not liveStatePatchPairCount or #ARGV ~= 8 + (liveStatePatchPairCount * 2) the
|
||||
return {'REJECTED', 'INVALID_PATCH_ARITY', tostring(currentRedisTimestampMS)}
|
||||
end
|
||||
|
||||
-- A lifecycle replacement requires Mongo authority and an explicit cleanup
|
||||
-- under the lifecycle lock. Revision numbers from different runs are unrelated.
|
||||
if redisAuctionLifecycleId and redisAuctionLifecycleId ~= mongoAuctionLifecycleId then
|
||||
return {'REJECTED', 'LIFECYCLE_CONFLICT', tostring(currentRedisTimestampMS)}
|
||||
end
|
||||
|
||||
local shouldHydrate = false
|
||||
if redisRevision < 0 then
|
||||
shouldHydrate = true
|
||||
elseif mongoRevision > redisRevision then
|
||||
shouldHydrate = true
|
||||
elseif mongoRevision == redisRevision and mongoAuctionLifecycleId ~= redisAuctionLifecycleId then
|
||||
shouldHydrate = true
|
||||
end
|
||||
|
||||
if not shouldHydrate then
|
||||
@@ -184,6 +186,7 @@ if phase == 'bidding' then
|
||||
redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId)
|
||||
else
|
||||
redis.call('ZREM', bidRoundDeadlineQueueKey, matchId)
|
||||
redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS')
|
||||
end
|
||||
|
||||
return {'OK', tostring(hydratedRevision), tostring(currentRedisTimestampMS)}
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { MatchStatsFragmentBuilder } from './match-stats-fragment.builder';
|
||||
import { MatchStatsRepository } from './match-stats.repository';
|
||||
import { MatchStatsService } from './match-stats.service';
|
||||
import { MATCH_STATS_BUILDER_VERSION } from './match-stats.types';
|
||||
|
||||
describe('MatchStatsService outbox idempotency', () => {
|
||||
it('reuses the latest stored revision when an older outbox event is delivered', async () => {
|
||||
const h = createHarness();
|
||||
h.isCurrent.mockResolvedValue(true);
|
||||
|
||||
await h.service.buildAndUpsert(h.matchId, { sourceRevision: 10 });
|
||||
|
||||
expect(h.isCurrent).toHaveBeenCalledWith({
|
||||
matchId: h.matchId,
|
||||
sourceCompetitionId: h.competitionId,
|
||||
sourceRevision: 12,
|
||||
projectorVersion: 3,
|
||||
builderVersion: MATCH_STATS_BUILDER_VERSION,
|
||||
});
|
||||
expect(h.riotFind).not.toHaveBeenCalled();
|
||||
expect(h.builder.buildFragment).not.toHaveBeenCalled();
|
||||
expect(h.upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('builds once across repeated deliveries after the write succeeds', async () => {
|
||||
const h = createHarness();
|
||||
h.upsert.mockImplementation(async () => {
|
||||
h.isCurrent.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
await h.service.buildAndUpsert(h.matchId, { sourceRevision: 12 });
|
||||
await h.service.buildAndUpsert(h.matchId, { sourceRevision: 12 });
|
||||
|
||||
expect(h.riotFind).toHaveBeenCalledTimes(1);
|
||||
expect(h.builder.buildFragment).toHaveBeenCalledTimes(1);
|
||||
expect(h.upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('retries a failed write without treating an attempted build as complete', async () => {
|
||||
const h = createHarness();
|
||||
h.upsert.mockRejectedValueOnce(new Error('database unavailable'));
|
||||
await expect(
|
||||
h.service.buildAndUpsert(h.matchId, { sourceRevision: 12 }),
|
||||
).rejects.toThrow('database unavailable');
|
||||
|
||||
await h.service.buildAndUpsert(h.matchId, { sourceRevision: 12 });
|
||||
|
||||
expect(h.builder.buildFragment).toHaveBeenCalledTimes(2);
|
||||
expect(h.upsert).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each([{}, { sourceRevision: 12, includeTimeline: true }])(
|
||||
'still performs explicit rebuilds and timeline hydration (%j)',
|
||||
async (options) => {
|
||||
const h = createHarness();
|
||||
h.isCurrent.mockResolvedValue(true);
|
||||
|
||||
await h.service.buildAndUpsert(h.matchId, options);
|
||||
|
||||
expect(h.isCurrent).not.toHaveBeenCalled();
|
||||
expect(h.builder.buildFragment).toHaveBeenCalledTimes(1);
|
||||
expect(h.upsert).toHaveBeenCalledTimes(1);
|
||||
if ('includeTimeline' in options) {
|
||||
expect(h.getMatchTimeline).toHaveBeenCalledWith('KR_1', {
|
||||
region: 'asia',
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('deletes reversed zero-game stats even if their versions were current', async () => {
|
||||
const h = createHarness();
|
||||
h.competition.projection.games = [];
|
||||
h.isCurrent.mockResolvedValue(true);
|
||||
|
||||
await h.service.buildAndUpsert(h.matchId, { sourceRevision: 12 });
|
||||
|
||||
expect(h.isCurrent).not.toHaveBeenCalled();
|
||||
expect(h.deleteByMatch).toHaveBeenCalledWith(h.matchId, {
|
||||
sourceCompetitionId: h.competitionId,
|
||||
sourceRevision: 12,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function createHarness() {
|
||||
const matchId = new mongoose.Types.ObjectId();
|
||||
const competitionId = new mongoose.Types.ObjectId();
|
||||
const teamAId = new mongoose.Types.ObjectId();
|
||||
const teamBId = new mongoose.Types.ObjectId();
|
||||
const match = {
|
||||
_id: matchId,
|
||||
communityId: new mongoose.Types.ObjectId(),
|
||||
status: 'in-progress',
|
||||
scheduledAt: new Date(),
|
||||
teams: [],
|
||||
};
|
||||
const competition = {
|
||||
_id: competitionId,
|
||||
revision: 12,
|
||||
projectorVersion: 3,
|
||||
teams: [
|
||||
{ teamId: teamAId, displayOrder: 1 },
|
||||
{ teamId: teamBId, displayOrder: 2 },
|
||||
],
|
||||
projection: {
|
||||
phase: 'running',
|
||||
games: [
|
||||
{
|
||||
displayNumber: 1,
|
||||
evidence: { type: 'riot', riotMatchId: 'KR_1' },
|
||||
teams: { blue: { teamId: teamAId }, red: { teamId: teamBId } },
|
||||
winnerTeamId: teamAId,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const riotFind = jest.fn(() => query([{ matchId: 'KR_1', raw: {} }]));
|
||||
const models = {
|
||||
match: { findById: () => query(match) },
|
||||
competition: { findOne: () => query(competition) },
|
||||
riotMatch: { find: riotFind },
|
||||
community: { findById: () => query({ riotRegion: 'asia' }) },
|
||||
};
|
||||
const connection = {
|
||||
getModel: async (name: keyof typeof models) => models[name],
|
||||
} as unknown as ConnectionService;
|
||||
const isCurrent = jest.fn().mockResolvedValue(false);
|
||||
const upsert = jest.fn().mockResolvedValue(undefined);
|
||||
const deleteByMatch = jest.fn().mockResolvedValue(undefined);
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: isCurrent,
|
||||
upsert,
|
||||
deleteByMatch,
|
||||
} as unknown as MatchStatsRepository;
|
||||
const builder = {
|
||||
buildFragment: jest.fn().mockReturnValue({
|
||||
matchCard: {},
|
||||
records: [],
|
||||
gameDetails: [],
|
||||
userIds: [],
|
||||
championIds: [],
|
||||
gameCreationMin: 1,
|
||||
gameCreationMax: 2,
|
||||
}),
|
||||
} as unknown as jest.Mocked<MatchStatsFragmentBuilder>;
|
||||
const getMatchTimeline = jest.fn().mockResolvedValue(undefined);
|
||||
const service = new MatchStatsService(connection, repository, builder, {
|
||||
getMatchTimeline,
|
||||
} as never);
|
||||
return {
|
||||
service,
|
||||
matchId,
|
||||
competitionId,
|
||||
competition,
|
||||
isCurrent,
|
||||
upsert,
|
||||
deleteByMatch,
|
||||
riotFind,
|
||||
builder,
|
||||
getMatchTimeline,
|
||||
};
|
||||
}
|
||||
|
||||
function query<T>(value: T) {
|
||||
return {
|
||||
select: jest.fn().mockReturnThis(),
|
||||
lean: jest.fn().mockReturnThis(),
|
||||
exec: async () => value,
|
||||
};
|
||||
}
|
||||
@@ -4,6 +4,35 @@ import type { MatchStats } from '../global/database/schema';
|
||||
import { MatchStatsRepository } from './match-stats.repository';
|
||||
|
||||
describe('MatchStatsRepository', () => {
|
||||
it.each([null, { _id: new Types.ObjectId() }])(
|
||||
'checks persisted source identity and every build version without loading payloads (%j)',
|
||||
async (stored) => {
|
||||
const exists = jest.fn(() => ({ exec: async () => stored }));
|
||||
const connection = {
|
||||
getModel: jest.fn().mockResolvedValue({ exists }),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = new MatchStatsRepository(connection);
|
||||
const source = {
|
||||
matchId: new Types.ObjectId(),
|
||||
sourceCompetitionId: new Types.ObjectId(),
|
||||
sourceRevision: 12,
|
||||
projectorVersion: 3,
|
||||
builderVersion: 10,
|
||||
};
|
||||
|
||||
await expect(
|
||||
repository.hasCurrentCompetitionFragment(source),
|
||||
).resolves.toBe(Boolean(stored));
|
||||
expect(exists).toHaveBeenCalledWith({
|
||||
matchId: source.matchId,
|
||||
sourceCompetitionId: source.sourceCompetitionId,
|
||||
sourceRevision: { $gte: 12 },
|
||||
projectorVersion: { $gte: 3 },
|
||||
builderVersion: { $gte: 10 },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it('does not let a legacy upsert overwrite a Competition-backed fragment', async () => {
|
||||
const fragment = {
|
||||
...statsFragment(12),
|
||||
|
||||
@@ -22,6 +22,25 @@ export interface MatchStatsDeleteGuard {
|
||||
export class MatchStatsRepository {
|
||||
constructor(private readonly connectionService: ConnectionService) {}
|
||||
|
||||
public async hasCurrentCompetitionFragment(params: {
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
sourceCompetitionId: mongoose.Types.ObjectId;
|
||||
sourceRevision: number;
|
||||
projectorVersion: number;
|
||||
builderVersion: number;
|
||||
}): Promise<boolean> {
|
||||
const MatchStatsModel = await this.connectionService.getModel('matchStats');
|
||||
return Boolean(
|
||||
await MatchStatsModel.exists({
|
||||
matchId: params.matchId,
|
||||
sourceCompetitionId: params.sourceCompetitionId,
|
||||
sourceRevision: { $gte: params.sourceRevision },
|
||||
projectorVersion: { $gte: params.projectorVersion },
|
||||
builderVersion: { $gte: params.builderVersion },
|
||||
}).exec(),
|
||||
);
|
||||
}
|
||||
|
||||
public async upsert(
|
||||
fragment: Omit<MatchStats, 'createdAt' | 'updatedAt'>,
|
||||
): Promise<void> {
|
||||
|
||||
@@ -24,6 +24,7 @@ describe('MatchStatsService', () => {
|
||||
}),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: jest.fn().mockResolvedValue(false),
|
||||
deleteByMatch: jest.fn().mockResolvedValue(undefined),
|
||||
upsert: jest.fn(),
|
||||
} as unknown as jest.Mocked<MatchStatsRepository>;
|
||||
@@ -72,6 +73,7 @@ describe('MatchStatsService', () => {
|
||||
}),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: jest.fn().mockResolvedValue(false),
|
||||
deleteByMatch: jest.fn().mockResolvedValue(undefined),
|
||||
upsert: jest.fn(),
|
||||
} as unknown as jest.Mocked<MatchStatsRepository>;
|
||||
@@ -132,6 +134,7 @@ describe('MatchStatsService', () => {
|
||||
}),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: jest.fn().mockResolvedValue(false),
|
||||
upsert: jest.fn().mockResolvedValue(undefined),
|
||||
deleteByMatch: jest.fn(),
|
||||
} as unknown as jest.Mocked<MatchStatsRepository>;
|
||||
@@ -273,6 +276,7 @@ describe('MatchStatsService', () => {
|
||||
}),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: jest.fn().mockResolvedValue(false),
|
||||
upsert: jest.fn().mockResolvedValue(undefined),
|
||||
deleteByMatch: jest.fn(),
|
||||
deleteLegacyByMatch: jest.fn(),
|
||||
@@ -408,6 +412,7 @@ describe('MatchStatsService', () => {
|
||||
}),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: jest.fn().mockResolvedValue(false),
|
||||
upsert: jest.fn().mockResolvedValue(undefined),
|
||||
deleteByMatch: jest.fn(),
|
||||
} as unknown as jest.Mocked<MatchStatsRepository>;
|
||||
@@ -512,6 +517,7 @@ describe('MatchStatsService', () => {
|
||||
}),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: jest.fn().mockResolvedValue(false),
|
||||
upsert: jest.fn().mockResolvedValue(undefined),
|
||||
deleteByMatch: jest.fn().mockResolvedValue(undefined),
|
||||
deleteLegacyByMatch: jest.fn(),
|
||||
@@ -573,6 +579,7 @@ describe('MatchStatsService', () => {
|
||||
}),
|
||||
} as unknown as ConnectionService;
|
||||
const repository = {
|
||||
hasCurrentCompetitionFragment: jest.fn().mockResolvedValue(false),
|
||||
upsert: jest.fn(),
|
||||
deleteByMatch: jest.fn(),
|
||||
} as unknown as jest.Mocked<MatchStatsRepository>;
|
||||
|
||||
@@ -59,6 +59,24 @@ export class MatchStatsService {
|
||||
return;
|
||||
}
|
||||
|
||||
// Outbox redelivery (including older events observing the latest revision)
|
||||
// must not reload Riot payloads or replace an already enriched fragment.
|
||||
// Explicit rebuilds and timeline hydration still run at the same revision.
|
||||
if (
|
||||
options.sourceRevision !== undefined &&
|
||||
!options.includeTimeline &&
|
||||
competitionSource &&
|
||||
(await this.repository.hasCurrentCompetitionFragment({
|
||||
matchId,
|
||||
sourceCompetitionId: competitionSource.sourceCompetitionId,
|
||||
sourceRevision: competitionSource.sourceRevision,
|
||||
projectorVersion: competitionSource.projectorVersion,
|
||||
builderVersion: MATCH_STATS_BUILDER_VERSION,
|
||||
}))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const riotIds = competitionSource?.riotMatchIds ?? [
|
||||
...new Set(
|
||||
(match.result?.games ?? [])
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
export const AUCTION_CANCELLATION_CLEANUP_INDEX = {
|
||||
key: { auctionCancellationCleanupRetryAt: 1 as const },
|
||||
options: {
|
||||
name: 'auction_cancellation_cleanup_due',
|
||||
partialFilterExpression: { auctionCancellationCleanupPending: true },
|
||||
},
|
||||
};
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import type { mongo } from 'mongoose';
|
||||
import { AUCTION_CANCELLATION_CLEANUP_INDEX } from '../auction-cancellation-index';
|
||||
|
||||
/** Creates only this additive index; never drops or rebuilds other indexes. */
|
||||
export async function migrateAuctionCancellationIndex(
|
||||
db: mongo.Db,
|
||||
{ dryRun = true }: { dryRun?: boolean } = {},
|
||||
) {
|
||||
const { key, options } = AUCTION_CANCELLATION_CLEANUP_INDEX;
|
||||
const collection = db.collection('match');
|
||||
const indexes = await collection
|
||||
.listIndexes()
|
||||
.toArray()
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof Error && 'code' in error && error.code === 26)
|
||||
return [];
|
||||
throw error;
|
||||
});
|
||||
const existing = indexes.find((index) => index.name === options.name);
|
||||
if (
|
||||
existing &&
|
||||
(JSON.stringify(existing.key) !== JSON.stringify(key) ||
|
||||
JSON.stringify(existing.partialFilterExpression) !==
|
||||
JSON.stringify(options.partialFilterExpression) ||
|
||||
existing.unique === true ||
|
||||
existing.sparse === true ||
|
||||
existing.hidden === true)
|
||||
) {
|
||||
throw new Error(
|
||||
`Index ${options.name} already exists with a different definition.`,
|
||||
);
|
||||
}
|
||||
if (!dryRun) await collection.createIndex(key, options);
|
||||
return {
|
||||
dryRun,
|
||||
alreadyPresent: Boolean(existing),
|
||||
index: { key, ...options },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import 'reflect-metadata';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import { ConnectionService, resolveMongoDbName } from '../connection.service';
|
||||
import { SecretModule } from '../../secret/secret.module';
|
||||
import { migrateAuctionCancellationIndex } from './20261109-auction-cancellation-index';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ConfigModule.forRoot({
|
||||
isGlobal: true,
|
||||
envFilePath: [
|
||||
resolve(__dirname, '../../../../../../project.env'),
|
||||
resolve(process.cwd(), '../project.env'),
|
||||
resolve(process.cwd(), 'apps/project.env'),
|
||||
resolve(process.cwd(), 'project.env'),
|
||||
],
|
||||
}),
|
||||
SecretModule,
|
||||
],
|
||||
providers: [ConnectionService],
|
||||
})
|
||||
class MigrationModule {}
|
||||
|
||||
export function parseAuctionIndexOptions(argv: string[]) {
|
||||
let dryRun = true;
|
||||
let help = false;
|
||||
let nodeEnv = process.env.NODE_ENV ?? 'dev';
|
||||
for (let index = 0; index < argv.length; index++) {
|
||||
const argument = argv[index];
|
||||
if (argument === '--') continue;
|
||||
if (argument === '--apply') dryRun = false;
|
||||
else if (argument === '--dry-run') dryRun = true;
|
||||
else if (argument === '--help' || argument === '-h') help = true;
|
||||
else if (argument === '--env') nodeEnv = argv[++index] ?? '';
|
||||
else if (argument.startsWith('--env=')) nodeEnv = argument.slice(6);
|
||||
else throw new Error(`Unknown argument: ${argument}`);
|
||||
}
|
||||
if (!help && !['dev', 'staging', 'prod'].includes(nodeEnv)) {
|
||||
throw new Error('--env must be dev, staging, or prod.');
|
||||
}
|
||||
return { dryRun, help, nodeEnv };
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseAuctionIndexOptions(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
console.log(`Usage:
|
||||
pnpm --filter api auction:indexes -- --env staging --dry-run
|
||||
pnpm --filter api auction:indexes -- --env prod --apply
|
||||
|
||||
Defaults to dry-run. --env accepts dev, staging, or prod (NODE_ENV or dev by default).
|
||||
Creates the Match cancellation cleanup partial index; does not modify documents.`);
|
||||
return;
|
||||
}
|
||||
// Use the same optional machine environment as the existing migration runners.
|
||||
const machineHome = process.env.HOME ?? process.env.USERPROFILE;
|
||||
if (machineHome) {
|
||||
const contents = await readFile(
|
||||
resolve(machineHome, 'source/global-env-vars.txt'),
|
||||
'utf8',
|
||||
).catch(() => '');
|
||||
for (const line of contents.split(/\r?\n/)) {
|
||||
const match = line.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
|
||||
if (match) process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');
|
||||
}
|
||||
}
|
||||
process.env.NODE_ENV = options.nodeEnv;
|
||||
// Even a dry-run with auto-index enabled in the environment must not write.
|
||||
process.env.MONGODB_AUTO_INDEX = 'false';
|
||||
const app = await NestFactory.createApplicationContext(MigrationModule, {
|
||||
logger: ['warn', 'error'],
|
||||
});
|
||||
try {
|
||||
const connection = await app.get(ConnectionService).getConnection();
|
||||
if (!connection.db) throw new Error('MongoDB connection has no db handle.');
|
||||
const result = await migrateAuctionCancellationIndex(
|
||||
connection.db,
|
||||
options,
|
||||
);
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
env: options.nodeEnv,
|
||||
db: resolveMongoDbName(options.nodeEnv),
|
||||
...result,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
void main().catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
competitionModelDefinitions,
|
||||
} from '../../competition/competition.schema';
|
||||
import { ModelDefinition } from './types';
|
||||
import { AUCTION_CANCELLATION_CLEANUP_INDEX } from './auction-cancellation-index';
|
||||
|
||||
export type Position = 'TOP' | 'JGL' | 'MID' | 'ADC' | 'SUP';
|
||||
export const POSITIONS: Position[] = ['TOP', 'JGL', 'MID', 'ADC', 'SUP'];
|
||||
@@ -700,6 +701,12 @@ export interface Match extends MongooseTimestamps {
|
||||
teams: MatchTeam[];
|
||||
result?: MatchResult;
|
||||
resultPollState?: MatchResultPollState;
|
||||
/** 마지막으로 취소된 Redis 경매 lifecycle. 취소 재전송을 멱등 처리한다. */
|
||||
lastCancelledAuctionLifecycleId?: string;
|
||||
auctionCancellationCleanupPending?: boolean;
|
||||
auctionCancellationCleanupRetryAt?: Date;
|
||||
auctionCancellationCleanupAttempts?: number;
|
||||
auctionCancellationCleanupError?: string;
|
||||
doubleElimination?: DoubleEliminationBracket;
|
||||
legacyImported?: boolean;
|
||||
legacyImportSource?: {
|
||||
@@ -804,12 +811,41 @@ export interface AuctionBid {
|
||||
}
|
||||
|
||||
export interface AuctionBidHistoryEntry {
|
||||
bidRequestId?: string;
|
||||
revision?: number;
|
||||
bidRoundId?: string;
|
||||
playerUserId?: mongoose.Types.ObjectId;
|
||||
bidderTeamIndex: number;
|
||||
bidPointAmount: number;
|
||||
bidderUserId: mongoose.Types.ObjectId;
|
||||
bidPlacedAtTimestampMS: number;
|
||||
}
|
||||
|
||||
export interface AuctionBidRecord extends AuctionBidHistoryEntry {
|
||||
_id: string;
|
||||
matchId: mongoose.Types.ObjectId;
|
||||
auctionLifecycleId: string;
|
||||
revision: number;
|
||||
bidRequestId: string;
|
||||
}
|
||||
|
||||
export const AuctionBidRecordSchema = new Schema<AuctionBidRecord>(
|
||||
{
|
||||
_id: { type: String, required: true },
|
||||
matchId: { type: Schema.Types.ObjectId, required: true },
|
||||
auctionLifecycleId: { type: String, required: true },
|
||||
revision: { type: Number, required: true },
|
||||
bidRequestId: { type: String, required: true },
|
||||
bidRoundId: { type: String },
|
||||
playerUserId: { type: Schema.Types.ObjectId },
|
||||
bidderUserId: { type: Schema.Types.ObjectId, required: true },
|
||||
bidderTeamIndex: { type: Number, required: true },
|
||||
bidPointAmount: { type: Number, required: true },
|
||||
bidPlacedAtTimestampMS: { type: Number, required: true },
|
||||
},
|
||||
{ collection: 'auctionBidRecords', versionKey: false },
|
||||
);
|
||||
|
||||
export interface AuctionChatMessage {
|
||||
id: string;
|
||||
userId?: mongoose.Types.ObjectId;
|
||||
@@ -832,6 +868,8 @@ export interface Auction extends MongooseTimestamps {
|
||||
matchSideEffectStatus: AuctionMatchSideEffectStatus;
|
||||
matchSideEffectRevision: number;
|
||||
matchSideEffectError?: string;
|
||||
matchSideEffectTransitionId?: string;
|
||||
startupCompleted?: boolean;
|
||||
teams: AuctionTeam[];
|
||||
order: AuctionPlayerSnapshot[];
|
||||
passedQueue: AuctionPlayerSnapshot[];
|
||||
@@ -858,6 +896,7 @@ export interface Auction extends MongooseTimestamps {
|
||||
currentHighestBidPointAmount?: number;
|
||||
currentHighestBidderTeamIndex?: number;
|
||||
currentBidRoundDeadlineTimestampMS?: number;
|
||||
currentBidRoundId?: string;
|
||||
maximumBidRoundDurationSeconds?: number;
|
||||
bidRoundExtensionSeconds?: number;
|
||||
initialTeamBidPointAmount?: number;
|
||||
@@ -1971,6 +2010,11 @@ export const MatchSchema = new Schema<Match>(
|
||||
type: MatchResultPollStateSubSchema,
|
||||
required: false,
|
||||
},
|
||||
lastCancelledAuctionLifecycleId: { type: String, required: false },
|
||||
auctionCancellationCleanupPending: { type: Boolean },
|
||||
auctionCancellationCleanupRetryAt: { type: Date },
|
||||
auctionCancellationCleanupAttempts: { type: Number },
|
||||
auctionCancellationCleanupError: { type: String },
|
||||
doubleElimination: {
|
||||
type: DoubleEliminationBracketSubSchema,
|
||||
required: false,
|
||||
@@ -1995,6 +2039,10 @@ export const MatchSchema = new Schema<Match>(
|
||||
);
|
||||
|
||||
MatchSchema.index({ communityId: 1, scheduledAt: -1 });
|
||||
MatchSchema.index(
|
||||
AUCTION_CANCELLATION_CLEANUP_INDEX.key,
|
||||
AUCTION_CANCELLATION_CLEANUP_INDEX.options,
|
||||
);
|
||||
applyCompetitionDomainInvariantIndexes(MatchSchema, 'match');
|
||||
MatchSchema.index({ communityId: 1, status: 1, scheduledAt: -1 });
|
||||
MatchSchema.index(
|
||||
@@ -2174,6 +2222,8 @@ const AuctionBidSubSchema = new Schema<AuctionBid>(
|
||||
|
||||
const AuctionBidHistoryEntrySubSchema = new Schema<AuctionBidHistoryEntry>(
|
||||
{
|
||||
bidRoundId: { type: String },
|
||||
playerUserId: { type: Schema.Types.ObjectId, ref: 'user' },
|
||||
bidderTeamIndex: { type: Number, required: true, min: 1 },
|
||||
bidPointAmount: { type: Number, required: true },
|
||||
bidderUserId: { type: Schema.Types.ObjectId, ref: 'user', required: true },
|
||||
@@ -2195,6 +2245,9 @@ const AuctionChatMessageSubSchema = new Schema<AuctionChatMessage>(
|
||||
|
||||
export const AuctionSchema = new Schema<Auction>(
|
||||
{
|
||||
startupCompleted: { type: Boolean },
|
||||
matchSideEffectTransitionId: { type: String },
|
||||
currentBidRoundId: { type: String },
|
||||
matchId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'match',
|
||||
@@ -2520,6 +2573,11 @@ export const tournamentModelDefinitions: Array<
|
||||
},
|
||||
{ name: 'match', schema: MatchSchema, schemaName: 'match' },
|
||||
{ name: 'auction', schema: AuctionSchema, schemaName: 'auction' },
|
||||
{
|
||||
name: 'auctionBidRecord',
|
||||
schema: AuctionBidRecordSchema,
|
||||
schemaName: 'auctionBidRecords',
|
||||
},
|
||||
{
|
||||
name: 'auctionHistory',
|
||||
schema: AuctionHistoryEntrySchema,
|
||||
@@ -2553,6 +2611,7 @@ export interface TournamentModelTypes {
|
||||
legacyImportJob: LegacyImportJob;
|
||||
match: Match;
|
||||
auction: Auction;
|
||||
auctionBidRecord: AuctionBidRecord;
|
||||
auctionHistory: AuctionHistoryEntry;
|
||||
auctionSideEffectApplication: AuctionSideEffectApplication;
|
||||
matchStats: MatchStats;
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { MatchHistoryService } from './match-history.service';
|
||||
|
||||
const SEASON_START = Date.parse('2026-01-01T00:00:00Z');
|
||||
|
||||
describe('MatchHistoryService champion filtering', () => {
|
||||
it('paginates matching games beyond 120 newer games and excludes opponents using the champion', async () => {
|
||||
const harness = createHarness();
|
||||
const filters = { championId: 99, rank: 'solo', limit: 2 };
|
||||
const first = await harness.service.getRecentMatches({
|
||||
...harness.scope,
|
||||
filters,
|
||||
});
|
||||
|
||||
expect(first.matches.map((match) => match.matchId)).toEqual([
|
||||
'KR_3',
|
||||
'KR_2',
|
||||
]);
|
||||
expect(first.nextCursor).toBe(SEASON_START + 2_000);
|
||||
const second = await harness.service.getRecentMatches({
|
||||
...harness.scope,
|
||||
filters: { ...filters, cursor: first.nextCursor! },
|
||||
});
|
||||
expect(second.matches.map((match) => match.matchId)).toEqual(['KR_1']);
|
||||
expect(second.nextCursor).toBeNull();
|
||||
expect(harness.returnedCounts).toEqual([3, 1]);
|
||||
expect(harness.find).toHaveBeenLastCalledWith({
|
||||
participantPuuids: 'player',
|
||||
queueId: { $in: [420] },
|
||||
gameCreation: { $gte: SEASON_START, $lt: first.nextCursor },
|
||||
'raw.info.participants': {
|
||||
$elemMatch: { puuid: 'player', championId: 99 },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('summarizes matching games across the former scan boundary', async () => {
|
||||
const { service, scope, returnedCounts } = createHarness();
|
||||
const summary = await service.getRecentSummary({
|
||||
...scope,
|
||||
filters: { championId: 99, rank: 'solo', limit: 20 },
|
||||
});
|
||||
|
||||
expect(summary).toMatchObject({ windowSize: 3, wins: 3, losses: 0 });
|
||||
expect(summary.topChampions).toEqual([
|
||||
expect.objectContaining({ championId: 99, games: 3 }),
|
||||
]);
|
||||
expect(returnedCounts).toEqual([3]);
|
||||
});
|
||||
|
||||
it('keeps unfiltered pagination and does not transfer timeline payloads', async () => {
|
||||
const { service, scope, find, select, returnedCounts } = createHarness();
|
||||
const result = await service.getRecentMatches({
|
||||
...scope,
|
||||
filters: { rank: 'solo', limit: 2 },
|
||||
});
|
||||
|
||||
expect(result.matches.map((match) => match.matchId)).toEqual([
|
||||
'KR_130',
|
||||
'KR_129',
|
||||
]);
|
||||
expect(result.nextCursor).toBe(SEASON_START + 129_000);
|
||||
expect(find.mock.calls[0][0]['raw.info.participants']).toBeUndefined();
|
||||
expect(select).toHaveBeenCalledWith({ timeline: 0 });
|
||||
expect(returnedCounts).toEqual([3]);
|
||||
});
|
||||
});
|
||||
|
||||
interface HistoryQuery {
|
||||
participantPuuids: string;
|
||||
queueId: { $in: number[] };
|
||||
gameCreation?: { $gte?: number; $lt?: number };
|
||||
'raw.info.participants'?: {
|
||||
$elemMatch: { puuid: string; championId: number };
|
||||
};
|
||||
}
|
||||
|
||||
function createHarness() {
|
||||
const communityId = new mongoose.Types.ObjectId();
|
||||
const userId = new mongoose.Types.ObjectId();
|
||||
// In the newer games only the opponent played champion 99.
|
||||
const rows = [
|
||||
...Array.from({ length: 127 }, (_, index) => storedMatch(index + 4, 1)),
|
||||
storedMatch(3, 99),
|
||||
storedMatch(2, 99),
|
||||
storedMatch(1, 99),
|
||||
storedMatch(131, 99, 450), // Wrong queue.
|
||||
storedMatch(-1, 99), // Before the season.
|
||||
];
|
||||
const returnedCounts: number[] = [];
|
||||
const select = jest.fn();
|
||||
const find = jest.fn((filter: HistoryQuery) => {
|
||||
let limit = rows.length;
|
||||
const query = {
|
||||
sort: jest.fn().mockReturnThis(),
|
||||
limit: (value: number) => {
|
||||
limit = value;
|
||||
return query;
|
||||
},
|
||||
select: (projection: unknown) => {
|
||||
select(projection);
|
||||
return query;
|
||||
},
|
||||
exec: async () => {
|
||||
const participant = filter['raw.info.participants']?.$elemMatch;
|
||||
const result = rows
|
||||
.filter(
|
||||
(row) =>
|
||||
row.participantPuuids.includes(filter.participantPuuids) &&
|
||||
filter.queueId.$in.includes(row.queueId) &&
|
||||
(filter.gameCreation?.$gte === undefined ||
|
||||
row.gameCreation >= filter.gameCreation.$gte) &&
|
||||
(filter.gameCreation?.$lt === undefined ||
|
||||
row.gameCreation < filter.gameCreation.$lt) &&
|
||||
(!participant ||
|
||||
row.raw.info.participants.some(
|
||||
(entry) =>
|
||||
entry.puuid === participant.puuid &&
|
||||
entry.championId === participant.championId,
|
||||
)),
|
||||
)
|
||||
.sort((a, b) => b.gameCreation - a.gameCreation)
|
||||
.slice(0, limit);
|
||||
returnedCounts.push(result.length);
|
||||
return result;
|
||||
},
|
||||
};
|
||||
return query;
|
||||
});
|
||||
const models = {
|
||||
user: {
|
||||
findById: () => ({
|
||||
exec: async () => ({
|
||||
_id: userId,
|
||||
communityId,
|
||||
riotAccounts: [{ puuid: 'player', isPrimary: true }],
|
||||
}),
|
||||
}),
|
||||
},
|
||||
community: {
|
||||
findById: () => ({
|
||||
exec: async () => ({
|
||||
_id: communityId,
|
||||
seasonStartedAt: new Date(SEASON_START),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
riotMatch: { find },
|
||||
};
|
||||
const connection = {
|
||||
getModel: async (name: keyof typeof models) => models[name],
|
||||
} as unknown as ConnectionService;
|
||||
return {
|
||||
service: new MatchHistoryService(connection),
|
||||
scope: { communityId, userId },
|
||||
find,
|
||||
select,
|
||||
returnedCounts,
|
||||
};
|
||||
}
|
||||
|
||||
function storedMatch(sequence: number, championId: number, queueId = 420) {
|
||||
const gameCreation = SEASON_START + sequence * 1_000;
|
||||
const participant = {
|
||||
puuid: 'player',
|
||||
championId,
|
||||
teamId: 100,
|
||||
win: true,
|
||||
kills: 5,
|
||||
deaths: 2,
|
||||
assists: 3,
|
||||
totalMinionsKilled: 100,
|
||||
neutralMinionsKilled: 20,
|
||||
goldEarned: 10_000,
|
||||
totalDamageDealtToChampions: 10_000,
|
||||
totalDamageTaken: 5_000,
|
||||
perks: { styles: [] },
|
||||
};
|
||||
return {
|
||||
gameCreation,
|
||||
queueId,
|
||||
participantPuuids: ['player', 'opponent'],
|
||||
fetchedAt: new Date(),
|
||||
raw: {
|
||||
metadata: { matchId: `KR_${sequence}` },
|
||||
info: {
|
||||
gameCreation,
|
||||
queueId,
|
||||
gameDuration: 1_800,
|
||||
participants: [
|
||||
participant,
|
||||
{
|
||||
...participant,
|
||||
puuid: 'opponent',
|
||||
teamId: 200,
|
||||
championId: 99,
|
||||
win: false,
|
||||
},
|
||||
],
|
||||
teams: [
|
||||
{ teamId: 100, win: true },
|
||||
{ teamId: 200, win: false },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -36,7 +36,6 @@ const DEFAULT_RANK: RiotRankFilter = 'all';
|
||||
const DEFAULT_RECENT_LIMIT = 10;
|
||||
const DEFAULT_SUMMARY_LIMIT = 20;
|
||||
const MAX_AGGREGATE_MATCHES = 500;
|
||||
const MAX_RECENT_SCAN = 120;
|
||||
|
||||
interface HistoryContext {
|
||||
community: CommunityDocument;
|
||||
@@ -184,7 +183,6 @@ export class MatchHistoryService {
|
||||
queueIds,
|
||||
championId: params.filters.championId,
|
||||
limit,
|
||||
scanLimit: params.filters.championId ? MAX_RECENT_SCAN : undefined,
|
||||
});
|
||||
const views = docs
|
||||
.map((doc) => this.toRecentMatchView(doc, context.account.puuid))
|
||||
@@ -215,7 +213,6 @@ export class MatchHistoryService {
|
||||
championId: params.filters.championId,
|
||||
limit: limit + 1,
|
||||
cursor: params.filters.cursor,
|
||||
scanLimit: params.filters.championId ? MAX_RECENT_SCAN : undefined,
|
||||
});
|
||||
const views = docs
|
||||
.map((doc) => this.toRecentMatchView(doc, context.account.puuid))
|
||||
@@ -337,7 +334,6 @@ export class MatchHistoryService {
|
||||
limit: number;
|
||||
cursor?: number;
|
||||
championId?: number;
|
||||
scanLimit?: number;
|
||||
}): Promise<RiotMatchDocument[]> {
|
||||
if (params.queueIds.length === 0) return [];
|
||||
const RiotMatchModel = await this.connectionService.getModel('riotMatch');
|
||||
@@ -352,22 +348,21 @@ export class MatchHistoryService {
|
||||
queueId: { $in: params.queueIds },
|
||||
};
|
||||
if (Object.keys(gameCreation).length > 0) query.gameCreation = gameCreation;
|
||||
if (params.championId !== undefined) {
|
||||
// Both conditions must match the same participant, before pagination.
|
||||
query['raw.info.participants'] = {
|
||||
$elemMatch: {
|
||||
puuid: params.context.account.puuid,
|
||||
championId: params.championId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const docs = await RiotMatchModel.find(query)
|
||||
return RiotMatchModel.find(query)
|
||||
.sort({ gameCreation: -1 })
|
||||
.limit(params.scanLimit ?? params.limit)
|
||||
.limit(params.limit)
|
||||
.select({ timeline: 0 })
|
||||
.exec();
|
||||
|
||||
if (!params.championId) return docs.slice(0, params.limit);
|
||||
return docs
|
||||
.filter((doc) => {
|
||||
const participant = this.findParticipant(
|
||||
doc.raw as unknown as RiotMatchDto,
|
||||
params.context.account.puuid,
|
||||
);
|
||||
return participant?.championId === params.championId;
|
||||
})
|
||||
.slice(0, params.limit);
|
||||
}
|
||||
|
||||
private async aggregateChampionStats(params: {
|
||||
|
||||
@@ -90,6 +90,33 @@ function createMatch(overrides: MatchOverrides): MatchDocument {
|
||||
} as unknown as MatchDocument;
|
||||
}
|
||||
|
||||
it('releases reserved codes inside the caller transaction during auction cancellation', async () => {
|
||||
const session = createSession();
|
||||
const match = createMatch({
|
||||
tournamentCodeIds: [new mongoose.Types.ObjectId()],
|
||||
});
|
||||
const matchQuery = mongoQuery(match);
|
||||
const codeQuery = mongoQuery({ modifiedCount: 1 });
|
||||
const createSessionSpy = jest.fn();
|
||||
const service = new TournamentCodeService(
|
||||
{
|
||||
createSession: createSessionSpy,
|
||||
getModel: async (name: string) =>
|
||||
name === 'match'
|
||||
? { findById: () => matchQuery }
|
||||
: { updateMany: () => codeQuery },
|
||||
} as never,
|
||||
createAuctionRedisService() as never,
|
||||
);
|
||||
await service.releaseFromMatch(match, session as never);
|
||||
expect(createSessionSpy).not.toHaveBeenCalled();
|
||||
expect(matchQuery.session).not.toHaveBeenCalled();
|
||||
expect(codeQuery.session).toHaveBeenCalledWith(session);
|
||||
expect(match.save).toHaveBeenCalledWith({ session });
|
||||
expect(session.commitTransaction).not.toHaveBeenCalled();
|
||||
expect(match.tournamentCodeIds).toEqual([]);
|
||||
});
|
||||
|
||||
describe('TournamentCodeService round-robin scheduling', () => {
|
||||
it('keeps pair slot calculation unchanged', () => {
|
||||
const service = createService();
|
||||
|
||||
@@ -329,23 +329,31 @@ export class TournamentCodeService {
|
||||
|
||||
public async releaseFromMatch(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
session?: mongoose.ClientSession,
|
||||
): Promise<void> {
|
||||
const matchId = this.resolveMatchId(matchOrId);
|
||||
// Auction cancellation owns the surrounding transaction. Committing a
|
||||
// separate release would survive its rollback and conflict with Match.save.
|
||||
if (session) return this.releaseFromMatchInTransaction(matchOrId, session);
|
||||
await this.withLockedMatchTransaction(matchId, (session) =>
|
||||
this.releaseFromMatchInTransaction(matchId, session),
|
||||
);
|
||||
}
|
||||
|
||||
private async releaseFromMatchInTransaction(
|
||||
matchId: mongoose.Types.ObjectId,
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
session: mongoose.ClientSession,
|
||||
): Promise<void> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
const match = await this.execQuery<MatchDocument | null>(
|
||||
this.withQuerySession(MatchModel.findById(matchId), session),
|
||||
);
|
||||
// Preserve the caller's document and __v when participating in its session.
|
||||
const match =
|
||||
matchOrId instanceof mongoose.Types.ObjectId
|
||||
? await this.execQuery<MatchDocument | null>(
|
||||
this.withQuerySession(MatchModel.findById(matchOrId), session),
|
||||
)
|
||||
: matchOrId;
|
||||
if (!match) return;
|
||||
if (match.tournamentCodeIds.length === 0) return;
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Logger } from '@nestjs/common';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { CompetitionAssignmentProvisioner } from './competition-assignment.provisioner';
|
||||
import { CompetitionOutboxProcessor } from './competition-outbox.processor';
|
||||
|
||||
@@ -82,6 +83,7 @@ describe('CompetitionOutboxProcessor', () => {
|
||||
provisioner,
|
||||
stats,
|
||||
communityStats,
|
||||
availableLock(),
|
||||
);
|
||||
|
||||
await expect(processor.drainOnce()).resolves.toBe(1);
|
||||
@@ -147,6 +149,7 @@ describe('CompetitionOutboxProcessor', () => {
|
||||
{ provisionReadyAssignments: jest.fn() } as never,
|
||||
{ buildAndUpsert: jest.fn() } as never,
|
||||
{ invalidateCommunity: jest.fn() } as never,
|
||||
availableLock(),
|
||||
);
|
||||
|
||||
await expect(processor.drainOnce()).resolves.toBe(1);
|
||||
@@ -198,6 +201,7 @@ describe('CompetitionOutboxProcessor', () => {
|
||||
{ provisionReadyAssignments: jest.fn() } as never,
|
||||
{ buildAndUpsert: jest.fn() } as never,
|
||||
{ invalidateCommunity: jest.fn() } as never,
|
||||
availableLock(),
|
||||
);
|
||||
|
||||
await expect(processor.drainOnce()).resolves.toBe(1);
|
||||
@@ -215,6 +219,113 @@ describe('CompetitionOutboxProcessor', () => {
|
||||
expect.stringContaining('moved to dead-letter after 10 attempts'),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps a competing event retryable while another instance is applying the same competition', async () => {
|
||||
const competitionId = new mongoose.Types.ObjectId();
|
||||
const matchId = new mongoose.Types.ObjectId();
|
||||
const event = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
competitionId,
|
||||
revision: 2,
|
||||
type: 'competition.projection-updated',
|
||||
state: 'pending',
|
||||
attemptCount: 1,
|
||||
};
|
||||
const updates = jest.fn(() => executable({ matchedCount: 1 }));
|
||||
const models = {
|
||||
competitionOutbox: {
|
||||
findOneAndUpdate: jest.fn(() => leanExecutable(event)),
|
||||
updateOne: updates,
|
||||
},
|
||||
competition: {
|
||||
findById: () =>
|
||||
leanExecutable({
|
||||
_id: competitionId,
|
||||
matchId,
|
||||
communityId: new mongoose.Types.ObjectId(),
|
||||
revision: 2,
|
||||
phase: 'running',
|
||||
teams: [],
|
||||
projection: { phase: 'running', games: [] },
|
||||
}),
|
||||
},
|
||||
match: {
|
||||
findById: () => leanExecutable({ status: 'in-progress' }),
|
||||
updateOne: () => executable({ matchedCount: 1 }),
|
||||
},
|
||||
};
|
||||
const connection = {
|
||||
getModel: async (name: keyof typeof models) => models[name],
|
||||
} as unknown as ConnectionService;
|
||||
let releaseBuild!: () => void;
|
||||
let startedBuild!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
startedBuild = resolve;
|
||||
});
|
||||
const blocked = new Promise<void>((resolve) => {
|
||||
releaseBuild = resolve;
|
||||
});
|
||||
const buildAndUpsert = jest.fn(async () => {
|
||||
startedBuild();
|
||||
await blocked;
|
||||
});
|
||||
const heldKeys = new Set<string>();
|
||||
const runWithRenewedLock = jest.fn(
|
||||
async (
|
||||
options: { lockKey: string },
|
||||
operation: () => Promise<unknown>,
|
||||
) => {
|
||||
if (heldKeys.has(options.lockKey)) return null;
|
||||
heldKeys.add(options.lockKey);
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
heldKeys.delete(options.lockKey);
|
||||
}
|
||||
},
|
||||
);
|
||||
const makeProcessor = () =>
|
||||
new CompetitionOutboxProcessor(
|
||||
connection,
|
||||
{
|
||||
provisionReadyAssignments: jest.fn().mockResolvedValue(undefined),
|
||||
} as never,
|
||||
{ buildAndUpsert } as never,
|
||||
{
|
||||
invalidateCommunity: jest.fn().mockResolvedValue(undefined),
|
||||
} as never,
|
||||
{ runWithRenewedLock } as unknown as DistributedLockService,
|
||||
);
|
||||
const first = makeProcessor().drainOnce(1);
|
||||
await started;
|
||||
try {
|
||||
await expect(makeProcessor().drainOnce(1)).resolves.toBe(1);
|
||||
expect(buildAndUpsert).toHaveBeenCalledTimes(1);
|
||||
expect(updates).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
$set: { state: 'pending', availableAt: expect.any(Date) },
|
||||
$inc: { attemptCount: -1 },
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
releaseBuild();
|
||||
await first;
|
||||
}
|
||||
expect(runWithRenewedLock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
lockKey: `competition:projection-effects:${competitionId.toString()}`,
|
||||
}),
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(heldKeys.size).toBe(0);
|
||||
expect(updates).toHaveBeenLastCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
$set: expect.objectContaining({ state: 'published' }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
function leanExecutable<T>(value: T) {
|
||||
@@ -229,3 +340,11 @@ function leanExecutable<T>(value: T) {
|
||||
function executable<T>(value: T) {
|
||||
return { exec: jest.fn().mockResolvedValue(value) };
|
||||
}
|
||||
|
||||
function availableLock(): DistributedLockService {
|
||||
return {
|
||||
runWithRenewedLock: jest.fn(
|
||||
(_options: unknown, operation: () => Promise<unknown>) => operation(),
|
||||
),
|
||||
} as unknown as DistributedLockService;
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import mongoose from 'mongoose';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { CompetitionAssignmentProvisioner } from './competition-assignment.provisioner';
|
||||
import type { CompetitionResultSnapshot } from './competition-result.adapter';
|
||||
|
||||
@@ -54,6 +55,7 @@ export class CompetitionOutboxProcessor
|
||||
private readonly assignmentProvisioner: CompetitionAssignmentProvisioner,
|
||||
private readonly matchStatsService: MatchStatsService,
|
||||
private readonly communityStatsService: CommunityStatsService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public onModuleInit(): void {
|
||||
@@ -125,7 +127,30 @@ export class CompetitionOutboxProcessor
|
||||
if (event.type !== 'competition.projection-updated') {
|
||||
throw new Error(`Unsupported Competition outbox type ${event.type}.`);
|
||||
}
|
||||
await this.applyProjectionEffects(event);
|
||||
// Different revision events can be leased by different API instances.
|
||||
// Serialize their effects so the next builder can reuse committed stats.
|
||||
const applied = await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: `competition:projection-effects:${event.competitionId.toString()}`,
|
||||
lockExpirationDurationMilliseconds: PROCESSING_LEASE_MS,
|
||||
lockRenewalIntervalMilliseconds: PROCESSING_LEASE_MS / 3,
|
||||
},
|
||||
async () => {
|
||||
await this.applyProjectionEffects(event);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
if (!applied) {
|
||||
// Contention is not a failed delivery and must not exhaust retries.
|
||||
await OutboxModel.updateOne(leaseFilter(event), {
|
||||
$set: {
|
||||
state: 'pending',
|
||||
availableAt: new Date(Date.now() + DRAIN_INTERVAL_MS),
|
||||
},
|
||||
$inc: { attemptCount: -1 },
|
||||
}).exec();
|
||||
return;
|
||||
}
|
||||
await OutboxModel.updateOne(leaseFilter(event), {
|
||||
$set: { state: 'published', publishedAt: new Date() },
|
||||
$unset: { lastError: 1 },
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import type { MatchDocument } from '../global/database/schema';
|
||||
import type { RiotMatchDto } from '../riot/riot.types';
|
||||
import { TournamentResultService } from './tournament-result.service';
|
||||
|
||||
describe('TournamentResultService Competition cutover', () => {
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('accepts a manually collected Riot game through Competition without mutating legacy result state', async () => {
|
||||
const buildStats = jest.spyOn(
|
||||
MatchStatsService.prototype,
|
||||
'buildAndUpsert',
|
||||
);
|
||||
const communityId = new mongoose.Types.ObjectId();
|
||||
const matchId = new mongoose.Types.ObjectId();
|
||||
const competitionId = new mongoose.Types.ObjectId();
|
||||
@@ -192,15 +199,11 @@ describe('TournamentResultService Competition cutover', () => {
|
||||
replayed: false,
|
||||
}),
|
||||
};
|
||||
const matchStatsService = {
|
||||
buildAndUpsert: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const service = new TournamentResultService(
|
||||
connectionService as never,
|
||||
{ getMatch: jest.fn().mockResolvedValue(detail) } as never,
|
||||
{} as never,
|
||||
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
matchStatsService as never,
|
||||
{
|
||||
runWithLock: jest.fn(),
|
||||
runWithRenewedLock: jest.fn(),
|
||||
@@ -255,9 +258,7 @@ describe('TournamentResultService Competition cutover', () => {
|
||||
expect(match.result).toBe(legacyResult);
|
||||
expect(match.doubleElimination).toBe(legacyBracket);
|
||||
expect(match.status).toBe('finished');
|
||||
expect(matchStatsService.buildAndUpsert).toHaveBeenCalledWith(matchId, {
|
||||
sourceRevision: 1,
|
||||
});
|
||||
expect(buildStats).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -208,7 +208,6 @@ function createHarness(
|
||||
{ getMatchIdsByPuuid: () => Promise.resolve([]) } as never,
|
||||
queue as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
locks as never,
|
||||
{ bootstrap: () => Promise.resolve(competition) } as never,
|
||||
{} as never,
|
||||
|
||||
@@ -313,7 +313,6 @@ function pollHarness(params: {
|
||||
riotMatchService as never,
|
||||
riotJobQueueService as never,
|
||||
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
distributedLockService as never,
|
||||
{ bootstrap: jest.fn().mockResolvedValue(readyCompetition) } as never,
|
||||
commandService as never,
|
||||
|
||||
@@ -7,7 +7,6 @@ import {
|
||||
import { createHash } from 'node:crypto';
|
||||
import mongoose from 'mongoose';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import {
|
||||
CompetitionBootstrapService,
|
||||
type CompetitionBootstrapMatch,
|
||||
@@ -95,7 +94,6 @@ export class TournamentResultService {
|
||||
private readonly riotMatchService: RiotMatchService,
|
||||
private readonly riotJobQueueService: RiotJobQueueService,
|
||||
private readonly communityStatsService: CommunityStatsService,
|
||||
private readonly matchStatsService: MatchStatsService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
private readonly competitionBootstrapService: CompetitionBootstrapService,
|
||||
private readonly competitionCommandService: CompetitionCommandService,
|
||||
@@ -242,10 +240,15 @@ export class TournamentResultService {
|
||||
|
||||
public async cancelPolling(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
session?: mongoose.ClientSession,
|
||||
): Promise<void> {
|
||||
const match = await this.loadMatch(matchOrId);
|
||||
const match =
|
||||
matchOrId instanceof mongoose.Types.ObjectId
|
||||
? await this.loadMatch(matchOrId, session)
|
||||
: matchOrId;
|
||||
match.resultPollState = undefined;
|
||||
await match.save();
|
||||
if (session) await match.save({ session });
|
||||
else await match.save();
|
||||
}
|
||||
|
||||
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
|
||||
@@ -516,9 +519,7 @@ export class TournamentResultService {
|
||||
params.match,
|
||||
competition,
|
||||
);
|
||||
if (!result.replayed) {
|
||||
await this.rebuildMatchStatsSafe(params.match, result.revision);
|
||||
}
|
||||
// acceptGame commits the stats rebuild to the durable Competition outbox.
|
||||
return {
|
||||
kind: 'accepted',
|
||||
displayNumber: mapped.displayNumber,
|
||||
@@ -708,21 +709,6 @@ export class TournamentResultService {
|
||||
}
|
||||
}
|
||||
|
||||
private async rebuildMatchStatsSafe(
|
||||
match: MatchDocument,
|
||||
sourceRevision: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.matchStatsService.buildAndUpsert(match._id, {
|
||||
sourceRevision,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to rebuild match stats for ${match._id.toString()} at Competition revision ${sourceRevision}: ${errorMessage(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureRiotMatchPersisted(
|
||||
riotMatchId: string,
|
||||
region: RiotRegionalRoute,
|
||||
@@ -927,12 +913,14 @@ export class TournamentResultService {
|
||||
|
||||
private async loadMatch(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
session?: mongoose.ClientSession,
|
||||
): Promise<MatchDocument> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const match =
|
||||
const query =
|
||||
matchOrId instanceof mongoose.Types.ObjectId
|
||||
? await MatchModel.findById(matchOrId)
|
||||
: await MatchModel.findById(matchOrId._id);
|
||||
? MatchModel.findById(matchOrId)
|
||||
: MatchModel.findById(matchOrId._id);
|
||||
const match = await (session ? query.session(session) : query);
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
return match;
|
||||
}
|
||||
|
||||
@@ -446,6 +446,13 @@ async function main(): Promise<void> {
|
||||
const bidResult = await auctionService.placeBid({
|
||||
amount: 10,
|
||||
bidRequestId: randomUUID(),
|
||||
clientObservedBidRoundId: String(
|
||||
(
|
||||
await auctionRedisStateReaderService.readAuctionStateView(
|
||||
resources.matchId.toString(),
|
||||
)
|
||||
)?.currentBidRoundId,
|
||||
),
|
||||
matchId: resources.matchId,
|
||||
teamIndex: 1,
|
||||
});
|
||||
@@ -537,6 +544,13 @@ async function main(): Promise<void> {
|
||||
|
||||
await auctionService.reset({
|
||||
auctionActionRequestId: randomUUID(),
|
||||
clientObservedAuctionLifecycleId: String(
|
||||
(
|
||||
await auctionRedisStateReaderService.readAuctionStateView(
|
||||
resources.matchId.toString(),
|
||||
)
|
||||
)?.auctionLifecycleId,
|
||||
),
|
||||
communityId: resources.communityId,
|
||||
matchId: resources.matchId,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
import { createClient } from 'redis';
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
|
||||
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
|
||||
import {
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY as queue,
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY as payloads,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY as deadQueue,
|
||||
AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY as deadPayloads,
|
||||
} from '../src/modules/auction/redis/auction-redis.keys';
|
||||
|
||||
const client = createClient({
|
||||
url: process.env.REDIS_TEST_URL,
|
||||
socket: { reconnectStrategy: false },
|
||||
});
|
||||
const redis = new AuctionRedisService(client);
|
||||
const matchId = new mongoose.Types.ObjectId().toString();
|
||||
function event(revision: number) {
|
||||
return {
|
||||
_id: `${matchId}:life:${String(revision).padStart(16, '0')}`,
|
||||
matchId,
|
||||
auctionLifecycleId: 'life',
|
||||
revision,
|
||||
bidRequestId: `bid-${revision}`,
|
||||
bidderUserId: new mongoose.Types.ObjectId().toString(),
|
||||
bidderTeamIndex: 1,
|
||||
bidPointAmount: revision,
|
||||
bidPlacedAtTimestampMS: Date.now(),
|
||||
};
|
||||
}
|
||||
function fixture() {
|
||||
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
|
||||
matchedCount: 0,
|
||||
upsertedCount: operations.length,
|
||||
}));
|
||||
const service = new AuctionBidArchiveService(
|
||||
{ getModel: async () => ({ bulkWrite }) } as never,
|
||||
redis,
|
||||
);
|
||||
return { service, bulkWrite };
|
||||
}
|
||||
async function enqueue(id: string, raw: string | null) {
|
||||
await client.zAdd(queue, { score: 1, value: id });
|
||||
if (raw !== null) await client.hSet(payloads, id, raw);
|
||||
}
|
||||
beforeAll(async () => {
|
||||
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL)
|
||||
throw new Error('Use the isolated Redis test runner.');
|
||||
client.on('error', () => {});
|
||||
await client.connect();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await client.flushDb();
|
||||
});
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
afterAll(async () => {
|
||||
if (client.isOpen) await client.quit();
|
||||
});
|
||||
|
||||
it('isolates malformed, missing and invalid payloads while saving valid bids and retaining the originals', async () => {
|
||||
const { service, bulkWrite } = fixture();
|
||||
const good = event(10);
|
||||
const invalid = [
|
||||
['broken', '{broken'],
|
||||
['empty', ''],
|
||||
['null', 'null'],
|
||||
['array', '[]'],
|
||||
['missing', null],
|
||||
[event(1)._id, JSON.stringify({ ...event(1), revision: -1 })],
|
||||
[event(2)._id, JSON.stringify({ ...event(2), bidderUserId: 'bad' })],
|
||||
[event(3)._id, JSON.stringify({ ...event(3), bidderUserId: undefined })],
|
||||
[event(4)._id, JSON.stringify({ ...event(4), bidPointAmount: '4' })],
|
||||
] as Array<[string, string | null]>;
|
||||
for (const [id, raw] of invalid) await enqueue(id, raw);
|
||||
await enqueue(good._id, JSON.stringify(good));
|
||||
await expect(service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(bulkWrite.mock.calls[0][0]).toHaveLength(1);
|
||||
expect(await client.zCard(queue)).toBe(0);
|
||||
expect(await client.hLen(payloads)).toBe(0);
|
||||
expect(await client.zCard(deadQueue)).toBe(invalid.length);
|
||||
for (const [id, raw] of invalid) {
|
||||
expect(JSON.parse((await client.hGet(deadPayloads, id))!)).toMatchObject({
|
||||
id,
|
||||
rawPayload: raw,
|
||||
error: expect.any(String),
|
||||
quarantinedAtTimestampMS: expect.any(Number),
|
||||
});
|
||||
}
|
||||
expect(await client.ttl(deadPayloads)).toBe(-1);
|
||||
await expect(service.persistPendingBids()).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it('retains valid bids during a Mongo outage, after moving malformed entries out of the pending queue', async () => {
|
||||
const { service, bulkWrite } = fixture();
|
||||
const good = event(1);
|
||||
await enqueue('broken', '{broken');
|
||||
await enqueue(good._id, JSON.stringify(good));
|
||||
bulkWrite.mockRejectedValueOnce(new Error('Mongo offline'));
|
||||
await expect(service.persistPendingBids()).rejects.toThrow('Mongo offline');
|
||||
expect(await client.zRange(queue, 0, -1)).toEqual([good._id]);
|
||||
expect(await client.hGet(payloads, good._id)).toBe(JSON.stringify(good));
|
||||
expect(await client.zCard(deadQueue)).toBe(1);
|
||||
await expect(service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(await client.zCard(queue)).toBe(0);
|
||||
});
|
||||
|
||||
it('does not quarantine a payload repaired concurrently after HMGET', async () => {
|
||||
const { service } = fixture();
|
||||
const good = event(1);
|
||||
await enqueue(good._id, '{broken');
|
||||
const send = client.sendCommand.bind(client);
|
||||
jest.spyOn(client, 'sendCommand').mockImplementation(async (command) => {
|
||||
const result = await send(command);
|
||||
if (command[0] === 'HMGET')
|
||||
await client.hSet(payloads, good._id, JSON.stringify(good));
|
||||
return result;
|
||||
});
|
||||
await expect(service.persistPendingBids()).resolves.toBe(0);
|
||||
expect(await client.zCard(deadQueue)).toBe(0);
|
||||
expect(await client.zRange(queue, 0, -1)).toEqual([good._id]);
|
||||
await expect(service.persistPendingBids()).resolves.toBe(1);
|
||||
});
|
||||
|
||||
it('preserves pending payloads if quarantine cannot be written', async () => {
|
||||
const { service, bulkWrite } = fixture();
|
||||
const good = event(1);
|
||||
await enqueue('broken', '{broken');
|
||||
await enqueue(good._id, JSON.stringify(good));
|
||||
await client.set(deadPayloads, 'wrong-type');
|
||||
await expect(service.persistPendingBids()).rejects.toThrow('WRONGTYPE');
|
||||
expect(await client.zCard(queue)).toBe(2);
|
||||
expect(await client.hGet(payloads, 'broken')).toBe('{broken');
|
||||
expect(bulkWrite).not.toHaveBeenCalled();
|
||||
await client.del(deadPayloads);
|
||||
await expect(service.persistPendingBids()).resolves.toBe(1);
|
||||
});
|
||||
|
||||
it('does not acknowledge a payload changed while Mongo was saving', async () => {
|
||||
const { service, bulkWrite } = fixture();
|
||||
const good = event(1);
|
||||
await enqueue(good._id, JSON.stringify(good));
|
||||
bulkWrite.mockImplementationOnce(async () => {
|
||||
await client.hSet(payloads, good._id, '{replaced');
|
||||
return { matchedCount: 0, upsertedCount: 1 };
|
||||
});
|
||||
await expect(service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(await client.hGet(payloads, good._id)).toBe('{replaced');
|
||||
expect(await client.zCard(queue)).toBe(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['bidRoundId', null],
|
||||
['bidRoundId', 1],
|
||||
['bidRoundId', {}],
|
||||
['bidRoundId', []],
|
||||
['bidRoundId', ' '],
|
||||
['playerUserId', ''],
|
||||
])('continues quarantining invalid %s=%j', async (field, value) => {
|
||||
const { service, bulkWrite } = fixture();
|
||||
const bad = { ...event(1), [field]: value };
|
||||
await enqueue(bad._id, JSON.stringify(bad));
|
||||
await expect(service.persistPendingBids()).resolves.toBe(0);
|
||||
expect(bulkWrite).not.toHaveBeenCalled();
|
||||
expect(await client.zCard(deadQueue)).toBe(1);
|
||||
});
|
||||
|
||||
it('archives a pre-upgrade bid with an empty round ID instead of quarantining it', async () => {
|
||||
const { service, bulkWrite } = fixture();
|
||||
const legacy = { ...event(1), bidRoundId: '' };
|
||||
await enqueue(legacy._id, JSON.stringify(legacy));
|
||||
await expect(service.persistPendingBids()).resolves.toBe(1);
|
||||
expect(await client.zCard(deadQueue)).toBe(0);
|
||||
expect(await client.zCard(queue)).toBe(0);
|
||||
const [[operations]] = bulkWrite.mock.calls as unknown as [
|
||||
[Array<{ updateOne: { update: { $setOnInsert: object } } }>],
|
||||
];
|
||||
expect(operations[0].updateOne.update.$setOnInsert).not.toHaveProperty(
|
||||
'bidRoundId',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,769 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionService } from '../src/modules/auction/auction.service';
|
||||
import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler';
|
||||
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
|
||||
import { migrateAuctionCancellationIndex } from '../src/modules/global/database/migrations/20261109-auction-cancellation-index';
|
||||
import { AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY } from '../src/modules/auction/redis/auction-redis.keys';
|
||||
import { TournamentCodeService } from '../src/modules/tournament-code/tournament-code.service';
|
||||
import { TournamentResultService } from '../src/modules/tournament-result/tournament-result.service';
|
||||
import {
|
||||
MatchSchema,
|
||||
AuctionSchema,
|
||||
AuctionHistoryEntrySchema,
|
||||
AuctionSideEffectApplicationSchema,
|
||||
TournamentCodeSchema,
|
||||
AuctionBidRecordSchema,
|
||||
} from '../src/modules/global/database/schema';
|
||||
|
||||
let connection: mongoose.Connection;
|
||||
let models: Record<string, mongoose.Model<any>>;
|
||||
const redis = {
|
||||
acquireTokenLock: async () => true,
|
||||
releaseTokenLock: async () => true,
|
||||
getCurrentRedisTimestampMS: async () => Date.now(),
|
||||
raw: () => ({
|
||||
sendCommand: async (command: string[]) =>
|
||||
command[0] === 'SCAN' ? ['0', []] : 1,
|
||||
}),
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
if (
|
||||
process.env.MONGO_TEST_ISOLATED !== '1' ||
|
||||
!process.env.MONGO_TEST_URL?.startsWith('mongodb://127.0.0.1:')
|
||||
)
|
||||
throw new Error('Use test:mongodb with an isolated local replica set.');
|
||||
connection = await mongoose
|
||||
.createConnection(process.env.MONGO_TEST_URL, {
|
||||
autoIndex: false,
|
||||
autoCreate: false,
|
||||
})
|
||||
.asPromise();
|
||||
models = {
|
||||
match: connection.model('match', MatchSchema),
|
||||
auction: connection.model('auction', AuctionSchema),
|
||||
auctionHistory: connection.model(
|
||||
'auctionHistory',
|
||||
AuctionHistoryEntrySchema,
|
||||
),
|
||||
auctionSideEffectApplication: connection.model(
|
||||
'auctionSideEffectApplication',
|
||||
AuctionSideEffectApplicationSchema,
|
||||
),
|
||||
tournamentCode: connection.model('tournamentCode', TournamentCodeSchema),
|
||||
auctionBidRecord: connection.model(
|
||||
'auctionBidRecord',
|
||||
AuctionBidRecordSchema,
|
||||
),
|
||||
};
|
||||
await Promise.all(
|
||||
Object.values(models).map((model) => model.createCollection()),
|
||||
);
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await Promise.all(Object.values(models).map((model) => model.deleteMany({})));
|
||||
});
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
afterAll(async () => {
|
||||
await connection?.close();
|
||||
});
|
||||
|
||||
async function fixture(reserved: boolean) {
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId(),
|
||||
auctionId = new mongoose.Types.ObjectId(),
|
||||
codeId = new mongoose.Types.ObjectId();
|
||||
await models.match.create({
|
||||
_id: matchId,
|
||||
communityId,
|
||||
title: 'Isolated auction test',
|
||||
scheduledAt: new Date(),
|
||||
status: 'team-built',
|
||||
teamBuiltAt: new Date(),
|
||||
teams: [],
|
||||
participants: [],
|
||||
tournamentCodeIds: reserved ? [codeId] : [],
|
||||
gameSchedule: [],
|
||||
__v: 0,
|
||||
});
|
||||
await models.auction.collection.insertOne({
|
||||
_id: auctionId,
|
||||
matchId,
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: 'life',
|
||||
phase: 'complete',
|
||||
revision: 10,
|
||||
matchSideEffectRevision: 0,
|
||||
matchSideEffectStatus: 'none',
|
||||
});
|
||||
if (reserved)
|
||||
await models.tournamentCode.collection.insertOne({
|
||||
_id: codeId,
|
||||
code: 'ISOLATED-TEST',
|
||||
state: 'reserved',
|
||||
matchId,
|
||||
});
|
||||
const database = {
|
||||
getModel: async (name: string) => models[name],
|
||||
createSession: () => connection.startSession(),
|
||||
};
|
||||
const codes = new TournamentCodeService(database as never, redis as never);
|
||||
const results = new TournamentResultService(
|
||||
database as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const service = new AuctionService(
|
||||
database as never,
|
||||
{} as never,
|
||||
codes,
|
||||
results,
|
||||
redis as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
return {
|
||||
matchId,
|
||||
communityId,
|
||||
auctionId,
|
||||
codeId,
|
||||
service,
|
||||
database,
|
||||
codes,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
it.each([false, true])(
|
||||
'cancels the auction atomically with reserved codes=%s',
|
||||
async (reserved) => {
|
||||
const f = await fixture(reserved);
|
||||
await f.service.reset({
|
||||
matchId: f.matchId,
|
||||
communityId: f.communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
});
|
||||
const match = await models.match.findById(f.matchId).lean();
|
||||
expect(match).toMatchObject({
|
||||
status: 'recruiting',
|
||||
teams: [],
|
||||
tournamentCodeIds: [],
|
||||
gameSchedule: [],
|
||||
});
|
||||
expect(match?.teamBuiltAt).toBeUndefined();
|
||||
expect(await models.auction.findById(f.auctionId)).toBeNull();
|
||||
if (reserved)
|
||||
expect(
|
||||
(await models.tournamentCode.findById(f.codeId).lean())?.state,
|
||||
).toBe('released');
|
||||
},
|
||||
);
|
||||
|
||||
it('rolls back code release and Match changes if cancellation later fails', async () => {
|
||||
const f = await fixture(true);
|
||||
jest.spyOn(models.auctionHistory, 'deleteMany').mockImplementation(() => {
|
||||
throw new Error('forced history failure');
|
||||
});
|
||||
await expect(
|
||||
f.service.reset({
|
||||
matchId: f.matchId,
|
||||
communityId: f.communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
}),
|
||||
).rejects.toThrow('forced history failure');
|
||||
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
|
||||
'reserved',
|
||||
);
|
||||
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
|
||||
status: 'team-built',
|
||||
__v: 0,
|
||||
tournamentCodeIds: [f.codeId],
|
||||
});
|
||||
expect(await models.auction.findById(f.auctionId)).not.toBeNull();
|
||||
});
|
||||
|
||||
it('retries cancellation cleanup from Mongo after Redis fails, without another cancellation request', async () => {
|
||||
const f = await fixture(true);
|
||||
const cleanup = jest
|
||||
.spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle')
|
||||
.mockRejectedValueOnce(new Error('Redis offline'))
|
||||
.mockResolvedValue(undefined);
|
||||
await f.service.reset({
|
||||
matchId: f.matchId,
|
||||
communityId: f.communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
});
|
||||
expect(cleanup).not.toHaveBeenCalled();
|
||||
await f.service.retryPendingCancellations();
|
||||
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupAttempts: 1,
|
||||
});
|
||||
await models.match.updateOne(
|
||||
{ _id: f.matchId },
|
||||
{ $set: { auctionCancellationCleanupRetryAt: new Date(0) } },
|
||||
);
|
||||
await f.service.retryPendingCancellations();
|
||||
expect(cleanup).toHaveBeenCalledTimes(2);
|
||||
expect(
|
||||
(await models.match.findById(f.matchId).lean())
|
||||
?.auctionCancellationCleanupPending,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps all 1005 Mongo bid records and idempotently retries when Redis acknowledgement fails', async () => {
|
||||
const f = await fixture(false);
|
||||
const events = Array.from({ length: 1005 }, (_, i) => ({
|
||||
_id: `${f.matchId.toString()}:life:${String(i + 1).padStart(16, '0')}`,
|
||||
matchId: f.matchId.toString(),
|
||||
auctionLifecycleId: 'life',
|
||||
revision: i + 1,
|
||||
bidRequestId: `bid:${i}`,
|
||||
bidRoundId: 'round',
|
||||
bidderTeamIndex: 1,
|
||||
bidPointAmount: i + 1,
|
||||
bidderUserId: new mongoose.Types.ObjectId().toString(),
|
||||
playerUserId: new mongoose.Types.ObjectId().toString(),
|
||||
bidPlacedAtTimestampMS: Date.now(),
|
||||
}));
|
||||
let failAck = true;
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGE') return events.map((e) => e._id);
|
||||
if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e));
|
||||
if (failAck) throw new Error('Redis acknowledgement failed');
|
||||
return 1;
|
||||
});
|
||||
const archive = new AuctionBidArchiveService(
|
||||
f.database as never,
|
||||
{ raw: () => ({ sendCommand }) } as never,
|
||||
);
|
||||
await expect(archive.persistPendingBids(2000)).rejects.toThrow(
|
||||
'acknowledgement failed',
|
||||
);
|
||||
expect(await models.auctionBidRecord.countDocuments()).toBe(1005);
|
||||
failAck = false;
|
||||
await expect(archive.persistPendingBids(2000)).resolves.toBe(1005);
|
||||
expect(await models.auctionBidRecord.countDocuments()).toBe(1005);
|
||||
const page = await archive.list(f.matchId, 1000, 100);
|
||||
expect(page.items.map((item) => item.revision)).toEqual([
|
||||
1001, 1002, 1003, 1004, 1005,
|
||||
]);
|
||||
expect(page.hasMore).toBe(false);
|
||||
});
|
||||
|
||||
it('migrates the cleanup index idempotently and examines only due pending matches', async () => {
|
||||
const f = await fixture(false);
|
||||
const documents = Array.from({ length: 10000 }, (_, i) => ({
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
lastCancelledAuctionLifecycleId: `life-${i}`,
|
||||
...(i < 5
|
||||
? {
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: new Date(0),
|
||||
}
|
||||
: i === 5
|
||||
? {
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: new Date(Date.now() + 3600000),
|
||||
}
|
||||
: {}),
|
||||
}));
|
||||
await models.match.collection.insertMany(documents);
|
||||
const find = jest.spyOn(models.match, 'find');
|
||||
const cleanup = jest
|
||||
.spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle')
|
||||
.mockResolvedValue(undefined);
|
||||
await f.service.retryPendingCancellations();
|
||||
expect(cleanup).toHaveBeenCalledTimes(5);
|
||||
const query = find.mock.calls[0][0] as unknown as Record<string, unknown>;
|
||||
// Restore only the five due jobs so explain measures the actual worker query.
|
||||
await models.match.updateMany(
|
||||
{ _id: { $in: documents.slice(0, 5).map((d) => d._id) } },
|
||||
{
|
||||
$set: {
|
||||
auctionCancellationCleanupPending: true,
|
||||
auctionCancellationCleanupRetryAt: new Date(0),
|
||||
},
|
||||
},
|
||||
);
|
||||
const explain = () =>
|
||||
models.match.collection
|
||||
.find(query)
|
||||
.sort({ auctionCancellationCleanupRetryAt: 1 })
|
||||
.limit(20)
|
||||
.explain('executionStats');
|
||||
expect(
|
||||
(await explain()).executionStats.totalDocsExamined,
|
||||
).toBeGreaterThanOrEqual(10000);
|
||||
expect(await migrateAuctionCancellationIndex(connection.db!)).toMatchObject({
|
||||
dryRun: true,
|
||||
alreadyPresent: false,
|
||||
});
|
||||
expect(
|
||||
(await models.match.collection.listIndexes().toArray()).some(
|
||||
(i) => i.name === 'auction_cancellation_cleanup_due',
|
||||
),
|
||||
).toBe(false);
|
||||
await migrateAuctionCancellationIndex(connection.db!, { dryRun: false });
|
||||
expect(
|
||||
await migrateAuctionCancellationIndex(connection.db!, { dryRun: false }),
|
||||
).toMatchObject({ alreadyPresent: true });
|
||||
const plan = await explain();
|
||||
expect(plan.executionStats.nReturned).toBe(5);
|
||||
expect(plan.executionStats.totalDocsExamined).toBe(5);
|
||||
expect(JSON.stringify(plan.queryPlanner.winningPlan)).toContain('IXSCAN');
|
||||
});
|
||||
|
||||
function archiveEvent(matchId: mongoose.Types.ObjectId, revision: number) {
|
||||
return {
|
||||
_id: `${matchId.toString()}:life:${String(revision).padStart(16, '0')}`,
|
||||
matchId: matchId.toString(),
|
||||
auctionLifecycleId: 'life',
|
||||
revision,
|
||||
bidRequestId: `bid-${revision}`,
|
||||
bidRoundId: 'round',
|
||||
bidderUserId: new mongoose.Types.ObjectId().toString(),
|
||||
playerUserId: new mongoose.Types.ObjectId().toString(),
|
||||
bidderTeamIndex: 1,
|
||||
bidPointAmount: revision,
|
||||
bidPlacedAtTimestampMS: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
it.each(['', undefined, 'round'])(
|
||||
'persists and lists bids with optional round ID %s without duplicating retries',
|
||||
async (bidRoundId) => {
|
||||
const f = await fixture(false);
|
||||
const event = { ...archiveEvent(f.matchId, 1), bidRoundId };
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGE') return [event._id];
|
||||
if (command[0] === 'HMGET') return [JSON.stringify(event)];
|
||||
return 1;
|
||||
});
|
||||
const archive = new AuctionBidArchiveService(
|
||||
f.database as never,
|
||||
{ raw: () => ({ sendCommand }) } as never,
|
||||
);
|
||||
await expect(archive.persistPendingBids()).resolves.toBe(1);
|
||||
await expect(archive.persistPendingBids()).resolves.toBe(1);
|
||||
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
|
||||
const page = await archive.list(f.matchId);
|
||||
expect(page.items).toHaveLength(1);
|
||||
expect(String(page.items[0].playerUserId)).toBe(event.playerUserId);
|
||||
if (bidRoundId) expect(page.items[0].bidRoundId).toBe(bidRoundId);
|
||||
else expect(page.items[0]).not.toHaveProperty('bidRoundId');
|
||||
expect(
|
||||
sendCommand.mock.calls
|
||||
.filter(([command]) => command[0] === 'EVAL')
|
||||
.every(([command]) => command[2] === '2'),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
'json',
|
||||
'revision',
|
||||
'object-id',
|
||||
'required-field',
|
||||
'identity',
|
||||
'missing-payload',
|
||||
])(
|
||||
'quarantines an invalid %s without losing the valid Mongo bid',
|
||||
async (kind) => {
|
||||
const f = await fixture(false);
|
||||
const bad = archiveEvent(f.matchId, 1),
|
||||
good = archiveEvent(f.matchId, 2);
|
||||
const invalid: Record<string, unknown> = { ...bad };
|
||||
if (kind === 'revision') invalid.revision = -1;
|
||||
if (kind === 'object-id') invalid.bidderUserId = 'not-an-object-id';
|
||||
if (kind === 'required-field') delete invalid.bidderUserId;
|
||||
if (kind === 'identity')
|
||||
invalid.matchId = new mongoose.Types.ObjectId().toString();
|
||||
const raw =
|
||||
kind === 'json'
|
||||
? '{broken'
|
||||
: kind === 'missing-payload'
|
||||
? null
|
||||
: JSON.stringify(invalid);
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGE') return [bad._id, good._id];
|
||||
if (command[0] === 'HMGET') return [raw, JSON.stringify(good)];
|
||||
return 1;
|
||||
});
|
||||
const archive = new AuctionBidArchiveService(
|
||||
f.database as never,
|
||||
{ raw: () => ({ sendCommand }) } as never,
|
||||
);
|
||||
await expect(archive.persistPendingBids()).resolves.toBe(1);
|
||||
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
|
||||
expect(
|
||||
await models.auctionBidRecord.findById(good._id).lean(),
|
||||
).toMatchObject({ revision: 2 });
|
||||
expect(await models.auctionBidRecord.findById(bad._id)).toBeNull();
|
||||
const quarantine = sendCommand.mock.calls
|
||||
.map((c) => c[0])
|
||||
.find((c) => c[0] === 'EVAL' && c[2] === '4')!;
|
||||
expect(JSON.parse(quarantine[11])).toMatchObject({
|
||||
id: bad._id,
|
||||
rawPayload: raw,
|
||||
error: expect.any(String),
|
||||
});
|
||||
const ack = sendCommand.mock.calls
|
||||
.map((c) => c[0])
|
||||
.find((c) => c[0] === 'EVAL' && c[2] === '2')!;
|
||||
expect(ack.slice(5)).toEqual([good._id, JSON.stringify(good)]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['partial-write-error', 'partial-acknowledgement'])(
|
||||
'retains the batch after %s and retries Mongo upserts without duplicates',
|
||||
async (failure) => {
|
||||
const f = await fixture(false);
|
||||
const events = [archiveEvent(f.matchId, 1), archiveEvent(f.matchId, 2)];
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGE') return events.map((e) => e._id);
|
||||
if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e));
|
||||
return 1;
|
||||
});
|
||||
const archive = new AuctionBidArchiveService(
|
||||
f.database as never,
|
||||
{ raw: () => ({ sendCommand }) } as never,
|
||||
);
|
||||
const write = models.auctionBidRecord.bulkWrite.bind(
|
||||
models.auctionBidRecord,
|
||||
);
|
||||
jest
|
||||
.spyOn(models.auctionBidRecord, 'bulkWrite')
|
||||
.mockImplementationOnce(async (operations, options) => {
|
||||
const result = await write(operations.slice(0, 1), options);
|
||||
if (failure === 'partial-write-error')
|
||||
throw new Error('Lost Mongo acknowledgement');
|
||||
return result;
|
||||
});
|
||||
await expect(archive.persistPendingBids()).rejects.toThrow();
|
||||
expect(await models.auctionBidRecord.countDocuments()).toBe(1);
|
||||
expect(
|
||||
sendCommand.mock.calls.filter(([c]) => c[0] === 'EVAL'),
|
||||
).toHaveLength(0);
|
||||
await expect(archive.persistPendingBids()).resolves.toBe(2);
|
||||
expect(await models.auctionBidRecord.countDocuments()).toBe(2);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['Mongo', 'Redis mirroring'])(
|
||||
'reconciles a terminal status after %s recovers',
|
||||
async (outage) => {
|
||||
const f = await fixture(false);
|
||||
const payload = {
|
||||
auctionId: f.auctionId.toString(),
|
||||
matchId: f.matchId.toString(),
|
||||
auctionLifecycleId: 'life',
|
||||
auctionStateTransitionId: 'complete-10',
|
||||
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
|
||||
afterAuctionMongoProjection: { matchSideEffectRevision: 1 },
|
||||
};
|
||||
await models.auction.updateOne(
|
||||
{ _id: f.auctionId },
|
||||
{
|
||||
$set: {
|
||||
matchSideEffectTransitionId: 'complete-10',
|
||||
matchSideEffectRevision: 1,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
},
|
||||
);
|
||||
let unavailable = outage === 'Mongo';
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGEBYSCORE') return ['dead-letter'];
|
||||
if (command[0] === 'HGETALL')
|
||||
return { payloadJson: JSON.stringify(payload) };
|
||||
if (command[0] === 'HGET') return 'retry limit exceeded';
|
||||
return 1;
|
||||
});
|
||||
const mirror = jest.fn().mockResolvedValue(true),
|
||||
broadcast = jest.fn();
|
||||
if (outage === 'Redis mirroring')
|
||||
mirror.mockRejectedValueOnce(new Error('Redis unavailable'));
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
{
|
||||
getModel: async (name: string) => {
|
||||
if (unavailable) throw new Error('Mongo offline');
|
||||
return models[name];
|
||||
},
|
||||
} as never,
|
||||
{
|
||||
raw: () => ({ sendCommand }),
|
||||
getCurrentRedisTimestampMS: async () => 1000,
|
||||
} as never,
|
||||
undefined,
|
||||
undefined,
|
||||
{ mirrorMatchSideEffectStatus: mirror } as never,
|
||||
{ broadcastMatchSideEffect: broadcast } as never,
|
||||
);
|
||||
await scheduler.reconcileDeadLetterStatuses();
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'ZADD',
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
'6000',
|
||||
'dead-letter',
|
||||
]);
|
||||
unavailable = false;
|
||||
await scheduler.reconcileDeadLetterStatuses();
|
||||
expect(
|
||||
(await models.auction.findById(f.auctionId).lean())
|
||||
?.matchSideEffectStatus,
|
||||
).toBe('failed');
|
||||
expect(mirror).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
auctionStateTransitionId: 'complete-10',
|
||||
matchSideEffectStatus: 'failed',
|
||||
}),
|
||||
);
|
||||
expect(broadcast).toHaveBeenCalledTimes(1);
|
||||
// Replaying the old notification must not overwrite a new generation.
|
||||
await models.auction.updateOne(
|
||||
{ _id: f.auctionId },
|
||||
{
|
||||
$set: {
|
||||
matchSideEffectTransitionId: 'new',
|
||||
matchSideEffectStatus: 'processing',
|
||||
matchSideEffectRevision: 20,
|
||||
},
|
||||
},
|
||||
);
|
||||
await scheduler.reconcileDeadLetterStatuses();
|
||||
expect(
|
||||
(await models.auction.findById(f.auctionId).lean())
|
||||
?.matchSideEffectStatus,
|
||||
).toBe('processing');
|
||||
expect(broadcast).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
it('cleans up reserved codes and Match metadata after undoing completion', async () => {
|
||||
const f = await fixture(true);
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
f.database as never,
|
||||
redis as never,
|
||||
f.codes,
|
||||
f.results,
|
||||
undefined,
|
||||
undefined,
|
||||
{
|
||||
readAuctionStateView: async () => ({
|
||||
auctionLifecycleId: 'life',
|
||||
phase: 'bidding',
|
||||
matchSideEffectTransitionId: 'undo',
|
||||
}),
|
||||
} as never,
|
||||
);
|
||||
await scheduler.applyDurabilityPayload({
|
||||
auctionId: f.auctionId,
|
||||
matchId: f.matchId,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 11,
|
||||
auctionHistorySequence: 11,
|
||||
auctionActionType: 'undo',
|
||||
auctionStateTransitionId: 'undo',
|
||||
snapshotBefore: {},
|
||||
historyMutation: { type: 'append' },
|
||||
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
|
||||
afterAuctionMongoProjection: {
|
||||
phase: 'bidding',
|
||||
revision: 11,
|
||||
matchSideEffectTransitionId: 'undo',
|
||||
matchSideEffectRevision: 1,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
});
|
||||
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
|
||||
status: 'team-built',
|
||||
tournamentCodeIds: [],
|
||||
gameSchedule: [],
|
||||
teams: [],
|
||||
});
|
||||
expect(
|
||||
(await models.match.findById(f.matchId).lean())?.teamBuiltAt,
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
|
||||
).toBe('completed');
|
||||
});
|
||||
|
||||
it('reads an uncommitted Match in the provided session when cancelling polling by id', async () => {
|
||||
const f = await fixture(false);
|
||||
const session = await connection.startSession();
|
||||
try {
|
||||
session.startTransaction();
|
||||
const [created] = await models.match.create(
|
||||
[
|
||||
{
|
||||
communityId: f.communityId,
|
||||
title: 'Uncommitted',
|
||||
scheduledAt: new Date(),
|
||||
},
|
||||
],
|
||||
{ session },
|
||||
);
|
||||
expect(await models.match.findById(created._id)).toBeNull();
|
||||
await f.results.cancelPolling(
|
||||
created._id as mongoose.Types.ObjectId,
|
||||
session,
|
||||
);
|
||||
expect(
|
||||
await models.match.findById(created._id).session(session),
|
||||
).not.toBeNull();
|
||||
await session.abortTransaction();
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
});
|
||||
|
||||
it('rolls back failed Undo cleanup and successfully retries with fresh Match versions', async () => {
|
||||
const f = await fixture(true);
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
f.database as never,
|
||||
redis as never,
|
||||
f.codes,
|
||||
f.results,
|
||||
undefined,
|
||||
undefined,
|
||||
{
|
||||
readAuctionStateView: async () => ({
|
||||
auctionLifecycleId: 'life',
|
||||
phase: 'bidding',
|
||||
matchSideEffectTransitionId: 'undo-retry',
|
||||
}),
|
||||
} as never,
|
||||
);
|
||||
const payload = {
|
||||
auctionId: f.auctionId,
|
||||
matchId: f.matchId,
|
||||
auctionLifecycleId: 'life',
|
||||
revision: 11,
|
||||
auctionHistorySequence: 11,
|
||||
auctionActionType: 'undo',
|
||||
auctionStateTransitionId: 'undo-retry',
|
||||
snapshotBefore: {},
|
||||
historyMutation: { type: 'append' },
|
||||
matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' },
|
||||
afterAuctionMongoProjection: {
|
||||
phase: 'bidding',
|
||||
revision: 11,
|
||||
matchSideEffectTransitionId: 'undo-retry',
|
||||
matchSideEffectRevision: 1,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
};
|
||||
jest
|
||||
.spyOn(f.results, 'cancelPolling')
|
||||
.mockRejectedValueOnce(new Error('temporary poll cleanup failure'));
|
||||
await expect(scheduler.applyDurabilityPayload(payload)).rejects.toThrow(
|
||||
'temporary poll cleanup failure',
|
||||
);
|
||||
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
|
||||
'reserved',
|
||||
);
|
||||
expect(await models.match.findById(f.matchId).lean()).toMatchObject({
|
||||
__v: 0,
|
||||
tournamentCodeIds: [f.codeId],
|
||||
});
|
||||
await scheduler.applyDurabilityPayload(payload);
|
||||
expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe(
|
||||
'released',
|
||||
);
|
||||
expect(
|
||||
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
|
||||
).toBe('completed');
|
||||
});
|
||||
|
||||
it('answers a retransmitted cancellation of the same lifecycle instead of 404', async () => {
|
||||
const f = await fixture(true);
|
||||
const request = {
|
||||
matchId: f.matchId,
|
||||
communityId: f.communityId,
|
||||
clientObservedAuctionLifecycleId: 'life',
|
||||
};
|
||||
await f.service.reset(request);
|
||||
await expect(f.service.reset(request)).resolves.toBeUndefined();
|
||||
expect(
|
||||
(await models.match.findById(f.matchId).lean())
|
||||
?.lastCancelledAuctionLifecycleId,
|
||||
).toBe('life');
|
||||
await expect(
|
||||
f.service.reset({ ...request, clientObservedAuctionLifecycleId: 'other' }),
|
||||
).rejects.toThrow('Auction not found.');
|
||||
});
|
||||
|
||||
it.each([
|
||||
'new-lifecycle',
|
||||
'new-transition',
|
||||
'completed',
|
||||
'revision-ahead',
|
||||
'mirror-superseded',
|
||||
])('fences terminal notifications against %s', async (scenario) => {
|
||||
const f = await fixture(false);
|
||||
await models.auction.updateOne(
|
||||
{ _id: f.auctionId },
|
||||
{
|
||||
$set: {
|
||||
matchSideEffectTransitionId: 'target',
|
||||
matchSideEffectRevision: 1,
|
||||
matchSideEffectStatus: 'pending',
|
||||
},
|
||||
},
|
||||
);
|
||||
const payload = {
|
||||
auctionId: f.auctionId.toString(),
|
||||
matchId: f.matchId.toString(),
|
||||
auctionLifecycleId: 'life',
|
||||
auctionStateTransitionId: 'target',
|
||||
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
|
||||
afterAuctionMongoProjection: { matchSideEffectRevision: 1 },
|
||||
};
|
||||
const state = {
|
||||
auctionLifecycleId: scenario === 'new-lifecycle' ? 'new-life' : 'life',
|
||||
matchSideEffectTransitionId:
|
||||
scenario === 'new-transition' ? 'new' : 'target',
|
||||
matchSideEffectStatus: scenario === 'completed' ? 'completed' : 'pending',
|
||||
matchSideEffectRevision: 40,
|
||||
};
|
||||
const sendCommand = jest.fn(async (command: string[]) => {
|
||||
if (command[0] === 'ZRANGEBYSCORE') return ['terminal'];
|
||||
if (command[0] === 'HGETALL')
|
||||
return { payloadJson: JSON.stringify(payload) };
|
||||
if (command[0] === 'HGET') return 'exhausted';
|
||||
return 1;
|
||||
});
|
||||
const broadcast = jest.fn();
|
||||
const mirror = jest.fn().mockResolvedValue(scenario !== 'mirror-superseded');
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
f.database as never,
|
||||
{
|
||||
raw: () => ({ sendCommand }),
|
||||
getCurrentRedisTimestampMS: async () => 1000,
|
||||
} as never,
|
||||
undefined,
|
||||
undefined,
|
||||
{ mirrorMatchSideEffectStatus: mirror } as never,
|
||||
{ broadcastMatchSideEffect: broadcast } as never,
|
||||
{ readAuctionStateView: async () => state } as never,
|
||||
);
|
||||
await scheduler.reconcileDeadLetterStatuses();
|
||||
const changed = ['revision-ahead', 'mirror-superseded'].includes(scenario);
|
||||
expect(
|
||||
(await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus,
|
||||
).toBe(changed ? 'failed' : 'pending');
|
||||
if (changed)
|
||||
expect(mirror).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ matchSideEffectRevision: 41 }),
|
||||
);
|
||||
expect(broadcast).toHaveBeenCalledTimes(
|
||||
scenario === 'revision-ahead' ? 1 : 0,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,598 @@
|
||||
import { createClient } from 'redis';
|
||||
import mongoose from 'mongoose';
|
||||
import { AuctionService } from '../src/modules/auction/auction.service';
|
||||
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
|
||||
import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service';
|
||||
import { AuctionRedisStateReaderService } from '../src/modules/auction/redis/auction-redis-state-reader.service';
|
||||
import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys';
|
||||
import {
|
||||
AuctionLifecycleLock,
|
||||
AuctionLifecycleBusyError,
|
||||
} from '../src/modules/auction/auction-lifecycle-lock';
|
||||
import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler';
|
||||
import { AuctionSideEffectService } from '../src/modules/auction/auction-side-effect.service';
|
||||
import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service';
|
||||
import {
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_QUEUE_KEY,
|
||||
AUCTION_BID_ARCHIVE_PAYLOADS_KEY,
|
||||
AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY,
|
||||
} from '../src/modules/auction/redis/auction-redis.keys';
|
||||
|
||||
const client = createClient({
|
||||
url: process.env.REDIS_TEST_URL,
|
||||
socket: { reconnectStrategy: false },
|
||||
});
|
||||
const redis = new AuctionRedisService(client);
|
||||
const commands = new AuctionRedisCommandService(redis);
|
||||
const reader = new AuctionRedisStateReaderService(redis);
|
||||
|
||||
beforeAll(async () => {
|
||||
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) {
|
||||
throw new Error('Use test:redis to start an isolated local Redis server.');
|
||||
}
|
||||
client.on('error', () => {});
|
||||
await client.connect();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await client.flushDb();
|
||||
});
|
||||
afterAll(async () => {
|
||||
if (client.isOpen) await client.quit();
|
||||
});
|
||||
|
||||
it.each(['count', 'elapsed'])(
|
||||
'dead-letters permanently pending side effects by %s while retaining the payload',
|
||||
async (limit) => {
|
||||
const key = 'review:pending';
|
||||
const now = Date.now();
|
||||
await client.hSet(key, {
|
||||
payloadJson: '{}',
|
||||
pendingAttempts: limit === 'count' ? '119' : '1',
|
||||
pendingSinceTimestampMS: String(limit === 'elapsed' ? now - 600000 : now),
|
||||
});
|
||||
await client.zAdd(AUCTION_DURABILITY_RETRY_QUEUE_KEY, {
|
||||
score: now,
|
||||
value: key,
|
||||
});
|
||||
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
|
||||
const internal = scheduler as unknown as {
|
||||
deferDurabilityPayload(
|
||||
key: string,
|
||||
time: number,
|
||||
error: unknown,
|
||||
): Promise<string>;
|
||||
};
|
||||
expect(
|
||||
await internal.deferDurabilityPayload(
|
||||
key,
|
||||
now,
|
||||
new Error('state unavailable'),
|
||||
),
|
||||
).toBe('dead-lettered');
|
||||
expect(
|
||||
await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key),
|
||||
).toBeNull();
|
||||
expect(
|
||||
await client.zScore(AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, key),
|
||||
).toBe(now);
|
||||
expect(await client.hGet(key, 'payloadJson')).toBe('{}');
|
||||
expect(await client.hGet(key, 'attempts')).toBeNull();
|
||||
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
|
||||
now,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('defers a temporary coordination wait without consuming failed-work attempts or recreating removed payloads', async () => {
|
||||
const key = 'review:pending';
|
||||
const now = Date.now();
|
||||
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
|
||||
const internal = scheduler as unknown as {
|
||||
deferDurabilityPayload(
|
||||
key: string,
|
||||
time: number,
|
||||
error: unknown,
|
||||
): Promise<string>;
|
||||
};
|
||||
expect(
|
||||
await internal.deferDurabilityPayload(key, now, new Error('busy')),
|
||||
).toBe('gone');
|
||||
expect(await client.exists(key)).toBe(0);
|
||||
await client.hSet(key, 'payloadJson', '{}');
|
||||
expect(
|
||||
await internal.deferDurabilityPayload(key, now, new Error('busy')),
|
||||
).toBe('requeued');
|
||||
expect(await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key)).toBe(
|
||||
now + 5000,
|
||||
);
|
||||
expect(await client.hGet(key, 'attempts')).toBeNull();
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const matchId = new mongoose.Types.ObjectId();
|
||||
const id = matchId.toString();
|
||||
const lifecycle = `life:${id}`;
|
||||
const playerA = new mongoose.Types.ObjectId().toString();
|
||||
const playerB = new mongoose.Types.ObjectId().toString();
|
||||
const captain = new mongoose.Types.ObjectId().toString();
|
||||
const initialRound = `initial:${id}`;
|
||||
const structure = {
|
||||
auctionPlayerOrder: [{ userId: playerA }, { userId: playerB }],
|
||||
auctionTeams: [
|
||||
{
|
||||
teamIndex: 1,
|
||||
captainUserId: captain,
|
||||
remainingBidPointAmount: 100000,
|
||||
members: [],
|
||||
},
|
||||
],
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentAuctionRound: 1,
|
||||
passedAuctionPlayerQueue: [],
|
||||
maximumBidRoundDurationSeconds: 30,
|
||||
bidRoundExtensionSeconds: 5,
|
||||
initialTeamBidPointAmount: 100000,
|
||||
topTierAuctionPriorityThreshold: 'NONE',
|
||||
manualTopTierAuctionPriorityUserIds: [],
|
||||
};
|
||||
await commands.hydrateState(id, {
|
||||
matchId: id,
|
||||
auctionLifecycleId: lifecycle,
|
||||
revision: 0,
|
||||
auctionHistorySequence: 1,
|
||||
phase: 'bidding',
|
||||
maximumBidRoundDurationSeconds: 30,
|
||||
structureStateJson: JSON.stringify(structure),
|
||||
liveStatePatch: {
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentBidRoundPlayerUserId: playerA,
|
||||
currentBidRoundId: initialRound,
|
||||
maximumBidRoundDurationSeconds: 30,
|
||||
bidRoundExtensionSeconds: 5,
|
||||
'team:1:captainUserId': captain,
|
||||
'team:1:remainingBidPointAmount': 100000,
|
||||
'team:1:assignedAuctionTeamMemberCount': 1,
|
||||
},
|
||||
});
|
||||
const auction = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
matchId,
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: lifecycle,
|
||||
phase: 'bidding',
|
||||
revision: 1,
|
||||
};
|
||||
const service = new AuctionService(
|
||||
{ getModel: async () => ({ findOne: async () => auction }) } as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
redis,
|
||||
commands,
|
||||
reader,
|
||||
);
|
||||
const bid = (
|
||||
requestId: string,
|
||||
amount: number,
|
||||
player = playerA,
|
||||
round = initialRound,
|
||||
) => ({
|
||||
matchId: id,
|
||||
bidderTeamIndex: 1,
|
||||
bidderUserId: captain,
|
||||
bidPointAmount: amount,
|
||||
bidRequestId: requestId,
|
||||
clientObservedAuctionLifecycleId: lifecycle,
|
||||
clientObservedCurrentBidRoundPlayerUserId: player,
|
||||
clientObservedBidRoundId: round,
|
||||
maximumAuctionTeamMemberCount: 5,
|
||||
minimumBidPointReservePerRemainingAuctionTeamMemberSlot: 10,
|
||||
maximumStoredAuctionBidHistoryCount: 1000,
|
||||
maximumStoredBidRequestResultCount: 500,
|
||||
minimumBidRequestResultReplayWindowMS: 300000,
|
||||
});
|
||||
const undo = async (snapshot: Record<string, unknown>) => {
|
||||
const internal = service as unknown as {
|
||||
undoRedisAuction(params: unknown): Promise<unknown>;
|
||||
};
|
||||
return internal.undoRedisAuction({
|
||||
auction,
|
||||
snapshot,
|
||||
targetHistorySequence: 1,
|
||||
undoingCompletedAuction: false,
|
||||
});
|
||||
};
|
||||
return {
|
||||
id,
|
||||
matchId,
|
||||
lifecycle,
|
||||
playerA,
|
||||
playerB,
|
||||
initialRound,
|
||||
auction,
|
||||
service,
|
||||
bid,
|
||||
undo,
|
||||
keys: buildAuctionRedisKeys(id),
|
||||
};
|
||||
}
|
||||
|
||||
it('waits briefly for a busy recovery lock and returns the recovered state', async () => {
|
||||
const f = await fixture();
|
||||
f.auction.revision = 10;
|
||||
const key = `auction:${f.id}:lifecycleMutationLock`;
|
||||
await redis.acquireTokenLock(key, 'other-reader', 30000);
|
||||
const released = new Promise((resolve, reject) => {
|
||||
setTimeout(() => {
|
||||
void redis.releaseTokenLock(key, 'other-reader').then(resolve, reject);
|
||||
}, 100);
|
||||
});
|
||||
const [view] = await Promise.all([
|
||||
f.service.getStateByMatchId(f.matchId),
|
||||
released,
|
||||
]);
|
||||
expect(view).toMatchObject({ auctionLifecycleId: f.lifecycle, revision: 11 });
|
||||
});
|
||||
|
||||
it('bounds recovery waits and reports a retriable state error without releasing another owner lock', async () => {
|
||||
const f = await fixture();
|
||||
f.auction.revision = 10;
|
||||
const key = `auction:${f.id}:lifecycleMutationLock`;
|
||||
await redis.acquireTokenLock(key, 'slow-owner', 30000);
|
||||
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toMatchObject({
|
||||
response: { code: 'AUCTION_STATE_RECOVERING', retryAfterMS: 500 },
|
||||
});
|
||||
expect(await client.get(key)).toBe('slow-owner');
|
||||
});
|
||||
|
||||
it('keeps accepting bids after 1000 history entries and preserves accepted request replay across players', async () => {
|
||||
const f = await fixture();
|
||||
const first = await commands.placeBid(
|
||||
f.id,
|
||||
f.bid('accepted-before-next', 10),
|
||||
);
|
||||
expect(first).toMatchObject({ ok: true });
|
||||
await client.rPush(
|
||||
f.keys.bidHistoryKey,
|
||||
Array.from({ length: 999 }, () => JSON.stringify({ bidPointAmount: 10 })),
|
||||
);
|
||||
await client.hSet(f.keys.liveStateKey, 'phase', 'sold');
|
||||
await f.service.next(f.matchId);
|
||||
await f.service.openBidding(f.matchId);
|
||||
const state = (await reader.readAuctionStateView(f.id))!;
|
||||
const result = await commands.placeBid(
|
||||
f.id,
|
||||
f.bid('next-player', 10, f.playerB, String(state.currentBidRoundId)),
|
||||
);
|
||||
expect(result).toMatchObject({ ok: true });
|
||||
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
|
||||
expect(
|
||||
await commands.placeBid(f.id, f.bid('accepted-before-next', 10)),
|
||||
).toMatchObject({ ...first, isReplay: true });
|
||||
});
|
||||
|
||||
it('500 rejected requests do not block a valid bid, and rejection retention stays bounded', async () => {
|
||||
const f = await fixture();
|
||||
await client.hSet(f.keys.liveStateKey, 'currentHighestBidPointAmount', '10');
|
||||
for (let i = 0; i < 501; i++) {
|
||||
expect(
|
||||
await commands.placeBid(f.id, f.bid(`too-low:${i}`, 5)),
|
||||
).toMatchObject({ rejected: true, code: 'TOO_LOW' });
|
||||
}
|
||||
expect(await client.lLen(f.keys.rejectedBidRequestResultOrderKey)).toBe(500);
|
||||
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(0);
|
||||
expect(await commands.placeBid(f.id, f.bid('valid', 20))).toMatchObject({
|
||||
ok: true,
|
||||
});
|
||||
expect(await commands.placeBid(f.id, f.bid('valid', 30))).toMatchObject({
|
||||
rejected: true,
|
||||
code: 'DUPLICATE_REQUEST_CONFLICT',
|
||||
});
|
||||
});
|
||||
|
||||
it('still protects 500 accepted results within their replay window', async () => {
|
||||
const f = await fixture();
|
||||
for (let i = 1; i <= 500; i++)
|
||||
expect(
|
||||
await commands.placeBid(f.id, f.bid(`accepted:${i}`, i)),
|
||||
).toMatchObject({ ok: true });
|
||||
expect(await commands.placeBid(f.id, f.bid('overflow', 501))).toMatchObject({
|
||||
rejected: true,
|
||||
code: 'BID_REQUEST_RESULT_LIMIT',
|
||||
});
|
||||
expect(await commands.placeBid(f.id, f.bid('accepted:1', 1))).toMatchObject({
|
||||
ok: true,
|
||||
isReplay: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('migrates rejected results from the former shared quota without losing accepted replays', async () => {
|
||||
const f = await fixture();
|
||||
await commands.placeBid(f.id, f.bid('accepted', 10));
|
||||
await client.hDel(f.keys.bidRequestResultsKey, '_acceptedOnlyOrderV1');
|
||||
for (let i = 0; i < 499; i++) {
|
||||
const id = `legacy-rejected:${i}`;
|
||||
await client.hSet(f.keys.bidRequestResultsKey, {
|
||||
[`${id}:status`]: 'rejected',
|
||||
[`${id}:createdAtTimestampMS`]: String(Date.now()),
|
||||
});
|
||||
await client.rPush(f.keys.bidRequestResultOrderKey, id);
|
||||
}
|
||||
expect(
|
||||
await commands.placeBid(f.id, f.bid('next-accepted', 20)),
|
||||
).toMatchObject({ ok: true });
|
||||
expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(2);
|
||||
expect(await commands.placeBid(f.id, f.bid('accepted', 10))).toMatchObject({
|
||||
ok: true,
|
||||
isReplay: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('encodes opaque round identities safely in bid history', async () => {
|
||||
const f = await fixture();
|
||||
const round = 'round:with"quotes\\and-slashes';
|
||||
await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', round);
|
||||
expect(
|
||||
await commands.placeBid(f.id, f.bid('quoted-round', 10, f.playerA, round)),
|
||||
).toMatchObject({ ok: true, auctionBidHistoryEntry: { bidRoundId: round } });
|
||||
expect((await reader.readAuctionStateView(f.id))?.auctionBidHistory).toEqual([
|
||||
expect.objectContaining({ bidRoundId: round }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('rejects delayed pre-undo bids and omitted round identities while accepting the restored round', async () => {
|
||||
const f = await fixture();
|
||||
const snapshot = (await reader.readAuctionStateView(f.id))!;
|
||||
await f.undo(snapshot);
|
||||
const restored = (await reader.readAuctionStateView(f.id))!;
|
||||
expect(restored.currentBidRoundId).not.toBe(f.initialRound);
|
||||
expect(await commands.placeBid(f.id, f.bid('old-round', 100))).toMatchObject({
|
||||
code: 'STALE_BID_ROUND',
|
||||
});
|
||||
expect(
|
||||
await commands.placeBid(f.id, {
|
||||
...f.bid('missing-round', 100),
|
||||
clientObservedBidRoundId: undefined,
|
||||
}),
|
||||
).toMatchObject({ code: 'BID_CLIENT_UPDATE_REQUIRED' });
|
||||
expect(
|
||||
await commands.placeBid(
|
||||
f.id,
|
||||
f.bid('new-round', 100, f.playerA, String(restored.currentBidRoundId)),
|
||||
),
|
||||
).toMatchObject({
|
||||
ok: true,
|
||||
auctionBidHistoryEntry: {
|
||||
playerUserId: f.playerA,
|
||||
bidRoundId: restored.currentBidRoundId,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('removes completion TTLs when resuming an auction', async () => {
|
||||
const f = await fixture();
|
||||
const snapshot = (await reader.readAuctionStateView(f.id))!;
|
||||
await client.expire(f.keys.liveStateKey, 604800);
|
||||
await client.expire(f.keys.structureStateKey, 604800);
|
||||
await f.undo(snapshot);
|
||||
expect(await client.ttl(f.keys.liveStateKey)).toBe(-1);
|
||||
expect(await client.ttl(f.keys.structureStateKey)).toBe(-1);
|
||||
});
|
||||
|
||||
it('fences direct foreign-lifecycle hydration and repairs an orphan using current Mongo authority', async () => {
|
||||
const f = await fixture();
|
||||
await client.hSet(f.keys.liveStateKey, 'revision', '150');
|
||||
expect(
|
||||
await commands.hydrateState(f.id, {
|
||||
matchId: f.id,
|
||||
auctionLifecycleId: 'replacement',
|
||||
revision: 0,
|
||||
auctionHistorySequence: 1,
|
||||
phase: 'presenting',
|
||||
maximumBidRoundDurationSeconds: 30,
|
||||
structureStateJson: '{}',
|
||||
}),
|
||||
).toMatchObject({ rejected: true, code: 'LIFECYCLE_CONFLICT' });
|
||||
const current = {
|
||||
...f.auction,
|
||||
auctionLifecycleId: 'replacement',
|
||||
phase: 'presenting',
|
||||
revision: 1,
|
||||
auctionTeams: [],
|
||||
auctionPlayerOrder: [],
|
||||
};
|
||||
jest.spyOn(f.service, 'getByMatchId').mockResolvedValue(current as never);
|
||||
expect(await f.service.getStateByMatchId(f.matchId)).toMatchObject({
|
||||
auctionLifecycleId: 'replacement',
|
||||
phase: 'presenting',
|
||||
});
|
||||
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
|
||||
'replacement',
|
||||
);
|
||||
});
|
||||
|
||||
it('does not let an old Mongo read overwrite the newly started lifecycle', async () => {
|
||||
const f = await fixture();
|
||||
const stale = { ...f.auction, auctionLifecycleId: 'obsolete', revision: 500 };
|
||||
jest
|
||||
.spyOn(f.service, 'getByMatchId')
|
||||
.mockResolvedValueOnce(stale as never)
|
||||
.mockResolvedValue(f.auction as never);
|
||||
await expect(f.service.getStateByMatchId(f.matchId)).rejects.toThrow(
|
||||
'lifecycle has changed',
|
||||
);
|
||||
expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe(
|
||||
f.lifecycle,
|
||||
);
|
||||
});
|
||||
|
||||
it('serializes lifecycle mutations without deleting another owner lock', async () => {
|
||||
const lock = new AuctionLifecycleLock(redis);
|
||||
await lock.run('match', async () => {
|
||||
await expect(lock.run('match', async () => {})).rejects.toThrow(
|
||||
AuctionLifecycleBusyError,
|
||||
);
|
||||
});
|
||||
await expect(lock.run('match', async () => 'done')).resolves.toBe('done');
|
||||
});
|
||||
|
||||
it('persists the exact Redis deadline when opening bidding and restoring a bidding snapshot', async () => {
|
||||
const f = await fixture();
|
||||
await client.hSet(f.keys.liveStateKey, 'phase', 'presenting');
|
||||
await client.hDel(f.keys.liveStateKey, 'currentBidRoundDeadlineTimestampMS');
|
||||
await f.service.openBidding(f.matchId);
|
||||
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
|
||||
const verify = async () => {
|
||||
const live = (await reader.readAuctionStateView(f.id))!;
|
||||
const queued = await client.zRange(
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
0,
|
||||
-1,
|
||||
);
|
||||
const payloads = await Promise.all(
|
||||
queued.map((key) => scheduler.readDurabilityPayload(key)),
|
||||
);
|
||||
const payload = payloads.find(
|
||||
(entry) => entry?.revision === live.revision,
|
||||
)!;
|
||||
expect(
|
||||
payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS,
|
||||
).toBe(live.currentBidRoundDeadlineTimestampMS);
|
||||
expect(
|
||||
new Date(
|
||||
String(payload.afterAuctionMongoProjection.currentDeadline),
|
||||
).getTime(),
|
||||
).toBe(live.currentBidRoundDeadlineTimestampMS);
|
||||
expect(
|
||||
payload.afterAuctionMongoProjection.passedAuctionPlayerQueue,
|
||||
).toEqual([]);
|
||||
return live;
|
||||
};
|
||||
const snapshot = await verify();
|
||||
await f.undo({ ...snapshot, currentBidRoundDeadlineTimestampMS: 1 });
|
||||
await verify();
|
||||
});
|
||||
|
||||
it('archives every accepted bid beyond the recent 1000, with no duplicate archive event for a replay', async () => {
|
||||
const f = await fixture();
|
||||
for (let amount = 1; amount <= 1005; amount++) {
|
||||
expect(
|
||||
await commands.placeBid(f.id, {
|
||||
...f.bid(`archive:${amount}`, amount),
|
||||
maximumStoredBidRequestResultCount: 2000,
|
||||
}),
|
||||
).toMatchObject({ ok: true });
|
||||
}
|
||||
expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000);
|
||||
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
|
||||
expect(
|
||||
await commands.placeBid(f.id, {
|
||||
...f.bid('archive:1', 1),
|
||||
maximumStoredBidRequestResultCount: 2000,
|
||||
}),
|
||||
).toMatchObject({ ok: true, isReplay: true });
|
||||
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005);
|
||||
const [first] = await client.zRange(AUCTION_BID_ARCHIVE_QUEUE_KEY, 0, 0);
|
||||
expect(
|
||||
JSON.parse((await client.hGet(AUCTION_BID_ARCHIVE_PAYLOADS_KEY, first))!),
|
||||
).toMatchObject({
|
||||
bidPointAmount: 1,
|
||||
bidRequestId: 'archive:1',
|
||||
playerUserId: f.playerA,
|
||||
auctionLifecycleId: f.lifecycle,
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['missing', 'empty'])(
|
||||
'archives an accepted pre-upgrade bid with a %s Redis round ID',
|
||||
async (kind) => {
|
||||
const f = await fixture();
|
||||
if (kind === 'missing')
|
||||
await client.hDel(f.keys.liveStateKey, 'currentBidRoundId');
|
||||
else await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', '');
|
||||
expect(
|
||||
await commands.placeBid(f.id, {
|
||||
...f.bid('legacy-round', 10),
|
||||
clientObservedBidRoundId: undefined,
|
||||
}),
|
||||
).toMatchObject({
|
||||
ok: true,
|
||||
auctionBidHistoryEntry: { bidRoundId: '', playerUserId: f.playerA },
|
||||
});
|
||||
const bulkWrite = jest.fn(async (operations: unknown[]) => ({
|
||||
matchedCount: 0,
|
||||
upsertedCount: operations.length,
|
||||
}));
|
||||
const archive = new AuctionBidArchiveService(
|
||||
{ getModel: async () => ({ bulkWrite }) } as never,
|
||||
redis,
|
||||
);
|
||||
await expect(archive.persistPendingBids()).resolves.toBe(1);
|
||||
const write = bulkWrite.mock.calls[0][0][0] as {
|
||||
updateOne: { update: { $setOnInsert: Record<string, unknown> } };
|
||||
};
|
||||
expect(write.updateOne.update.$setOnInsert).not.toHaveProperty(
|
||||
'bidRoundId',
|
||||
);
|
||||
expect(String(write.updateOne.update.$setOnInsert.playerUserId)).toBe(
|
||||
f.playerA,
|
||||
);
|
||||
expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it('does not mirror an old generation failure over a newer Redis side effect', async () => {
|
||||
const f = await fixture();
|
||||
await client.hSet(f.keys.liveStateKey, {
|
||||
matchSideEffectTransitionId: 'new',
|
||||
matchSideEffectRevision: '2',
|
||||
matchSideEffectStatus: 'pending',
|
||||
});
|
||||
const sideEffects = new AuctionSideEffectService(redis);
|
||||
expect(
|
||||
await sideEffects.mirrorMatchSideEffectStatus({
|
||||
matchId: f.id,
|
||||
auctionLifecycleId: f.lifecycle,
|
||||
auctionStateTransitionId: 'old',
|
||||
matchSideEffectRevision: 100,
|
||||
matchSideEffectStatus: 'failed',
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(await client.hGet(f.keys.liveStateKey, 'matchSideEffectStatus')).toBe(
|
||||
'pending',
|
||||
);
|
||||
});
|
||||
|
||||
it('queues a terminal notification for ordinary failures and backfills old dead letters', async () => {
|
||||
const key = 'review:ordinary-failure';
|
||||
await client.hSet(key, { payloadJson: '{}', attempts: '9' });
|
||||
const scheduler = new AuctionDurabilityScheduler({} as never, redis);
|
||||
const internal = scheduler as unknown as {
|
||||
recordDurabilityPayloadFailure(
|
||||
key: string,
|
||||
now: number,
|
||||
error: unknown,
|
||||
): Promise<string>;
|
||||
};
|
||||
expect(
|
||||
await internal.recordDurabilityPayloadFailure(
|
||||
key,
|
||||
1000,
|
||||
new Error('failed before worker'),
|
||||
),
|
||||
).toBe('dead-lettered');
|
||||
expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe(
|
||||
1000,
|
||||
);
|
||||
// Simulate an older deployment which did not create notification jobs.
|
||||
await client.zRem(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key);
|
||||
await scheduler.reconcileDeadLetterStatuses();
|
||||
expect(
|
||||
await client.hGet(key, 'deadLetterStatusReconciledAtTimestampMS'),
|
||||
).not.toBeNull();
|
||||
expect(
|
||||
await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key),
|
||||
).toBeNull();
|
||||
});
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"rootDir": "..",
|
||||
"testEnvironment": "node",
|
||||
"testRegex": "test/.*\\.mongo-spec\\.ts$",
|
||||
"transform": { "^.+\\.ts$": "ts-jest" },
|
||||
"testTimeout": 15000
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
import { CommunityStatsService } from '../src/modules/community-stats/community-stats.service';
|
||||
import { PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT } from '../src/modules/riot-job-queue/riot-job-queue.scripts';
|
||||
import mongoose from 'mongoose';
|
||||
import { CompetitionOutboxProcessor } from '../src/modules/tournament-result/competition-outbox.processor';
|
||||
|
||||
const client = createClient({
|
||||
url: process.env.REDIS_TEST_URL,
|
||||
@@ -165,6 +166,89 @@ describe('Redis native operations', () => {
|
||||
expect(await client.exists('lock')).toBe(0);
|
||||
});
|
||||
|
||||
it('defers locked Competition effects without spending a retry, then releases its own lock after delivery', async () => {
|
||||
const locks = new DistributedLockService(redis);
|
||||
const competitionId = new mongoose.Types.ObjectId();
|
||||
const matchId = new mongoose.Types.ObjectId();
|
||||
const lockKey = `competition:projection-effects:${competitionId.toString()}`;
|
||||
const owner = await locks.acquireLock({
|
||||
lockKey,
|
||||
lockExpirationDurationMilliseconds: 60_000,
|
||||
});
|
||||
expect(owner).not.toBeNull();
|
||||
const event = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
competitionId,
|
||||
revision: 2,
|
||||
type: 'competition.projection-updated',
|
||||
state: 'pending',
|
||||
attemptCount: 10,
|
||||
};
|
||||
const query = <T>(value: T) => ({
|
||||
lean: jest.fn().mockReturnThis(),
|
||||
exec: async () => value,
|
||||
});
|
||||
const publish = jest.fn(() => query({ matchedCount: 1 }));
|
||||
const models = {
|
||||
competitionOutbox: {
|
||||
findOneAndUpdate: () => query(event),
|
||||
updateOne: publish,
|
||||
},
|
||||
competition: {
|
||||
findById: () =>
|
||||
query({
|
||||
_id: competitionId,
|
||||
matchId,
|
||||
communityId: new mongoose.Types.ObjectId(),
|
||||
revision: 2,
|
||||
phase: 'running',
|
||||
teams: [],
|
||||
projection: { phase: 'running', games: [] },
|
||||
}),
|
||||
},
|
||||
match: {
|
||||
findById: () => query({ status: 'in-progress' }),
|
||||
updateOne: () => query({ matchedCount: 1 }),
|
||||
},
|
||||
};
|
||||
const buildAndUpsert = jest.fn().mockResolvedValue(undefined);
|
||||
const invalidateCommunity = jest.fn().mockResolvedValue(undefined);
|
||||
const processor = new CompetitionOutboxProcessor(
|
||||
{ getModel: async (name: keyof typeof models) => models[name] } as never,
|
||||
{
|
||||
provisionReadyAssignments: jest.fn().mockResolvedValue(undefined),
|
||||
} as never,
|
||||
{ buildAndUpsert } as never,
|
||||
{ invalidateCommunity } as never,
|
||||
locks,
|
||||
);
|
||||
try {
|
||||
await processor.drainOnce(1);
|
||||
expect(buildAndUpsert).not.toHaveBeenCalled();
|
||||
expect(publish).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ attemptCount: 10 }),
|
||||
{
|
||||
$set: { state: 'pending', availableAt: expect.any(Date) },
|
||||
$inc: { attemptCount: -1 },
|
||||
},
|
||||
);
|
||||
expect(await client.get(lockKey)).toBe(owner!.lockOwnershipToken);
|
||||
} finally {
|
||||
await locks.releaseLockIfOwned(owner!);
|
||||
}
|
||||
|
||||
await processor.drainOnce(1);
|
||||
expect(buildAndUpsert).toHaveBeenCalledTimes(1);
|
||||
expect(invalidateCommunity).toHaveBeenCalledTimes(1);
|
||||
expect(publish).toHaveBeenLastCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
$set: expect.objectContaining({ state: 'published' }),
|
||||
}),
|
||||
);
|
||||
expect(await client.exists(lockKey)).toBe(0);
|
||||
});
|
||||
|
||||
it('does not clear the dedup token of a newer refresh job', async () => {
|
||||
const job = {
|
||||
jobId: 'old',
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createRequire } from 'node:module';
|
||||
import { mkdtemp, mkdir, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, relative, resolve } from 'node:path';
|
||||
import { createServer } from 'node:net';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const listener = createServer();
|
||||
await new Promise((accept) => listener.listen(0, '127.0.0.1', accept));
|
||||
const port = listener.address().port;
|
||||
await new Promise((accept) => listener.close(accept));
|
||||
const directory = await mkdtemp(join(tmpdir(), 'lolympus-mongo-test-'));
|
||||
await mkdir(join(directory, 'data'));
|
||||
const mongo = spawn(
|
||||
process.env.MONGODB_TEST_SERVER || 'mongod',
|
||||
[
|
||||
'--bind_ip',
|
||||
'127.0.0.1',
|
||||
'--port',
|
||||
String(port),
|
||||
'--replSet',
|
||||
'auction_review',
|
||||
'--dbpath',
|
||||
join(directory, 'data'),
|
||||
'--logpath',
|
||||
join(directory, 'mongod.log'),
|
||||
'--wiredTigerCacheSizeGB',
|
||||
'0.25',
|
||||
],
|
||||
{ windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] },
|
||||
);
|
||||
let startupError;
|
||||
mongo.on('error', (error) => {
|
||||
startupError = error;
|
||||
});
|
||||
const closed = new Promise((accept) => mongo.once('close', accept));
|
||||
let output = '';
|
||||
mongo.stdout.on('data', (chunk) => {
|
||||
output += chunk;
|
||||
});
|
||||
mongo.stderr.on('data', (chunk) => {
|
||||
output += chunk;
|
||||
});
|
||||
const uri = `mongodb://127.0.0.1:${port}/auction_review`;
|
||||
const client = new mongoose.mongo.MongoClient(`${uri}?directConnection=true`, {
|
||||
serverSelectionTimeoutMS: 500,
|
||||
});
|
||||
try {
|
||||
let connected = false;
|
||||
for (let attempt = 0; attempt < 60; attempt++) {
|
||||
if (startupError) throw startupError;
|
||||
if (mongo.exitCode !== null) throw new Error(`MongoDB exited: ${output}`);
|
||||
try {
|
||||
await client.connect();
|
||||
connected = true;
|
||||
break;
|
||||
} catch {
|
||||
await delay(100);
|
||||
}
|
||||
}
|
||||
if (!connected) throw new Error(`MongoDB did not start: ${output}`);
|
||||
await client
|
||||
.db('admin')
|
||||
.command({
|
||||
replSetInitiate: {
|
||||
_id: 'auction_review',
|
||||
members: [{ _id: 0, host: `127.0.0.1:${port}` }],
|
||||
},
|
||||
});
|
||||
let primary = false;
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
if ((await client.db('admin').command({ hello: 1 })).isWritablePrimary) {
|
||||
primary = true;
|
||||
break;
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
if (!primary)
|
||||
throw new Error('MongoDB replica set did not elect its primary.');
|
||||
const build = await client.db('admin').command({ buildInfo: 1 });
|
||||
console.log(
|
||||
`Integration tests using isolated MongoDB ${build.version}, Mongoose ${mongoose.version}, loopback port ${port}`,
|
||||
);
|
||||
const jest = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
require.resolve('jest/bin/jest'),
|
||||
'--config',
|
||||
'test/jest-mongo.json',
|
||||
'--runInBand',
|
||||
...process.argv.slice(2),
|
||||
],
|
||||
{
|
||||
cwd: resolve(import.meta.dirname, '..'),
|
||||
windowsHide: true,
|
||||
stdio: 'inherit',
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_ENV: 'test',
|
||||
MONGO_TEST_ISOLATED: '1',
|
||||
MONGO_TEST_URL: `${uri}?replicaSet=auction_review`,
|
||||
},
|
||||
},
|
||||
);
|
||||
process.exitCode = await new Promise((accept, reject) => {
|
||||
jest.once('error', reject);
|
||||
jest.once('exit', (code) => accept(code ?? 1));
|
||||
});
|
||||
} finally {
|
||||
await client
|
||||
.db('admin')
|
||||
.command({ shutdown: 1, force: true })
|
||||
.catch(() => {});
|
||||
await client.close().catch(() => {});
|
||||
if (mongo.pid && mongo.exitCode === null) mongo.kill();
|
||||
await closed;
|
||||
// Only remove the unique temporary directory created by this runner.
|
||||
const target = relative(resolve(tmpdir()), resolve(directory));
|
||||
if (
|
||||
!target.startsWith('lolympus-mongo-test-') ||
|
||||
target.includes('..') ||
|
||||
/[/\\]/.test(target)
|
||||
)
|
||||
throw new Error('Unsafe MongoDB test directory.');
|
||||
await rm(directory, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
maxRetries: 10,
|
||||
retryDelay: 100,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { ApiAuction } from './types';
|
||||
|
||||
export interface AuctionResyncReply {
|
||||
ok?: boolean;
|
||||
state?: ApiAuction;
|
||||
retryAfterMS?: number;
|
||||
}
|
||||
|
||||
function waitForRetry(
|
||||
milliseconds: number,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const finish = () => {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener('abort', finish);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(finish, milliseconds);
|
||||
signal.addEventListener('abort', finish, { once: true });
|
||||
if (signal.aborted) finish();
|
||||
});
|
||||
}
|
||||
|
||||
/** Coalesces simultaneous gaps and discards replies from an old connection. */
|
||||
export function createAuctionResyncRequester(
|
||||
fetchState: () => Promise<AuctionResyncReply>,
|
||||
applyState: (state: ApiAuction) => void,
|
||||
wait = waitForRetry,
|
||||
) {
|
||||
let pending: Promise<void> | undefined;
|
||||
let controller = new AbortController();
|
||||
let minimumRevision = -1;
|
||||
let lifecycleId: string | undefined;
|
||||
return {
|
||||
request(revision = -1, lifecycle?: string): Promise<void> {
|
||||
if (revision >= 0) {
|
||||
minimumRevision =
|
||||
lifecycle === lifecycleId
|
||||
? Math.max(minimumRevision, revision)
|
||||
: revision;
|
||||
lifecycleId = lifecycle;
|
||||
}
|
||||
if (pending) return pending;
|
||||
const signal = controller.signal;
|
||||
pending = (async () => {
|
||||
for (let attempt = 0; attempt < 3 && !signal.aborted; attempt++) {
|
||||
let retryAfterMS = 500 * (attempt + 1);
|
||||
try {
|
||||
const reply = await fetchState();
|
||||
if (signal.aborted) return;
|
||||
if (reply.ok && reply.state) {
|
||||
applyState(reply.state);
|
||||
// Another gap may have arrived while this snapshot was in flight.
|
||||
if (
|
||||
reply.state.auctionLifecycleId !== lifecycleId ||
|
||||
(reply.state.revision ?? -1) >= minimumRevision
|
||||
)
|
||||
return;
|
||||
}
|
||||
retryAfterMS = Math.min(
|
||||
3000,
|
||||
Math.max(retryAfterMS, reply.retryAfterMS ?? 0),
|
||||
);
|
||||
} catch {
|
||||
/* A timeout gets a bounded retry, never another room join. */
|
||||
}
|
||||
if (attempt < 2 && !signal.aborted) await wait(retryAfterMS, signal);
|
||||
}
|
||||
})().finally(() => {
|
||||
if (!signal.aborted) pending = undefined;
|
||||
});
|
||||
return pending;
|
||||
},
|
||||
cancel() {
|
||||
controller.abort();
|
||||
controller = new AbortController();
|
||||
pending = undefined;
|
||||
minimumRevision = -1;
|
||||
lifecycleId = undefined;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
import type { ApiAuction } from "./types";
|
||||
|
||||
export interface BidDeltaPayload {
|
||||
currentBidRoundId?: string;
|
||||
auctionBidHistoryEntry?: {
|
||||
bidderTeamIndex?: number;
|
||||
bidPointAmount?: number;
|
||||
bidderUserId?: string;
|
||||
bidPlacedAtTimestampMS?: number;
|
||||
bidRoundId?: string;
|
||||
playerUserId?: string;
|
||||
};
|
||||
auctionLifecycleId?: string;
|
||||
bidRequestId?: string;
|
||||
currentBidRoundDeadlineTimestampMS?: number;
|
||||
currentHighestBidderTeamIndex?: number;
|
||||
currentHighestBidPointAmount?: number;
|
||||
currentServerTimestampMS?: number;
|
||||
revision?: number;
|
||||
}
|
||||
|
||||
export function shouldApplyIncomingAuctionState(
|
||||
current: ApiAuction | null,
|
||||
incoming: ApiAuction,
|
||||
): boolean {
|
||||
if (!current) return true;
|
||||
if (incoming.shouldForceAuctionStateResync) return true;
|
||||
if (
|
||||
current.auctionLifecycleId &&
|
||||
incoming.auctionLifecycleId &&
|
||||
current.auctionLifecycleId !== incoming.auctionLifecycleId
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
typeof current.revision === "number" &&
|
||||
typeof incoming.revision === "number" &&
|
||||
incoming.revision < current.revision
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Full state can repair a delta at the same revision; side effects have a
|
||||
* separate revision and must not regress when a delayed full state arrives. */
|
||||
export function mergeAuctionState(
|
||||
current: ApiAuction | null,
|
||||
incoming: ApiAuction,
|
||||
): ApiAuction {
|
||||
if (
|
||||
current?.auctionLifecycleId === incoming.auctionLifecycleId &&
|
||||
current?.matchSideEffectTransitionId ===
|
||||
incoming.matchSideEffectTransitionId &&
|
||||
(current?.matchSideEffectRevision ?? -1) >
|
||||
(incoming.matchSideEffectRevision ?? -1)
|
||||
) {
|
||||
return {
|
||||
...incoming,
|
||||
matchSideEffectRevision: current!.matchSideEffectRevision,
|
||||
matchSideEffectStatus: current!.matchSideEffectStatus,
|
||||
};
|
||||
}
|
||||
return incoming;
|
||||
}
|
||||
|
||||
export function shouldResyncForBidDelta(
|
||||
current: ApiAuction | null,
|
||||
payload: BidDeltaPayload,
|
||||
): boolean {
|
||||
if (!current) return true;
|
||||
if (
|
||||
payload.auctionLifecycleId &&
|
||||
current.auctionLifecycleId &&
|
||||
payload.auctionLifecycleId !== current.auctionLifecycleId
|
||||
)
|
||||
return true;
|
||||
if (
|
||||
typeof payload.revision !== "number" ||
|
||||
typeof current.revision !== "number"
|
||||
)
|
||||
return true;
|
||||
if (payload.revision <= current.revision) return false;
|
||||
return (
|
||||
payload.revision !== current.revision + 1 ||
|
||||
current.phase !== "bidding" ||
|
||||
Boolean(
|
||||
current.currentBidRoundId &&
|
||||
payload.currentBidRoundId !== current.currentBidRoundId,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function applyBidDelta(
|
||||
current: ApiAuction | null,
|
||||
payload: BidDeltaPayload,
|
||||
): ApiAuction | null {
|
||||
if (!current) return current;
|
||||
if (shouldResyncForBidDelta(current, payload)) return current;
|
||||
if (
|
||||
payload.auctionLifecycleId &&
|
||||
current.auctionLifecycleId &&
|
||||
payload.auctionLifecycleId !== current.auctionLifecycleId
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
if (
|
||||
typeof current.revision === "number" &&
|
||||
typeof payload.revision === "number" &&
|
||||
payload.revision <= current.revision
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
|
||||
const bidPointAmount = Number(payload.currentHighestBidPointAmount);
|
||||
const bidderTeamIndex = Number(payload.currentHighestBidderTeamIndex);
|
||||
if (!Number.isFinite(bidPointAmount) || !Number.isFinite(bidderTeamIndex)) {
|
||||
return current;
|
||||
}
|
||||
|
||||
const bidHistoryEntry = normalizeBidHistoryEntry(
|
||||
payload.auctionBidHistoryEntry,
|
||||
);
|
||||
const currentBidRoundDeadlineTimestampMS =
|
||||
typeof payload.currentBidRoundDeadlineTimestampMS === "number"
|
||||
? payload.currentBidRoundDeadlineTimestampMS
|
||||
: current.currentBidRoundDeadlineTimestampMS;
|
||||
|
||||
return {
|
||||
...current,
|
||||
auctionBidHistory: appendUniqueAuctionBidHistoryEntry(
|
||||
current.auctionBidHistory ?? [],
|
||||
bidHistoryEntry,
|
||||
),
|
||||
bids: bidHistoryEntry
|
||||
? appendUniqueLegacyBidEntry(current.bids ?? [], bidHistoryEntry)
|
||||
: current.bids,
|
||||
currentBid: bidPointAmount,
|
||||
currentBidderTeamIndex: bidderTeamIndex,
|
||||
currentDeadline: currentBidRoundDeadlineTimestampMS
|
||||
? new Date(currentBidRoundDeadlineTimestampMS).toISOString()
|
||||
: current.currentDeadline,
|
||||
currentHighestBidPointAmount: bidPointAmount,
|
||||
currentHighestBidderTeamIndex: bidderTeamIndex,
|
||||
currentBidRoundDeadlineTimestampMS,
|
||||
currentServerTimestampMS:
|
||||
payload.currentServerTimestampMS ?? current.currentServerTimestampMS,
|
||||
revision: payload.revision ?? current.revision,
|
||||
currentBidRoundId: payload.currentBidRoundId ?? current.currentBidRoundId,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBidHistoryEntry(
|
||||
entry: BidDeltaPayload["auctionBidHistoryEntry"],
|
||||
): NonNullable<ApiAuction["auctionBidHistory"]>[number] | null {
|
||||
if (!entry) return null;
|
||||
const bidderTeamIndex = Number(entry.bidderTeamIndex);
|
||||
const bidPointAmount = Number(entry.bidPointAmount);
|
||||
const bidPlacedAtTimestampMS = Number(entry.bidPlacedAtTimestampMS);
|
||||
if (!Number.isFinite(bidderTeamIndex) || !Number.isFinite(bidPointAmount)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
bidRoundId: entry.bidRoundId,
|
||||
playerUserId: entry.playerUserId,
|
||||
bidderTeamIndex,
|
||||
bidPointAmount,
|
||||
bidderUserId: entry.bidderUserId ?? "",
|
||||
bidPlacedAtTimestampMS: Number.isFinite(bidPlacedAtTimestampMS)
|
||||
? bidPlacedAtTimestampMS
|
||||
: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function appendUniqueAuctionBidHistoryEntry(
|
||||
current: NonNullable<ApiAuction["auctionBidHistory"]>,
|
||||
entry: NonNullable<ApiAuction["auctionBidHistory"]>[number] | null,
|
||||
) {
|
||||
if (!entry) return current;
|
||||
if (
|
||||
current.some(
|
||||
(existing) =>
|
||||
existing.bidderTeamIndex === entry.bidderTeamIndex &&
|
||||
existing.bidPointAmount === entry.bidPointAmount &&
|
||||
existing.bidPlacedAtTimestampMS === entry.bidPlacedAtTimestampMS,
|
||||
)
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
return [...current, entry].slice(-1000);
|
||||
}
|
||||
|
||||
function appendUniqueLegacyBidEntry(
|
||||
current: ApiAuction["bids"],
|
||||
entry: NonNullable<ApiAuction["auctionBidHistory"]>[number],
|
||||
) {
|
||||
const currentBids = current ?? [];
|
||||
const legacyBid = {
|
||||
amount: entry.bidPointAmount,
|
||||
bidderUserId: entry.bidderUserId,
|
||||
teamIndex: entry.bidderTeamIndex,
|
||||
timestamp: new Date(entry.bidPlacedAtTimestampMS).toISOString(),
|
||||
};
|
||||
if (
|
||||
currentBids.some(
|
||||
(existing) =>
|
||||
existing.teamIndex === legacyBid.teamIndex &&
|
||||
existing.amount === legacyBid.amount &&
|
||||
existing.timestamp === legacyBid.timestamp,
|
||||
)
|
||||
) {
|
||||
return currentBids;
|
||||
}
|
||||
return [...currentBids, legacyBid].slice(-1000);
|
||||
}
|
||||
|
||||
/** HTTP 폴링 결과를 소켓 내부 상태(입찰 기준가·라운드 ID)에도 반영할지 결정한다.
|
||||
* lifecycle 교체는 소켓 이벤트로만 받는다. 늦게 도착한 이전 경매의 응답이
|
||||
* 새 경매를 덮어쓰지 않도록 기존 상태가 있으면 같은 lifecycle만 반영한다. */
|
||||
export function shouldApplyPolledAuctionState(
|
||||
current: ApiAuction | null,
|
||||
incoming: ApiAuction,
|
||||
matchId: string | undefined,
|
||||
): boolean {
|
||||
if (!matchId || incoming.matchId !== matchId) return false;
|
||||
if (!incoming.auctionLifecycleId || !Number.isFinite(incoming.revision))
|
||||
return false;
|
||||
if (!current) return true;
|
||||
if (
|
||||
!current.auctionLifecycleId ||
|
||||
current.auctionLifecycleId !== incoming.auctionLifecycleId
|
||||
)
|
||||
return false;
|
||||
return (
|
||||
typeof current.revision === "number" &&
|
||||
typeof incoming.revision === "number" &&
|
||||
incoming.revision >= current.revision
|
||||
);
|
||||
}
|
||||
@@ -28,7 +28,10 @@ const REASON_TO_USER_MESSAGE: Record<string, string> = {
|
||||
AUCTION_TRANSITION_REJECTED_BECAUSE_REQUEST_COORDINATION_RETRYABLE:
|
||||
'잠시 처리 중이라 반영되지 않았어요. 다시 시도해 주세요.',
|
||||
// 입찰
|
||||
BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED: '이미 다음 선수로 넘어갔어요.',
|
||||
BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED:
|
||||
'경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.',
|
||||
BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED:
|
||||
'이미 다음 선수로 넘어갔어요.',
|
||||
BID_REJECTED_BECAUSE_BID_ROUND_DEADLINE_PASSED: '마감되어 반영되지 않았어요.',
|
||||
BID_REJECTED_BECAUSE_AUCTION_NOT_IN_BIDDING_PHASE:
|
||||
'지금은 입찰할 수 없는 단계예요.',
|
||||
@@ -100,6 +103,8 @@ export function resolveAuctionErrorMessage(message?: string): string {
|
||||
}
|
||||
|
||||
/** 최신 상태로 rebaseline 후 재시도가 권장되는 거부인지. */
|
||||
export function isAuctionResyncRecommended(ack: AuctionRejectionAckLike): boolean {
|
||||
export function isAuctionResyncRecommended(
|
||||
ack: AuctionRejectionAckLike,
|
||||
): boolean {
|
||||
return ack.resyncRecommended === true;
|
||||
}
|
||||
@@ -197,6 +197,8 @@ export interface ApiAuction {
|
||||
currentAuctionPlayerOrderIndex: number;
|
||||
bids?: ApiAuctionBid[];
|
||||
auctionBidHistory?: Array<{
|
||||
bidRoundId?: string;
|
||||
playerUserId?: string;
|
||||
bidderTeamIndex: number;
|
||||
bidPointAmount: number;
|
||||
bidderUserId: string;
|
||||
@@ -213,6 +215,7 @@ export interface ApiAuction {
|
||||
currentDeadline?: string;
|
||||
currentBidRoundDeadlineTimestampMS?: number;
|
||||
currentBidRoundPlayerUserId?: string;
|
||||
currentBidRoundId?: string;
|
||||
currentServerTimestampMS?: number;
|
||||
initialPoints?: number;
|
||||
initialTeamBidPointAmount: number;
|
||||
@@ -224,6 +227,7 @@ export interface ApiAuction {
|
||||
auctionLifecycleId?: string;
|
||||
matchSideEffectRevision?: number;
|
||||
matchSideEffectStatus?: 'none' | 'pending' | 'processing' | 'completed' | 'failed';
|
||||
matchSideEffectTransitionId?: string;
|
||||
canPlaceBidInCurrentBidRound?: boolean;
|
||||
isAuctionStateTransitionInProgress?: boolean;
|
||||
shouldForceAuctionStateResync?: boolean;
|
||||
|
||||
@@ -3,6 +3,18 @@ import { io, type Socket } from 'socket.io-client';
|
||||
import { API_BASE_URL, useAuthUserId } from '../config';
|
||||
import { acquireAccessToken } from '../lib/auth/token-provider';
|
||||
import type { ApiAuction, ApiAuctionChatMessage, Position } from './types';
|
||||
import {
|
||||
createAuctionResyncRequester,
|
||||
type AuctionResyncReply,
|
||||
} from './auction-resync';
|
||||
import {
|
||||
applyBidDelta,
|
||||
shouldApplyIncomingAuctionState,
|
||||
shouldResyncForBidDelta,
|
||||
shouldApplyPolledAuctionState,
|
||||
mergeAuctionState,
|
||||
type BidDeltaPayload,
|
||||
} from './auction-state';
|
||||
import { resolveAuctionActionRejection } from './auction-action-retry';
|
||||
import {
|
||||
isAuctionResyncRecommended,
|
||||
@@ -21,6 +33,7 @@ type ConnectionStatus =
|
||||
| 'error';
|
||||
|
||||
interface BidPayload {
|
||||
clientObservedBidRoundId?: string;
|
||||
bidderTeamIndex: number;
|
||||
bidPointAmount: number;
|
||||
bidRequestId?: string;
|
||||
@@ -52,33 +65,19 @@ interface ChatHistoryPayload {
|
||||
|
||||
interface ResetPayload {
|
||||
matchId?: string;
|
||||
auctionLifecycleId?: string;
|
||||
}
|
||||
|
||||
type MatchSideEffectStatus = NonNullable<ApiAuction['matchSideEffectStatus']>;
|
||||
|
||||
interface MatchSideEffectPayload {
|
||||
auctionLifecycleId?: string;
|
||||
auctionStateTransitionId?: string;
|
||||
currentServerTimestampMS?: number;
|
||||
matchSideEffectRevision?: number;
|
||||
matchSideEffectStatus?: MatchSideEffectStatus;
|
||||
}
|
||||
|
||||
interface BidDeltaPayload {
|
||||
auctionBidHistoryEntry?: {
|
||||
bidderTeamIndex?: number;
|
||||
bidPointAmount?: number;
|
||||
bidderUserId?: string;
|
||||
bidPlacedAtTimestampMS?: number;
|
||||
};
|
||||
auctionLifecycleId?: string;
|
||||
bidRequestId?: string;
|
||||
currentBidRoundDeadlineTimestampMS?: number;
|
||||
currentHighestBidderTeamIndex?: number;
|
||||
currentHighestBidPointAmount?: number;
|
||||
currentServerTimestampMS?: number;
|
||||
revision?: number;
|
||||
}
|
||||
|
||||
interface BidServerTimingPayload {
|
||||
fallbackFullStatePath?: boolean;
|
||||
gatewayTotalDurationMS?: number;
|
||||
@@ -145,6 +144,9 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
? JSON.stringify([matchId, authUserId ?? null])
|
||||
: null;
|
||||
const socketRef = useRef<Socket | null>(null);
|
||||
const resyncRequesterRef = useRef<ReturnType<
|
||||
typeof createAuctionResyncRequester
|
||||
> | null>(null);
|
||||
const pendingBidTimeoutsRef = useRef<
|
||||
Record<string, ReturnType<typeof setTimeout>>
|
||||
>({});
|
||||
@@ -160,6 +162,7 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
const timeSyncSequenceRef = useRef(0);
|
||||
const hasSyncedServerTimeRef = useRef(false);
|
||||
const latestAuctionRef = useRef<ApiAuction | null>(null);
|
||||
const cancelledLifecycleIdsRef = useRef(new Set<string>());
|
||||
const latestConfirmedBidPointAmountRef = useRef(0);
|
||||
const [auction, setAuction] = useState<ApiAuction | null>(null);
|
||||
const [chatMessages, setChatMessages] = useState<ApiAuctionChatMessage[]>([]);
|
||||
@@ -187,6 +190,11 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
|
||||
const commitAuctionState = useCallback(
|
||||
(nextAuction: ApiAuction, source = 'socket-state'): boolean => {
|
||||
if (
|
||||
nextAuction.auctionLifecycleId &&
|
||||
cancelledLifecycleIdsRef.current.has(nextAuction.auctionLifecycleId)
|
||||
)
|
||||
return false;
|
||||
const currentAuction = latestAuctionRef.current;
|
||||
const applied = shouldApplyIncomingAuctionState(
|
||||
currentAuction,
|
||||
@@ -198,13 +206,12 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
: -1;
|
||||
const incomingRevision =
|
||||
typeof nextAuction?.revision === 'number' ? nextAuction.revision : -1;
|
||||
// 단조성을 깨고 적용되는 경우(=revision이 같거나 뒤로 가는데도 반영됨).
|
||||
// force-resync/lifecycle 교체로만 발생하며, "2~3 버전 롤백"의 직접 증거가 된다.
|
||||
// 같은 revision의 전체 상태는 delta 누락 복구이므로 역행으로 기록하지 않는다.
|
||||
const isBackwardApply =
|
||||
applied &&
|
||||
currentRevision >= 0 &&
|
||||
incomingRevision >= 0 &&
|
||||
incomingRevision <= currentRevision;
|
||||
incomingRevision < currentRevision;
|
||||
addNewRelicPageAction('AuctionClientStateApply', {
|
||||
matchId: matchId ?? 'none',
|
||||
source,
|
||||
@@ -219,6 +226,7 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
return false;
|
||||
}
|
||||
|
||||
nextAuction = mergeAuctionState(currentAuction, nextAuction);
|
||||
latestAuctionRef.current = nextAuction;
|
||||
latestConfirmedBidPointAmountRef.current =
|
||||
resolveCurrentBidPointAmount(nextAuction);
|
||||
@@ -230,6 +238,13 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
|
||||
const commitBidDelta = useCallback((payload: BidDeltaPayload): boolean => {
|
||||
const currentAuction = latestAuctionRef.current;
|
||||
if (shouldResyncForBidDelta(currentAuction, payload)) {
|
||||
void resyncRequesterRef.current?.request(
|
||||
payload.revision,
|
||||
payload.auctionLifecycleId,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
const nextAuction = applyBidDelta(currentAuction, payload);
|
||||
if (!nextAuction || nextAuction === currentAuction) {
|
||||
return false;
|
||||
@@ -242,6 +257,29 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
return true;
|
||||
}, []);
|
||||
|
||||
// HTTP 폴링은 화면만 갱신하므로, 소켓이 전이를 놓친 경우 입찰 기준가와
|
||||
// 라운드 ID가 낡은 채 남는다. 같은 경매의 더 새 상태면 소켓 상태에도 반영한다.
|
||||
const syncPolledAuctionState = useCallback(
|
||||
(polledAuction: ApiAuction | null | undefined) => {
|
||||
if (!polledAuction) return;
|
||||
if (
|
||||
polledAuction.auctionLifecycleId &&
|
||||
cancelledLifecycleIdsRef.current.has(polledAuction.auctionLifecycleId)
|
||||
)
|
||||
return;
|
||||
if (
|
||||
!shouldApplyPolledAuctionState(
|
||||
latestAuctionRef.current,
|
||||
polledAuction,
|
||||
matchId,
|
||||
)
|
||||
)
|
||||
return;
|
||||
commitAuctionState(polledAuction, 'http-poll');
|
||||
},
|
||||
[commitAuctionState, matchId],
|
||||
);
|
||||
|
||||
const setError = useCallback(
|
||||
(message: string | null) => {
|
||||
setErrorState({ key: connectionKey ?? '', message });
|
||||
@@ -336,7 +374,10 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
: undefined);
|
||||
const clientObservedRemainingBidRoundTimeMS =
|
||||
typeof currentDeadlineTimestampMS === 'number'
|
||||
? Math.max(0, currentDeadlineTimestampMS - (Date.now() + serverTimeOffsetMs))
|
||||
? Math.max(
|
||||
0,
|
||||
currentDeadlineTimestampMS - (Date.now() + serverTimeOffsetMs),
|
||||
)
|
||||
: undefined;
|
||||
const serverDeadlineTimestampMS =
|
||||
ack.bidDiagnostics?.currentBidRoundDeadlineTimestampMS ??
|
||||
@@ -399,6 +440,7 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
}
|
||||
|
||||
const activeConnectionKey = connectionKey;
|
||||
const cancelledLifecycleIds = cancelledLifecycleIdsRef.current;
|
||||
let disposed = false;
|
||||
|
||||
void (async () => {
|
||||
@@ -418,6 +460,33 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
});
|
||||
|
||||
socketRef.current = socket;
|
||||
const receiveAuctionState = (nextAuction: ApiAuction, source: string) => {
|
||||
const previousRevision = latestAuctionRef.current?.revision ?? -1;
|
||||
const didApplyState = commitAuctionState(nextAuction, source);
|
||||
if (didApplyState && (nextAuction.revision ?? 0) > previousRevision) {
|
||||
clearPendingActionRequests();
|
||||
}
|
||||
applyFallbackServerTimestamp(nextAuction.currentServerTimestampMS);
|
||||
if (nextAuction.chatMessages) {
|
||||
setChatMessages(mergeChatMessages([], nextAuction.chatMessages));
|
||||
}
|
||||
if (didApplyState) {
|
||||
setError(null);
|
||||
}
|
||||
};
|
||||
const resyncRequester = createAuctionResyncRequester(
|
||||
async () => {
|
||||
if (!socket.connected)
|
||||
throw new Error('Auction socket is disconnected.');
|
||||
return (await socket
|
||||
.timeout(3000)
|
||||
.emitWithAck('resync', {})) as AuctionResyncReply;
|
||||
},
|
||||
(state) => {
|
||||
receiveAuctionState(state, 'socket-resync');
|
||||
},
|
||||
);
|
||||
resyncRequesterRef.current = resyncRequester;
|
||||
|
||||
socket.on('connect', () => {
|
||||
setConnectionState({
|
||||
@@ -437,6 +506,7 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
});
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
resyncRequester.cancel();
|
||||
clearTimeSyncTimers();
|
||||
setConnectionState({
|
||||
key: activeConnectionKey,
|
||||
@@ -444,18 +514,12 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
});
|
||||
});
|
||||
|
||||
socket.on('state-pending', () => {
|
||||
void resyncRequester.request();
|
||||
});
|
||||
|
||||
socket.on('state', (nextAuction: ApiAuction) => {
|
||||
const didApplyState = commitAuctionState(nextAuction, 'socket-state');
|
||||
if (didApplyState) {
|
||||
clearPendingActionRequests();
|
||||
}
|
||||
applyFallbackServerTimestamp(nextAuction.currentServerTimestampMS);
|
||||
if (nextAuction.chatMessages) {
|
||||
setChatMessages(mergeChatMessages([], nextAuction.chatMessages));
|
||||
}
|
||||
if (didApplyState) {
|
||||
setError(null);
|
||||
}
|
||||
receiveAuctionState(nextAuction, 'socket-state');
|
||||
});
|
||||
|
||||
socket.on('chat-history', (payload: ChatHistoryPayload) => {
|
||||
@@ -509,6 +573,13 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
|
||||
const currentAuction = latestAuctionRef.current;
|
||||
if (!currentAuction) return;
|
||||
if (
|
||||
payload.auctionStateTransitionId &&
|
||||
currentAuction.matchSideEffectTransitionId &&
|
||||
payload.auctionStateTransitionId !==
|
||||
currentAuction.matchSideEffectTransitionId
|
||||
)
|
||||
return;
|
||||
if (
|
||||
payload.auctionLifecycleId &&
|
||||
currentAuction.auctionLifecycleId &&
|
||||
@@ -540,6 +611,18 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
|
||||
socket.on('reset', (payload: ResetPayload) => {
|
||||
if (payload.matchId && payload.matchId !== matchId) return;
|
||||
const lifecycle =
|
||||
payload.auctionLifecycleId ??
|
||||
latestAuctionRef.current?.auctionLifecycleId;
|
||||
if (lifecycle) cancelledLifecycleIdsRef.current.add(lifecycle);
|
||||
if (
|
||||
payload.auctionLifecycleId &&
|
||||
latestAuctionRef.current?.auctionLifecycleId &&
|
||||
payload.auctionLifecycleId !==
|
||||
latestAuctionRef.current.auctionLifecycleId
|
||||
)
|
||||
return;
|
||||
resyncRequester.cancel();
|
||||
latestAuctionRef.current = null;
|
||||
latestConfirmedBidPointAmountRef.current = 0;
|
||||
setAuction(null);
|
||||
@@ -556,6 +639,8 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
resyncRequesterRef.current?.cancel();
|
||||
resyncRequesterRef.current = null;
|
||||
const socket = socketRef.current;
|
||||
socket?.removeAllListeners();
|
||||
socket?.disconnect();
|
||||
@@ -570,6 +655,7 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
pendingBidTimeoutsRef.current = {};
|
||||
pendingActionTimeoutsRef.current = {};
|
||||
latestAuctionRef.current = null;
|
||||
cancelledLifecycleIds.clear();
|
||||
latestConfirmedBidPointAmountRef.current = 0;
|
||||
hasSyncedServerTimeRef.current = false;
|
||||
setPendingBidRequests({});
|
||||
@@ -621,12 +707,11 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
}
|
||||
|
||||
// 거부 등으로 클라 상태가 낡았을 때 서버 최신 state를 다시 받아 rebaseline한다.
|
||||
// join 핸들러가 getStateByMatchId → broadcastState로 fresh state를 내려주고,
|
||||
// revision 가드상 더 높은 revision이라 정상 적용된다(force-resync 아님).
|
||||
// 요청한 소켓만 최신 상태를 받으며 진행 중인 동기화 요청은 합친다.
|
||||
function resyncAuctionState() {
|
||||
const socket = socketRef.current;
|
||||
if (!socket || !socket.connected || !matchId) return;
|
||||
socket.emit('join', { matchId });
|
||||
void resyncRequesterRef.current?.request();
|
||||
}
|
||||
|
||||
// retryAttempt: 거부 후 자동 재시도 횟수(루프 방지를 위해 1회로 제한).
|
||||
@@ -655,10 +740,13 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
...current,
|
||||
[auctionActionRequestId]: { action: eventName },
|
||||
}));
|
||||
pendingActionTimeoutsRef.current[auctionActionRequestId] = setTimeout(() => {
|
||||
clearPendingActionRequest(auctionActionRequestId);
|
||||
setError('경매 진행 응답이 지연되어 최신 상태를 다시 확인해주세요.');
|
||||
}, 10_000);
|
||||
pendingActionTimeoutsRef.current[auctionActionRequestId] = setTimeout(
|
||||
() => {
|
||||
clearPendingActionRequest(auctionActionRequestId);
|
||||
setError('경매 진행 응답이 지연되어 최신 상태를 다시 확인해주세요.');
|
||||
},
|
||||
10_000,
|
||||
);
|
||||
addNewRelicPageAction('AuctionClientActionSent', {
|
||||
matchId: matchId ?? 'none',
|
||||
action: eventName,
|
||||
@@ -731,7 +819,10 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
sentPlayerUserId:
|
||||
payload.clientObservedCurrentBidRoundPlayerUserId ?? 'none',
|
||||
sentPhase: payload.clientObservedPhase ?? 'none',
|
||||
...describeAuctionForTelemetry('observed', latestAuctionRef.current),
|
||||
...describeAuctionForTelemetry(
|
||||
'observed',
|
||||
latestAuctionRef.current,
|
||||
),
|
||||
...describeAuctionForTelemetry('server', ack?.state ?? null),
|
||||
});
|
||||
if (canAutoRetry) {
|
||||
@@ -772,7 +863,9 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
|
||||
const currentAuction = auction;
|
||||
const visibleAuction =
|
||||
currentAuction && currentAuction.matchId === matchId ? currentAuction : null;
|
||||
currentAuction && currentAuction.matchId === matchId
|
||||
? currentAuction
|
||||
: null;
|
||||
const visibleConnectionStatus: ConnectionStatus =
|
||||
connectionKey === null
|
||||
? 'idle'
|
||||
@@ -797,6 +890,7 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
bidLatencyDiagnostics,
|
||||
serverRoundTripMs: matchId ? serverRoundTripMs : null,
|
||||
serverTimeOffsetMs: matchId ? serverTimeOffsetMs : 0,
|
||||
syncPolledAuctionState,
|
||||
placeBid(payload: BidPayload) {
|
||||
const socket = socketRef.current;
|
||||
if (!socket || !socket.connected) {
|
||||
@@ -807,6 +901,12 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
return;
|
||||
}
|
||||
if (payload.bidPointAmount <= latestConfirmedBidPointAmountRef.current) {
|
||||
// 조용히 무시하면 화면과 내부 기준가가 어긋났을 때 입찰 버튼이 반응하지
|
||||
// 않는 것처럼 보인다. 안내하고 최신 상태를 다시 받는다.
|
||||
setError(
|
||||
'현재 최고 입찰가보다 높은 금액만 입찰할 수 있어요. 최신 경매 상태를 다시 불러올게요.',
|
||||
);
|
||||
resyncAuctionState();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -815,6 +915,8 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
const bidPayload: BidPayload = {
|
||||
...payload,
|
||||
bidRequestId,
|
||||
clientObservedBidRoundId:
|
||||
payload.clientObservedBidRoundId ?? currentAuction?.currentBidRoundId,
|
||||
clientObservedAuctionLifecycleId:
|
||||
payload.clientObservedAuctionLifecycleId ??
|
||||
currentAuction?.auctionLifecycleId,
|
||||
@@ -835,36 +937,32 @@ export function useAuctionSocket(matchId: string | undefined) {
|
||||
clearPendingBidRequest(bidRequestId);
|
||||
setError('입찰 응답이 지연되어 최신 상태를 다시 확인해주세요.');
|
||||
}, 10_000);
|
||||
socket.emit(
|
||||
'bid',
|
||||
bidPayload,
|
||||
(ack?: BidAckPayload) => {
|
||||
captureBidLatencyDiagnostics(ack);
|
||||
if (ack?.rejected) {
|
||||
clearPendingBidRequest(bidRequestId);
|
||||
// 입찰 거부도 최신 state로 rebaseline(예: 0초 입찰 EXPIRED → 만료된
|
||||
// 라운드 화면에서 벗어나게). 단 입찰은 자동 재시도하지 않는다(의도치
|
||||
// 않은 금액 재입찰 방지).
|
||||
if (ack.state) {
|
||||
commitAuctionState(ack.state);
|
||||
} else if (isAuctionResyncRecommended(ack)) {
|
||||
resyncAuctionState();
|
||||
}
|
||||
setError(resolveAuctionRejectionMessage(ack));
|
||||
return;
|
||||
socket.emit('bid', bidPayload, (ack?: BidAckPayload) => {
|
||||
captureBidLatencyDiagnostics(ack);
|
||||
if (ack?.rejected) {
|
||||
clearPendingBidRequest(bidRequestId);
|
||||
// 입찰 거부도 최신 state로 rebaseline(예: 0초 입찰 EXPIRED → 만료된
|
||||
// 라운드 화면에서 벗어나게). 단 입찰은 자동 재시도하지 않는다(의도치
|
||||
// 않은 금액 재입찰 방지).
|
||||
if (ack.state) {
|
||||
commitAuctionState(ack.state);
|
||||
} else if (isAuctionResyncRecommended(ack)) {
|
||||
resyncAuctionState();
|
||||
}
|
||||
if (ack?.bidDelta) {
|
||||
clearPendingBidRequest(bidRequestId);
|
||||
commitBidDelta(ack.bidDelta);
|
||||
return;
|
||||
}
|
||||
const ackState = ack?.state;
|
||||
if (ackState) {
|
||||
clearPendingBidRequest(bidRequestId);
|
||||
commitAuctionState(ackState);
|
||||
}
|
||||
},
|
||||
);
|
||||
setError(resolveAuctionRejectionMessage(ack));
|
||||
return;
|
||||
}
|
||||
if (ack?.bidDelta) {
|
||||
clearPendingBidRequest(bidRequestId);
|
||||
commitBidDelta(ack.bidDelta);
|
||||
return;
|
||||
}
|
||||
const ackState = ack?.state;
|
||||
if (ackState) {
|
||||
clearPendingBidRequest(bidRequestId);
|
||||
commitAuctionState(ackState);
|
||||
}
|
||||
});
|
||||
},
|
||||
openBidding() {
|
||||
emitAuctionAction('open-bidding', buildAuctionActionPayload());
|
||||
@@ -905,7 +1003,10 @@ function resolveCurrentBidPointAmount(auction: ApiAuction): number {
|
||||
}
|
||||
|
||||
// [AUCTION-DEBUG] 아래 텔레메트리 헬퍼들과 호출부는 조사 종료 후 제거 대상.
|
||||
type AuctionTelemetryAttributes = Record<string, string | number | boolean | null>;
|
||||
type AuctionTelemetryAttributes = Record<
|
||||
string,
|
||||
string | number | boolean | null
|
||||
>;
|
||||
|
||||
function readAuctionHistorySequence(auction: ApiAuction | null | undefined) {
|
||||
const value = (auction as { currentAuctionHistorySequence?: unknown } | null)
|
||||
@@ -914,7 +1015,9 @@ function readAuctionHistorySequence(auction: ApiAuction | null | undefined) {
|
||||
}
|
||||
|
||||
function summarizeAuctionTeams(auction: ApiAuction | null | undefined) {
|
||||
const teams = Array.isArray(auction?.auctionTeams) ? auction.auctionTeams : [];
|
||||
const teams = Array.isArray(auction?.auctionTeams)
|
||||
? auction.auctionTeams
|
||||
: [];
|
||||
let assignedMemberCount = 0;
|
||||
for (const team of teams) {
|
||||
const members = (team as { members?: unknown[] }).members;
|
||||
@@ -956,143 +1059,3 @@ function describeAuctionForTelemetry(
|
||||
[`${prefix}ForceResync`]: Boolean(auction?.shouldForceAuctionStateResync),
|
||||
};
|
||||
}
|
||||
|
||||
function shouldApplyIncomingAuctionState(
|
||||
current: ApiAuction | null,
|
||||
incoming: ApiAuction,
|
||||
): boolean {
|
||||
if (!current) return true;
|
||||
if (incoming.shouldForceAuctionStateResync) return true;
|
||||
if (
|
||||
current.auctionLifecycleId &&
|
||||
incoming.auctionLifecycleId &&
|
||||
current.auctionLifecycleId !== incoming.auctionLifecycleId
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
typeof current.revision === 'number' &&
|
||||
typeof incoming.revision === 'number' &&
|
||||
incoming.revision <= current.revision
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function applyBidDelta(
|
||||
current: ApiAuction | null,
|
||||
payload: BidDeltaPayload,
|
||||
): ApiAuction | null {
|
||||
if (!current) return current;
|
||||
if (
|
||||
payload.auctionLifecycleId &&
|
||||
current.auctionLifecycleId &&
|
||||
payload.auctionLifecycleId !== current.auctionLifecycleId
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
if (
|
||||
typeof current.revision === 'number' &&
|
||||
typeof payload.revision === 'number' &&
|
||||
payload.revision <= current.revision
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
|
||||
const bidPointAmount = Number(payload.currentHighestBidPointAmount);
|
||||
const bidderTeamIndex = Number(payload.currentHighestBidderTeamIndex);
|
||||
if (!Number.isFinite(bidPointAmount) || !Number.isFinite(bidderTeamIndex)) {
|
||||
return current;
|
||||
}
|
||||
|
||||
const bidHistoryEntry = normalizeBidHistoryEntry(payload.auctionBidHistoryEntry);
|
||||
const currentBidRoundDeadlineTimestampMS =
|
||||
typeof payload.currentBidRoundDeadlineTimestampMS === 'number'
|
||||
? payload.currentBidRoundDeadlineTimestampMS
|
||||
: current.currentBidRoundDeadlineTimestampMS;
|
||||
|
||||
return {
|
||||
...current,
|
||||
auctionBidHistory: appendUniqueAuctionBidHistoryEntry(
|
||||
current.auctionBidHistory ?? [],
|
||||
bidHistoryEntry,
|
||||
),
|
||||
bids: bidHistoryEntry
|
||||
? appendUniqueLegacyBidEntry(current.bids ?? [], bidHistoryEntry)
|
||||
: current.bids,
|
||||
currentBid: bidPointAmount,
|
||||
currentBidderTeamIndex: bidderTeamIndex,
|
||||
currentDeadline: currentBidRoundDeadlineTimestampMS
|
||||
? new Date(currentBidRoundDeadlineTimestampMS).toISOString()
|
||||
: current.currentDeadline,
|
||||
currentHighestBidPointAmount: bidPointAmount,
|
||||
currentHighestBidderTeamIndex: bidderTeamIndex,
|
||||
currentBidRoundDeadlineTimestampMS,
|
||||
currentServerTimestampMS:
|
||||
payload.currentServerTimestampMS ?? current.currentServerTimestampMS,
|
||||
revision: payload.revision ?? current.revision,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBidHistoryEntry(
|
||||
entry: BidDeltaPayload['auctionBidHistoryEntry'],
|
||||
): NonNullable<ApiAuction['auctionBidHistory']>[number] | null {
|
||||
if (!entry) return null;
|
||||
const bidderTeamIndex = Number(entry.bidderTeamIndex);
|
||||
const bidPointAmount = Number(entry.bidPointAmount);
|
||||
const bidPlacedAtTimestampMS = Number(entry.bidPlacedAtTimestampMS);
|
||||
if (!Number.isFinite(bidderTeamIndex) || !Number.isFinite(bidPointAmount)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
bidderTeamIndex,
|
||||
bidPointAmount,
|
||||
bidderUserId: entry.bidderUserId ?? '',
|
||||
bidPlacedAtTimestampMS: Number.isFinite(bidPlacedAtTimestampMS)
|
||||
? bidPlacedAtTimestampMS
|
||||
: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function appendUniqueAuctionBidHistoryEntry(
|
||||
current: NonNullable<ApiAuction['auctionBidHistory']>,
|
||||
entry: NonNullable<ApiAuction['auctionBidHistory']>[number] | null,
|
||||
) {
|
||||
if (!entry) return current;
|
||||
if (
|
||||
current.some(
|
||||
(existing) =>
|
||||
existing.bidderTeamIndex === entry.bidderTeamIndex &&
|
||||
existing.bidPointAmount === entry.bidPointAmount &&
|
||||
existing.bidPlacedAtTimestampMS === entry.bidPlacedAtTimestampMS,
|
||||
)
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
return [...current, entry];
|
||||
}
|
||||
|
||||
function appendUniqueLegacyBidEntry(
|
||||
current: ApiAuction['bids'],
|
||||
entry: NonNullable<ApiAuction['auctionBidHistory']>[number],
|
||||
) {
|
||||
const currentBids = current ?? [];
|
||||
const legacyBid = {
|
||||
amount: entry.bidPointAmount,
|
||||
bidderUserId: entry.bidderUserId,
|
||||
teamIndex: entry.bidderTeamIndex,
|
||||
timestamp: new Date(entry.bidPlacedAtTimestampMS).toISOString(),
|
||||
};
|
||||
if (
|
||||
currentBids.some(
|
||||
(existing) =>
|
||||
existing.teamIndex === legacyBid.teamIndex &&
|
||||
existing.amount === legacyBid.amount &&
|
||||
existing.timestamp === legacyBid.timestamp,
|
||||
)
|
||||
) {
|
||||
return currentBids;
|
||||
}
|
||||
return [...currentBids, legacyBid];
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
useDevCaptainTeamIndex,
|
||||
} from "../config";
|
||||
import { ApiError } from "../lib/api";
|
||||
import { mergeAuctionState } from "../api/auction-state";
|
||||
import {
|
||||
championDisplayName,
|
||||
championIconUrl,
|
||||
@@ -152,7 +153,10 @@ function pickLatestAuction(
|
||||
const socketRevision = getAuctionRevision(socketAuction);
|
||||
const queryRevision = getAuctionRevision(queryAuction);
|
||||
if (socketRevision !== null && queryRevision !== null) {
|
||||
return queryRevision > socketRevision ? queryAuction : socketAuction;
|
||||
// Polling can repair a partial socket state at the same revision.
|
||||
return queryRevision >= socketRevision
|
||||
? mergeAuctionState(socketAuction, queryAuction)
|
||||
: mergeAuctionState(queryAuction, socketAuction);
|
||||
}
|
||||
}
|
||||
return getAuctionUpdatedAt(queryAuction) > getAuctionUpdatedAt(socketAuction)
|
||||
@@ -1901,6 +1905,10 @@ export default function AuctionPage() {
|
||||
cancelConfirmInputRef.current?.focus();
|
||||
}, [showCancelModal]);
|
||||
|
||||
const { syncPolledAuctionState } = socket;
|
||||
useEffect(() => {
|
||||
syncPolledAuctionState(auctionQuery.data);
|
||||
}, [auctionQuery.data, syncPolledAuctionState]);
|
||||
const liveAuction = pickLatestAuction(socket.auction, auctionQuery.data);
|
||||
const auctionState = useMemo(
|
||||
() =>
|
||||
@@ -2517,7 +2525,7 @@ export default function AuctionPage() {
|
||||
</h2>
|
||||
<p className="mt-2 text-sm leading-6 text-base-content/60">
|
||||
정말 경매를 취소하고 모집 상태로 되돌리시겠습니까? 경매
|
||||
데이터, 입찰 기록, 팀 배정 정보가 삭제됩니다.
|
||||
진행 데이터와 팀 배정 정보가 초기화됩니다.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -7,6 +7,26 @@ import {
|
||||
|
||||
const rosterMessage = "팀 배정 상태를 확인해야 해 경매를 완료하지 못했습니다.";
|
||||
|
||||
test("asks an outdated client to reload instead of claiming the player advanced", () => {
|
||||
const message =
|
||||
"경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.";
|
||||
assert.equal(
|
||||
resolveAuctionRejectionMessage({
|
||||
auctionRejectionReason: "BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED",
|
||||
message,
|
||||
}),
|
||||
message,
|
||||
);
|
||||
// An older bundle without the new reason mapping still displays the server message.
|
||||
assert.equal(
|
||||
resolveAuctionRejectionMessage({
|
||||
auctionRejectionReason: "UNRECOGNIZED_NEW_CLIENT_REASON",
|
||||
message,
|
||||
}),
|
||||
message,
|
||||
);
|
||||
});
|
||||
|
||||
test("shows the common error message when roster validation blocks completion", () => {
|
||||
assert.equal(
|
||||
resolveAuctionRejectionMessage({
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
createAuctionResyncRequester,
|
||||
type AuctionResyncReply,
|
||||
} from '../src/api/auction-resync.ts';
|
||||
import type { ApiAuction } from '../src/api/types.ts';
|
||||
|
||||
const state = (revision: number) =>
|
||||
({ revision, auctionLifecycleId: 'life', phase: 'bidding' }) as ApiAuction;
|
||||
const noWait = async () => {};
|
||||
|
||||
test('coalesces simultaneous gaps into one snapshot request', async () => {
|
||||
let finish!: (reply: AuctionResyncReply) => void;
|
||||
let calls = 0;
|
||||
const received: number[] = [];
|
||||
const sync = createAuctionResyncRequester(
|
||||
() => {
|
||||
calls++;
|
||||
return new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
},
|
||||
(value) => received.push(value.revision!),
|
||||
noWait,
|
||||
);
|
||||
const requests = Array.from({ length: 100 }, () => sync.request(12, 'life'));
|
||||
assert.equal(calls, 1);
|
||||
finish({ ok: true, state: state(12) });
|
||||
await Promise.all(requests);
|
||||
assert.deepEqual(received, [12]);
|
||||
});
|
||||
|
||||
test('fetches again when a newer gap arrived while the snapshot was in flight', async () => {
|
||||
let finish!: (reply: AuctionResyncReply) => void;
|
||||
let calls = 0;
|
||||
const received: number[] = [];
|
||||
const sync = createAuctionResyncRequester(
|
||||
() =>
|
||||
++calls === 1
|
||||
? new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
})
|
||||
: Promise.resolve({ ok: true, state: state(14) }),
|
||||
(value) => received.push(value.revision!),
|
||||
noWait,
|
||||
);
|
||||
const pending = sync.request(12, 'life');
|
||||
void sync.request(14, 'life');
|
||||
void sync.request(); // A generic refresh cannot lower the required revision.
|
||||
finish({ ok: true, state: state(12) });
|
||||
await pending;
|
||||
assert.deepEqual(received, [12, 14]);
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test('bounds retries and honors retry delay without rejoining the room', async () => {
|
||||
let calls = 0;
|
||||
const waits: number[] = [];
|
||||
const sync = createAuctionResyncRequester(
|
||||
async () => {
|
||||
calls++;
|
||||
return { ok: false, retryAfterMS: 1500 };
|
||||
},
|
||||
() => assert.fail('must not apply'),
|
||||
async (ms) => {
|
||||
waits.push(ms);
|
||||
},
|
||||
);
|
||||
await sync.request();
|
||||
assert.equal(calls, 3);
|
||||
assert.deepEqual(waits, [1500, 1500]);
|
||||
});
|
||||
|
||||
test('drops an old connection reply after cancellation and permits a new request', async () => {
|
||||
let finish!: (reply: AuctionResyncReply) => void;
|
||||
let calls = 0;
|
||||
const received: number[] = [];
|
||||
const sync = createAuctionResyncRequester(
|
||||
() =>
|
||||
++calls === 1
|
||||
? new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
})
|
||||
: Promise.resolve({ ok: true, state: state(20) }),
|
||||
(value) => received.push(value.revision!),
|
||||
noWait,
|
||||
);
|
||||
const stale = sync.request();
|
||||
sync.cancel();
|
||||
await sync.request();
|
||||
finish({ ok: true, state: state(10) });
|
||||
await stale;
|
||||
assert.deepEqual(received, [20]);
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
applyBidDelta,
|
||||
mergeAuctionState,
|
||||
shouldApplyIncomingAuctionState,
|
||||
shouldApplyPolledAuctionState,
|
||||
shouldResyncForBidDelta,
|
||||
} from "../src/api/auction-state.ts";
|
||||
import type { ApiAuction } from "../src/api/types.ts";
|
||||
|
||||
const current = {
|
||||
auctionLifecycleId: "life",
|
||||
revision: 10,
|
||||
phase: "bidding",
|
||||
currentBidRoundId: "round-A",
|
||||
currentBidRoundPlayerUserId: "A",
|
||||
currentHighestBidPointAmount: 10,
|
||||
auctionBidHistory: [],
|
||||
bids: [],
|
||||
} as unknown as ApiAuction;
|
||||
const bid = {
|
||||
auctionLifecycleId: "life",
|
||||
revision: 11,
|
||||
currentBidRoundId: "round-A",
|
||||
currentHighestBidPointAmount: 20,
|
||||
currentHighestBidderTeamIndex: 1,
|
||||
auctionBidHistoryEntry: {
|
||||
bidPointAmount: 20,
|
||||
bidderTeamIndex: 1,
|
||||
bidderUserId: "captain",
|
||||
bidPlacedAtTimestampMS: 1000,
|
||||
playerUserId: "A",
|
||||
bidRoundId: "round-A",
|
||||
},
|
||||
};
|
||||
|
||||
test("a contiguous same-round bid updates price and preserves player identity", () => {
|
||||
const next = applyBidDelta(current, bid)!;
|
||||
assert.equal(shouldResyncForBidDelta(current, bid), false);
|
||||
assert.equal(next.currentHighestBidPointAmount, 20);
|
||||
assert.equal(next.currentBidRoundPlayerUserId, "A");
|
||||
assert.equal(next.auctionBidHistory?.[0].bidRoundId, "round-A");
|
||||
});
|
||||
|
||||
test("a revision gap resyncs instead of attaching a new player bid to the old structure", () => {
|
||||
const missedTransition = {
|
||||
...bid,
|
||||
revision: 13,
|
||||
currentBidRoundId: "round-B",
|
||||
};
|
||||
assert.equal(shouldResyncForBidDelta(current, missedTransition), true);
|
||||
assert.equal(applyBidDelta(current, missedTransition), current);
|
||||
});
|
||||
|
||||
test("a delta cannot open bidding or change the round by itself", () => {
|
||||
assert.equal(
|
||||
shouldResyncForBidDelta({ ...current, phase: "presenting" }, bid),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldResyncForBidDelta(current, { ...bid, currentBidRoundId: "round-B" }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("a full snapshot can repair the same revision but cannot regress auction revision", () => {
|
||||
assert.equal(
|
||||
shouldApplyIncomingAuctionState(current, {
|
||||
...current,
|
||||
currentBidRoundPlayerUserId: "B",
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldApplyIncomingAuctionState(current, { ...current, revision: 9 }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("full state repair preserves a newer independently versioned side effect", () => {
|
||||
const live = {
|
||||
...current,
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: "completed" as const,
|
||||
};
|
||||
const incoming = {
|
||||
...current,
|
||||
matchSideEffectRevision: 1,
|
||||
matchSideEffectStatus: "pending" as const,
|
||||
};
|
||||
assert.equal(
|
||||
mergeAuctionState(live, incoming).matchSideEffectStatus,
|
||||
"completed",
|
||||
);
|
||||
assert.equal(
|
||||
mergeAuctionState(live, { ...incoming, auctionLifecycleId: "new-life" })
|
||||
.matchSideEffectStatus,
|
||||
"pending",
|
||||
);
|
||||
});
|
||||
|
||||
test("replayed old bids do not resync or change the current state", () => {
|
||||
const replay = { ...bid, revision: 9 };
|
||||
assert.equal(shouldResyncForBidDelta(current, replay), false);
|
||||
assert.equal(applyBidDelta(current, replay), current);
|
||||
});
|
||||
|
||||
test("polled state repairs a socket that missed a transition", () => {
|
||||
const polled = {
|
||||
...current,
|
||||
matchId: "match",
|
||||
revision: 12,
|
||||
currentBidRoundId: "round-B",
|
||||
currentBidRoundPlayerUserId: "B",
|
||||
currentHighestBidPointAmount: 0,
|
||||
} as unknown as ApiAuction;
|
||||
const socketState = { ...current, matchId: "match" } as ApiAuction;
|
||||
assert.equal(
|
||||
shouldApplyPolledAuctionState(socketState, polled, "match"),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("polled state never regresses or replaces the socket lifecycle", () => {
|
||||
const socketState = { ...current, matchId: "match" } as ApiAuction;
|
||||
const same = { ...socketState } as ApiAuction;
|
||||
const older = { ...socketState, revision: 9 } as ApiAuction;
|
||||
const otherLifecycle = {
|
||||
...socketState,
|
||||
auctionLifecycleId: "old-life",
|
||||
revision: 99,
|
||||
} as ApiAuction;
|
||||
const otherMatch = { ...socketState, matchId: "other", revision: 11 };
|
||||
assert.equal(shouldApplyPolledAuctionState(socketState, same, "match"), true);
|
||||
assert.equal(
|
||||
shouldApplyPolledAuctionState(socketState, older, "match"),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldApplyPolledAuctionState(socketState, otherLifecycle, "match"),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldApplyPolledAuctionState(socketState, otherMatch, "match"),
|
||||
false,
|
||||
);
|
||||
assert.equal(shouldApplyPolledAuctionState(null, same, "match"), true);
|
||||
});
|
||||
|
||||
test("polling initializes a missing socket baseline but rejects another match or missing identity", () => {
|
||||
const incoming = {
|
||||
...current,
|
||||
matchId: "match",
|
||||
currentBidRoundId: "round-B",
|
||||
currentHighestBidPointAmount: 0,
|
||||
} as ApiAuction;
|
||||
assert.equal(shouldApplyPolledAuctionState(null, incoming, "match"), true);
|
||||
assert.equal(
|
||||
shouldApplyPolledAuctionState(null, incoming, "another-match"),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldApplyPolledAuctionState(
|
||||
null,
|
||||
{ ...incoming, auctionLifecycleId: undefined },
|
||||
"match",
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("a prior side-effect generation cannot replace the status of a new transition", () => {
|
||||
const old = {
|
||||
...current,
|
||||
matchSideEffectTransitionId: "old",
|
||||
matchSideEffectRevision: 100,
|
||||
matchSideEffectStatus: "failed" as const,
|
||||
};
|
||||
const next = {
|
||||
...current,
|
||||
revision: 12,
|
||||
matchSideEffectTransitionId: "new",
|
||||
matchSideEffectRevision: 3,
|
||||
matchSideEffectStatus: "pending" as const,
|
||||
};
|
||||
assert.equal(mergeAuctionState(old, next).matchSideEffectStatus, "pending");
|
||||
});
|
||||
Reference in new issue
Block a user