diff --git a/apps/api/README.md b/apps/api/README.md index 1b15cc36..4256b95e 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -62,8 +62,61 @@ $ pnpm run test:e2e # test coverage $ pnpm run test:cov + +# auction cancellation/undo transactions against a real local MongoDB replica set +$ pnpm run test:mongodb ``` +`test:mongodb` requires `mongod` on PATH, or `MONGODB_TEST_SERVER` set to its +executable path. The runner starts an isolated replica set on a random loopback +port, uses a temporary database directory, and removes it after the tests. It +does not use the application's database connection settings. + +## Auction durability + +Accepted Redis bids now enter `auction:bidArchivePayloads` and +`auction:bidArchiveQueue` atomically with the bid. The durability scheduler saves +them to `auctionBidRecords` using deterministic IDs before removing queue entries. +Recent state snapshots remain capped at 1000 bids; the archive is not capped and +retains accepted bids across undo/cancellation for audit. These records begin with +this deployment; bids already discarded by an older version cannot be recovered. +`GET /communities/:communityId/matches/:matchId/auction/bids/history` pages through +the current lifecycle's persisted archive using `afterRevision` and `limit` (1–200). +Persistence is asynchronous, so an empty last page may gain entries afterward. + +Invalid archive entries are moved atomically to `auction:bidArchiveDeadLetterQueue` +and `auction:bidArchiveDeadLetterPayloads`. The payload stores the original text +(or `null` for a missing payload), error, ID, and quarantine time. Monitor these +keys and review quarantined data before repairing/requeuing it; it is not retried +automatically or expired. Other bids continue to persist. Mongo write failures +leave valid entries in the pending queue for idempotent retries. + +Older in-flight rounds can produce `bidRoundId: ''`; the archive treats this as +an absent optional round ID. Empty player IDs and other invalid field types +remain validation errors. + +Cancellation commits its completion marker and pending cleanup fields in Match. +The scheduler retries cleanup after restarts, with a delay capped at 60 seconds. +Reset notifications include the lifecycle to protect a newly started auction. +Dead-letter status notifications use `auction:deadLetterStatusQueue`; failures to +persist or mirror the terminal status are retried without rerunning the side effect. +Older side-effect dead letters are also reconciled automatically. + +Before deploying the cleanup worker, create its partial Match index using the +additive, repeatable migration below. Schema declarations alone do not create +indexes when `MONGODB_AUTO_INDEX` is disabled (the default). Choose the target +environment explicitly; dry-run does not create indexes or modify documents. + +```bash +pnpm --filter api auction:indexes -- --env staging --dry-run +pnpm --filter api auction:indexes -- --env staging --apply +# Use --env prod for production. +``` + +The index is `{ auctionCancellationCleanupRetryAt: 1 }`, restricted to +`{ auctionCancellationCleanupPending: true }`. The worker only reads due pending +entries; it no longer scans legacy cancellation markers without a pending flag. + ## Deployment When you're ready to deploy your NestJS application to production, there are some key steps you can take to ensure it runs as efficiently as possible. Check out the [deployment documentation](https://docs.nestjs.com/deployment) for more information. diff --git a/apps/api/package.json b/apps/api/package.json index fde8950e..d6936101 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -9,6 +9,7 @@ "build": "nest build", "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", "competition:migrate": "ts-node src/modules/global/database/migrations/run-competition-domain.ts", + "auction:indexes": "ts-node src/modules/global/database/migrations/run-auction-cancellation-index.ts", "seed:dev": "ts-node src/seed.ts", "seed:dakao-participant": "ts-node src/seed-dakao-participant.ts", "start:dev": "NODE_ENV=dev NEW_RELIC_HOME=. node --import newrelic/esm-loader.mjs -r newrelic dist/main.js", @@ -24,6 +25,7 @@ "test": "jest", "typecheck": "tsc --noEmit --incremental false", "test:redis": "node test/run-redis-tests.mjs", + "test:mongodb": "node test/run-mongo-tests.mjs", "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", diff --git a/apps/api/src/modules/auction/auction-bid-archive.service.spec.ts b/apps/api/src/modules/auction/auction-bid-archive.service.spec.ts new file mode 100644 index 00000000..958b2bcd --- /dev/null +++ b/apps/api/src/modules/auction/auction-bid-archive.service.spec.ts @@ -0,0 +1,82 @@ +import mongoose from 'mongoose'; +import { AuctionBidArchiveService } from './auction-bid-archive.service'; + +const matchId = new mongoose.Types.ObjectId().toString(); + +function bid(overrides: Record = {}) { + return { + _id: `${matchId}:life:${String(1).padStart(16, '0')}`, + matchId, + auctionLifecycleId: 'life', + revision: 1, + bidRequestId: 'bid-1', + bidderUserId: new mongoose.Types.ObjectId().toString(), + playerUserId: new mongoose.Types.ObjectId().toString(), + bidderTeamIndex: 1, + bidPointAmount: 10, + bidPlacedAtTimestampMS: 1_700_000_000_000, + ...overrides, + }; +} + +function fixture(payload: Record) { + const raw = JSON.stringify(payload); + const commands: string[][] = []; + const sendCommand = jest.fn(async (command: string[]) => { + commands.push(command); + if (command[0] === 'ZRANGE') return [payload._id]; + if (command[0] === 'HMGET') return [raw]; + return 1; + }); + const bulkWrite = jest.fn(async (operations: unknown[]) => ({ + matchedCount: 0, + upsertedCount: operations.length, + })); + const service = new AuctionBidArchiveService( + { getModel: async () => ({ bulkWrite }) } as never, + { raw: () => ({ sendCommand }) } as never, + ); + return { service, bulkWrite, commands }; +} + +function savedEvent(bulkWrite: jest.Mock) { + const [[operations]] = bulkWrite.mock.calls as unknown as [ + [ + Array<{ + updateOne: { update: { $setOnInsert: Record } }; + }>, + ], + ]; + return operations[0].updateOne.update.$setOnInsert; +} + +function quarantined(commands: string[][]) { + return commands.some( + (command) => + command[0] === 'EVAL' && + command[1].includes("redis.call('ZADD', KEYS[4]"), + ); +} + +it('archives a pre-upgrade bid with an empty round ID without the field', async () => { + const f = fixture(bid({ bidRoundId: '' })); + await expect(f.service.persistPendingBids()).resolves.toBe(1); + expect(quarantined(f.commands)).toBe(false); + expect(savedEvent(f.bulkWrite)).not.toHaveProperty('bidRoundId'); +}); + +it('keeps a non-empty round ID on the archived bid', async () => { + const f = fixture(bid({ bidRoundId: 'round-1' })); + await expect(f.service.persistPendingBids()).resolves.toBe(1); + expect(savedEvent(f.bulkWrite)).toMatchObject({ bidRoundId: 'round-1' }); +}); + +it.each([' ', null, 1])( + 'still quarantines an invalid round ID %j', + async (bidRoundId) => { + const f = fixture(bid({ bidRoundId })); + await expect(f.service.persistPendingBids()).resolves.toBe(0); + expect(f.bulkWrite).not.toHaveBeenCalled(); + expect(quarantined(f.commands)).toBe(true); + }, +); diff --git a/apps/api/src/modules/auction/auction-bid-archive.service.ts b/apps/api/src/modules/auction/auction-bid-archive.service.ts new file mode 100644 index 00000000..f0c53520 --- /dev/null +++ b/apps/api/src/modules/auction/auction-bid-archive.service.ts @@ -0,0 +1,242 @@ +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; +import mongoose from 'mongoose'; +import type { AuctionBidRecord } from '../global/database/schema'; +import { ConnectionService } from '../global/database/connection.service'; +import { AuctionRedisService } from './redis/auction-redis.service'; +import { + AUCTION_BID_ARCHIVE_PAYLOADS_KEY, + AUCTION_BID_ARCHIVE_QUEUE_KEY, + AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY, + AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY, +} from './redis/auction-redis.keys'; + +// Compare the payload as well as the ID: a concurrent repair must not be moved +// or acknowledged by a worker that read an older value. +const QUARANTINE_BID_ARCHIVE_SCRIPT = ` +local moved = 0 +for i = 2, #ARGV, 4 do + local current = redis.call('HGET', KEYS[1], ARGV[i]) + local matches = (ARGV[i+1] == '0' and current == false) or (ARGV[i+1] == '1' and current == ARGV[i+2]) + if matches and redis.call('ZSCORE', KEYS[2], ARGV[i]) then + redis.call('HSET', KEYS[3], ARGV[i], ARGV[i+3]) + redis.call('ZADD', KEYS[4], ARGV[1], ARGV[i]) + redis.call('HDEL', KEYS[1], ARGV[i]) + redis.call('ZREM', KEYS[2], ARGV[i]) + moved = moved + 1 + end +end +return moved +`; + +@Injectable() +export class AuctionBidArchiveService { + private readonly logger = new Logger(AuctionBidArchiveService.name); + + constructor( + private readonly database: ConnectionService, + private readonly redis: AuctionRedisService, + ) {} + + public async persistPendingBids(limit = 200): Promise { + const client = this.redis.raw(); + const reply = await client.sendCommand([ + 'ZRANGE', + AUCTION_BID_ARCHIVE_QUEUE_KEY, + '0', + String(limit - 1), + ]); + const ids = Array.isArray(reply) ? reply.map(String) : []; + if (!ids.length) return 0; + const values = await client.sendCommand>([ + 'HMGET', + AUCTION_BID_ARCHIVE_PAYLOADS_KEY, + ...ids, + ]); + const valid: Array<{ event: AuctionBidRecord; raw: string }> = []; + const invalid: Array<{ id: string; raw: string | null; error: string }> = + []; + for (let index = 0; index < ids.length; index++) { + const raw = values[index] ?? null; + try { + valid.push({ event: parseBidArchiveEvent(ids[index], raw), raw: raw! }); + } catch (error) { + invalid.push({ + id: ids[index], + raw, + error: String(error).slice(0, 2000), + }); + } + } + if (invalid.length) { + const now = Date.now(); + const moved = await client.sendCommand([ + 'EVAL', + QUARANTINE_BID_ARCHIVE_SCRIPT, + '4', + AUCTION_BID_ARCHIVE_PAYLOADS_KEY, + AUCTION_BID_ARCHIVE_QUEUE_KEY, + AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY, + AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY, + String(now), + ...invalid.flatMap(({ id, raw, error }) => [ + id, + raw === null ? '0' : '1', + raw ?? '', + JSON.stringify({ + id, + rawPayload: raw, + error, + quarantinedAtTimestampMS: now, + }), + ]), + ]); + if (moved > 0) + this.logger.warn(`Quarantined ${moved} invalid bid archive events.`); + } + if (valid.length) { + const model = await this.database.getModel('auctionBidRecord'); + // The deterministic _id is the dedupe key, including after a crash between + // Mongo acknowledgement and Redis cleanup. Never trim unpersisted events. + const result = await model.bulkWrite( + valid.map(({ event }) => ({ + updateOne: { + filter: { _id: event._id }, + update: { $setOnInsert: event }, + upsert: true, + }, + })), + { ordered: false, throwOnValidationError: true }, + ); + // A partial/ambiguous write leaves valid entries queued. Replaying the + // upserts is safe; assuming that a resolved bulkWrite saved all is not. + if (result.matchedCount + result.upsertedCount !== valid.length) { + throw new Error( + 'Bid archive batch was not fully acknowledged by MongoDB.', + ); + } + await client.sendCommand([ + 'EVAL', + "for i = 1, #ARGV, 2 do if redis.call('HGET', KEYS[1], ARGV[i]) == ARGV[i+1] then redis.call('HDEL', KEYS[1], ARGV[i]); redis.call('ZREM', KEYS[2], ARGV[i]) end end return 1", + '2', + AUCTION_BID_ARCHIVE_PAYLOADS_KEY, + AUCTION_BID_ARCHIVE_QUEUE_KEY, + ...valid.flatMap(({ event, raw }) => [event._id, raw]), + ]); + } + return valid.length; + } + + public async list( + matchId: mongoose.Types.ObjectId, + afterRevision = -1, + limit = 100, + ) { + if ( + !Number.isSafeInteger(afterRevision) || + afterRevision < -1 || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 200 + ) { + throw new BadRequestException('Invalid bid history cursor or limit.'); + } + const AuctionModel = await this.database.getModel('auction'); + const auction = await AuctionModel.findOne({ matchId }) + .select({ auctionLifecycleId: 1 }) + .lean() + .exec(); + if (!auction) throw new NotFoundException('Auction not found.'); + const prefix = `${matchId.toString()}:${auction.auctionLifecycleId}:`; + const model = await this.database.getModel('auctionBidRecord'); + // The built-in _id index supports ordered pagination without an additional + // production index or an unbounded array inside an Auction document. + const items = await model + .find({ + _id: { + $gt: + afterRevision < 0 + ? prefix + : prefix + String(afterRevision).padStart(16, '0'), + $lt: prefix + '\uffff', + }, + }) + .sort({ _id: 1 }) + .limit(limit + 1) + .lean() + .exec(); + const hasMore = items.length > limit; + if (hasMore) items.pop(); + return { + items, + hasMore, + nextRevision: items.at(-1)?.revision ?? afterRevision, + }; + } +} + +function parseBidArchiveEvent( + id: string, + raw: string | null, +): AuctionBidRecord { + if (raw === null) throw new Error('Missing bid archive payload.'); + const value: unknown = JSON.parse(raw); + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Bid archive payload must be an object.'); + } + const event = value as Record; + const string = (field: string): string => { + const value = event[field]; + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`Invalid bid archive ${field}.`); + } + return value; + }; + const integer = (field: string, minimum = 0): number => { + const value = event[field]; + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < minimum + ) { + throw new Error(`Invalid bid archive ${field}.`); + } + return value; + }; + const objectId = (field: string): mongoose.Types.ObjectId => { + const value = string(field); + if (!mongoose.isObjectIdOrHexString(value)) { + throw new Error(`Invalid bid archive ${field}.`); + } + return new mongoose.Types.ObjectId(value); + }; + const matchId = objectId('matchId'); + const auctionLifecycleId = string('auctionLifecycleId'); + const revision = integer('revision'); + const expectedId = `${matchId.toString()}:${auctionLifecycleId}:${String(revision).padStart(16, '0')}`; + if (event._id !== id || id !== expectedId) { + throw new Error('Bid archive identity does not match its payload.'); + } + return { + _id: id, + matchId, + auctionLifecycleId, + revision, + bidRequestId: string('bidRequestId'), + bidderUserId: objectId('bidderUserId'), + bidderTeamIndex: integer('bidderTeamIndex', 1), + bidPointAmount: integer('bidPointAmount'), + bidPlacedAtTimestampMS: integer('bidPlacedAtTimestampMS'), + ...(event.playerUserId !== undefined + ? { playerUserId: objectId('playerUserId') } + : {}), + // Pre-upgrade rounds have no Redis round ID; the bid Lua records ''. + ...(event.bidRoundId !== undefined && event.bidRoundId !== '' + ? { bidRoundId: string('bidRoundId') } + : {}), + }; +} diff --git a/apps/api/src/modules/auction/auction-cancellation.spec.ts b/apps/api/src/modules/auction/auction-cancellation.spec.ts new file mode 100644 index 00000000..44da7e76 --- /dev/null +++ b/apps/api/src/modules/auction/auction-cancellation.spec.ts @@ -0,0 +1,154 @@ +import { NotFoundException } from '@nestjs/common'; +import mongoose from 'mongoose'; +import { AuctionService } from './auction.service'; + +const matchId = new mongoose.Types.ObjectId(); +const communityId = new mongoose.Types.ObjectId(); + +function sessionQuery(value: T) { + return { session: async () => value }; +} + +function fixture(options: { + auction: Record | null; + match: Record; + failRedisCleanup?: boolean; +}) { + const session = { + startTransaction: jest.fn(), + commitTransaction: jest.fn().mockResolvedValue(undefined), + abortTransaction: jest.fn().mockResolvedValue(undefined), + endSession: jest.fn().mockResolvedValue(undefined), + }; + const deleteMany = jest.fn(() => sessionQuery(undefined)); + const models: Record = { + auction: { findOne: () => sessionQuery(options.auction) }, + auctionHistory: { deleteMany }, + auctionSideEffectApplication: { deleteMany }, + match: { findById: () => sessionQuery(options.match) }, + }; + const redisCommands: string[][] = []; + const sendCommand = jest.fn(async (command: string[]) => { + redisCommands.push(command); + if (command[0] === 'EVAL' && options.failRedisCleanup) + throw new Error('redis unavailable'); + return command[0] === 'SCAN' ? ['0', []] : 1; + }); + const service = new AuctionService( + { + getModel: async (name: string) => models[name], + createSession: async () => session, + } as never, + {} as never, + { releaseFromMatch: jest.fn().mockResolvedValue(undefined) } as never, + { cancelPolling: jest.fn().mockResolvedValue(undefined) } as never, + { + acquireTokenLock: async () => true, + releaseTokenLock: async () => true, + raw: () => ({ sendCommand }), + } as never, + {} as never, + {} as never, + ); + return { service, session, redisCommands }; +} + +function liveMatch(): Record { + return { + communityId, + status: 'team-built', + teams: [], + tournamentCodeIds: [], + gameSchedule: [], + participants: [], + save: jest.fn().mockResolvedValue(undefined), + }; +} + +function redisAuction() { + return { + _id: new mongoose.Types.ObjectId(), + auctionEngineVersion: 'redis-v1', + auctionLifecycleId: 'life', + deleteOne: jest.fn().mockResolvedValue(undefined), + }; +} + +it('records the cancelled lifecycle and still succeeds when Redis cleanup fails', async () => { + const match = liveMatch(); + const f = fixture({ + auction: redisAuction(), + match, + failRedisCleanup: true, + }); + await expect( + f.service.reset({ + matchId, + communityId, + clientObservedAuctionLifecycleId: 'life', + }), + ).resolves.toBeUndefined(); + expect(f.session.commitTransaction).toHaveBeenCalledTimes(1); + expect(match.lastCancelledAuctionLifecycleId).toBe('life'); + expect(match.status).toBe('recruiting'); +}); + +it('answers a retransmitted cancellation and durably schedules Redis cleanup', async () => { + const match = { ...liveMatch(), lastCancelledAuctionLifecycleId: 'life' }; + const f = fixture({ auction: null, match }); + await expect( + f.service.reset({ + matchId, + communityId, + clientObservedAuctionLifecycleId: 'life', + }), + ).resolves.toBeUndefined(); + expect(match).toMatchObject({ + auctionCancellationCleanupPending: true, + auctionCancellationCleanupRetryAt: expect.any(Date), + }); + expect(f.redisCommands).toEqual([]); +}); + +it('preserves a committed cancellation even if its lock renewal fails before returning', async () => { + jest.useFakeTimers(); + try { + const match = liveMatch(); + const f = fixture({ auction: redisAuction(), match }); + f.session.commitTransaction.mockImplementation(async () => { + await jest.advanceTimersByTimeAsync(10000); + }); + await expect( + f.service.reset({ + matchId, + communityId, + clientObservedAuctionLifecycleId: 'life', + }), + ).resolves.toBeUndefined(); + expect(match.auctionCancellationCleanupPending).toBe(true); + expect(f.session.commitTransaction).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } +}); + +it('keeps 404 for a missing auction that was not cancelled with this lifecycle', async () => { + for (const [observed, recorded] of [ + ['other', 'life'], + [undefined, 'life'], + ['life', undefined], + ] as const) { + const f = fixture({ + auction: null, + match: { ...liveMatch(), lastCancelledAuctionLifecycleId: recorded }, + }); + await expect( + f.service.reset({ + matchId, + communityId, + clientObservedAuctionLifecycleId: observed, + }), + ).rejects.toBeInstanceOf(NotFoundException); + expect(f.session.abortTransaction).toHaveBeenCalled(); + } +}); diff --git a/apps/api/src/modules/auction/auction-correctness.spec.ts b/apps/api/src/modules/auction/auction-correctness.spec.ts new file mode 100644 index 00000000..cb440e8b --- /dev/null +++ b/apps/api/src/modules/auction/auction-correctness.spec.ts @@ -0,0 +1,607 @@ +import mongoose from 'mongoose'; +import { AuctionService } from './auction.service'; +import { + AuctionDurabilityScheduler, + buildAuctionProjectionGuardedUpdate, + type AuctionDurabilityPayload, +} from './auction-durability.scheduler'; +import { AuctionDeadlineScheduler } from './auction-deadline.scheduler'; +import { AuctionLifecycleLock } from './auction-lifecycle-lock'; + +const query = (value: T) => ({ + select() { + return this; + }, + session() { + return this; + }, + lean() { + return this; + }, + exec: async () => value, +}); +const session = () => ({ + startTransaction: jest.fn(), + commitTransaction: jest.fn(), + abortTransaction: jest.fn(), + endSession: jest.fn(), +}); +const state = { + auctionLifecycleId: 'life', + revision: 12, + phase: 'presenting', + currentAuctionHistorySequence: 7, + currentAuctionPlayerOrderIndex: 0, + currentAuctionRound: 1, + auctionTeams: [], + auctionPlayerOrder: [], + passedAuctionPlayerQueue: [], + currentHighestBidPointAmount: 0, + currentHighestBidderTeamIndex: -1, + auctionCompletedAtTimestampMS: 0, + currentBidRoundDeadlineTimestampMS: 0, +}; + +beforeEach(() => { + jest + .spyOn(AuctionLifecycleLock.prototype, 'run') + .mockImplementation(async (_id, work) => work()); +}); +afterEach(() => jest.restoreAllMocks()); + +function matches( + document: Record, + filter: Record, +): boolean { + return Object.entries(filter).every(([key, value]) => { + if (key === '$or') + return (value as Record[]).some((alternative) => + matches(document, alternative), + ); + if (value && typeof value === 'object') { + return Object.entries(value).every(([operator, limit]) => { + if (operator === '$exists') + return (document[key] !== undefined) === limit; + if (operator === '$lt') return Number(document[key]) < Number(limit); + if (operator === '$lte') return Number(document[key]) <= Number(limit); + return false; + }); + } + return document[key] === value; + }); +} +function update( + document: Record, + mutation: Record>, +) { + Object.assign(document, mutation.$set); + for (const [key, amount] of Object.entries(mutation.$inc ?? {})) + document[key] = Number(document[key] ?? 0) + Number(amount); + for (const key of Object.keys(mutation.$unset ?? {})) delete document[key]; +} + +function sideEffectFixture(withReader = false) { + const projection: Record = { + _id: 'auction', + auctionLifecycleId: 'life', + revision: 9, + matchSideEffectRevision: 0, + }; + const application: Record = {}; + const payload: AuctionDurabilityPayload = { + auctionId: 'auction', + matchId: 'match', + auctionLifecycleId: 'life', + auctionStateTransitionId: 'complete', + auctionActionType: 'complete', + auctionHistorySequence: 10, + revision: 10, + snapshotBefore: {}, + historyMutation: { type: 'append' }, + matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' }, + afterAuctionMongoProjection: { + auctionLifecycleId: 'life', + revision: 10, + phase: 'complete', + matchSideEffectRevision: 1, + matchSideEffectStatus: 'pending', + matchSideEffectTransitionId: 'complete', + auctionTeams: [{ teamIndex: 1, captainUserId: 'captain', members: [] }], + }, + }; + const AuctionModel = { + findById: () => query({ ...projection }), + updateOne: ( + filter: Record, + mutation: Record>, + ) => { + const matchedCount = Number(matches(projection, filter)); + if (matchedCount) update(projection, mutation); + return query({ matchedCount }); + }, + findOneAndUpdate: ( + filter: Record, + mutation: Record>, + ) => { + if (!matches(projection, filter)) return query(null); + update(projection, mutation); + return query({ ...projection }); + }, + }; + const ApplicationModel = { + findOne: () => query({ ...application }), + updateOne: ( + filter: Record, + mutation: Record>, + ) => { + if (mutation.$setOnInsert) { + if (!application.state) + Object.assign(application, mutation.$setOnInsert); + return query({ matchedCount: 1 }); + } + const matchedCount = Number(matches(application, filter)); + if (matchedCount) update(application, mutation); + return query({ matchedCount }); + }, + }; + const match = { + participants: [], + teams: [], + save: jest.fn().mockResolvedValue(undefined), + }; + const historyFindOne = jest.fn().mockReturnValue(query(null)); + const historyUpdateOne = jest + .fn() + .mockReturnValue(query({ matchedCount: 1 })); + const models: Record = { + auction: AuctionModel, + auctionHistory: { + findOne: historyFindOne, + updateOne: historyUpdateOne, + }, + auctionSideEffectApplication: ApplicationModel, + match: { + findById: () => ({ ...query(match), session: async () => match }), + }, + }; + const reserveForMatch = jest + .fn() + .mockRejectedValueOnce(new Error('temporary reservation error')) + .mockResolvedValue(match); + const scheduleInitialPoll = jest.fn().mockResolvedValue(undefined); + const releaseFromMatch = jest.fn().mockResolvedValue(undefined); + const cancelPolling = jest.fn().mockResolvedValue(undefined); + let queued = true; + const sendCommand = jest.fn(async (command: string[]) => { + if (command[0] === 'ZRANGEBYSCORE') return queued ? ['payload'] : []; + if (command[0] === 'HGETALL') + return { payloadJson: JSON.stringify(payload) }; + if (command[0] === 'DEL') queued = false; + if (command[0] === 'HINCRBY') return 1; + return 1; + }); + const scheduler = new AuctionDurabilityScheduler( + { + getModel: async (name: string) => models[name], + createSession: async () => session(), + } as never, + { + getCurrentRedisTimestampMS: async () => 10000, + raw: () => ({ sendCommand }), + } as never, + { reserveForMatch, releaseFromMatch } as never, + { scheduleInitialPoll, cancelPolling } as never, + undefined, + undefined, + withReader + ? ({ + readAuctionStateView: async () => ({ + auctionLifecycleId: 'life', + phase: payload.afterAuctionMongoProjection.phase, + matchSideEffectTransitionId: 'complete', + }), + } as never) + : undefined, + ); + return { + scheduler, + payload, + projection, + application, + reserveForMatch, + scheduleInitialPoll, + sendCommand, + match, + historyFindOne, + historyUpdateOne, + releaseFromMatch, + cancelPolling, + }; +} + +it('retries failed completion work with the current generation and only then deletes the payload', async () => { + const f = sideEffectFixture(); + expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({ + failed: 1, + }); + expect(f.projection.matchSideEffectStatus).toBe('failed'); + expect(await f.scheduler.processDueDurabilityPayloads(20)).toMatchObject({ + applied: 1, + }); + expect(f.reserveForMatch).toHaveBeenCalledTimes(2); + expect(f.scheduleInitialPoll).toHaveBeenCalledTimes(1); + expect(f.application.state).toBe('completed'); + expect(f.projection.matchSideEffectStatus).toBe('completed'); + expect(f.projection.matchSideEffectRevision).toBe(5); + expect(f.sendCommand).toHaveBeenCalledWith(['DEL', 'payload']); +}); + +it('keeps an outbox payload while another worker owns its side-effect lease', async () => { + const f = sideEffectFixture(); + Object.assign(f.application, { + auctionId: 'auction', + auctionStateTransitionId: 'complete', + sideEffectType: 'syncMatchTeamsAfterComplete', + state: 'processing', + processingOwnerToken: 'other', + processingLeaseExpiresAtTimestampMS: 50000, + }); + expect(await f.scheduler.processDueDurabilityPayloads(20)).toEqual({ + applied: 0, + failed: 0, + skipped: 1, + }); + expect(f.sendCommand).not.toHaveBeenCalledWith(['DEL', 'payload']); + expect( + f.sendCommand.mock.calls.some(([command]) => command[0] === 'HINCRBY'), + ).toBe(false); + expect(f.reserveForMatch).not.toHaveBeenCalled(); +}); + +it('recovers side effects when a later ordinary projection was persisted first', async () => { + const f = sideEffectFixture(true); + f.reserveForMatch.mockReset().mockResolvedValue(undefined); + Object.assign(f.projection, { revision: 12 }); + await f.scheduler.applyDurabilityPayload(f.payload); + expect(f.reserveForMatch).toHaveBeenCalledTimes(1); + expect(f.projection.revision).toBe(12); + expect(f.projection.matchSideEffectTransitionId).toBe('complete'); + expect(f.projection.matchSideEffectStatus).toBe('completed'); +}); + +it('still cleans up completed Match state when a second undo invalidates the first undo history', async () => { + const f = sideEffectFixture(true); + Object.assign(f.projection, { revision: 12 }); + Object.assign(f.payload, { + matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' }, + }); + f.payload.afterAuctionMongoProjection.phase = 'presenting'; + Object.assign(f.match, { + teams: [{ teamIndex: 1 }], + teamBuiltAt: new Date(), + tournamentCodeIds: ['code'], + gameSchedule: [{}], + }); + f.historyFindOne.mockReturnValue( + query({ + sequence: 12, + historyMutation: { + type: 'deleteAfterSequenceAndAppendUndo', + targetHistorySequence: 2, + }, + }), + ); + await f.scheduler.applyDurabilityPayload(f.payload); + expect(f.historyUpdateOne).not.toHaveBeenCalled(); + expect(f.releaseFromMatch).toHaveBeenCalledTimes(1); + expect(f.cancelPolling).toHaveBeenCalledTimes(1); + expect(f.match).toMatchObject({ + teams: [], + teamBuiltAt: undefined, + tournamentCodeIds: [], + gameSchedule: [], + }); + expect(f.projection.matchSideEffectStatus).toBe('completed'); +}); + +it('does not let a superseded side effect overwrite the newer generation', async () => { + const f = sideEffectFixture(); + Object.assign(f.projection, { + revision: 12, + matchSideEffectTransitionId: 'undo', + matchSideEffectRevision: 6, + }); + await f.scheduler.applyDurabilityPayload(f.payload); + expect(f.reserveForMatch).not.toHaveBeenCalled(); + expect(f.projection.matchSideEffectTransitionId).toBe('undo'); +}); + +it('removes obsolete projection fields both from Mongo updates and subsequent undo snapshots', () => { + const service = new AuctionService( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + ); + const internal = service as unknown as { + buildAuctionMongoProjectionFromState( + auction: unknown, + state: Record, + ): Record; + buildAuctionSnapshotFromRedisState( + auction: unknown, + state: Record, + ): Record; + }; + const old = { + _id: new mongoose.Types.ObjectId(), + matchId: new mongoose.Types.ObjectId(), + completedAt: new Date(), + auctionCompletedAtTimestampMS: 100, + currentHighestBidderTeamIndex: 2, + currentBidderTeamIndex: 2, + }; + const projection = internal.buildAuctionMongoProjectionFromState(old, state); + const mutation = buildAuctionProjectionGuardedUpdate({ + auctionId: old._id, + auctionLifecycleId: 'life', + revision: 12, + afterAuctionMongoProjection: projection, + } as AuctionDurabilityPayload).update; + const stored = { ...old } as Record; + update(stored, mutation); + const snapshot = internal.buildAuctionSnapshotFromRedisState(old, state); + for (const record of [stored, snapshot]) { + expect(record).not.toHaveProperty('completedAt'); + expect(record).not.toHaveProperty('currentHighestBidderTeamIndex'); + expect(record.phase).toBe('presenting'); + } +}); + +it('applies delayed undo history cleanup without deleting newer branch entries or rolling back projection', async () => { + const deleteMany = jest.fn().mockReturnValue(query({})); + const updateOne = jest.fn().mockReturnValue(query({})); + const projectionUpdate = jest.fn(); + const models: Record = { + auction: { + findById: () => query({ auctionLifecycleId: 'life', revision: 12 }), + updateOne: projectionUpdate, + }, + auctionHistory: { findOne: () => query(null), deleteMany, updateOne }, + auctionSideEffectApplication: {}, + }; + const scheduler = new AuctionDurabilityScheduler( + { + getModel: async (name: string) => models[name], + createSession: async () => null, + } as never, + {} as never, + ); + await scheduler.applyDurabilityPayload({ + auctionId: 'auction', + matchId: 'match', + auctionStateTransitionId: 'undo', + auctionLifecycleId: 'life', + revision: 11, + auctionHistorySequence: 9, + auctionActionType: 'undo', + historyMutation: { + type: 'deleteAfterSequenceAndAppendUndo', + targetHistorySequence: 3, + }, + matchSideEffectMutation: { type: 'none' }, + afterAuctionMongoProjection: {}, + snapshotBefore: {}, + } as AuctionDurabilityPayload); + expect(deleteMany).toHaveBeenCalledWith( + { auctionId: 'auction', sequence: { $gt: 3, $lt: 9 } }, + { session: null }, + ); + expect(updateOne).toHaveBeenCalledTimes(1); + expect(projectionUpdate).not.toHaveBeenCalled(); +}); + +it('rejects a cancellation from a prior lifecycle before deleting anything', async () => { + const matchId = new mongoose.Types.ObjectId(), + communityId = new mongoose.Types.ObjectId(); + const deleteOne = jest.fn(); + const service = new AuctionService( + { + createSession: async () => session(), + getModel: async () => ({ + findOne: () => ({ + session: async () => ({ + auctionEngineVersion: 'redis-v1', + auctionLifecycleId: 'new-life', + deleteOne, + }), + }), + }), + } as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + ); + await expect( + service.reset({ + matchId, + communityId, + clientObservedAuctionLifecycleId: 'old-life', + }), + ).rejects.toThrow('lifecycle has changed'); + expect(deleteOne).not.toHaveBeenCalled(); +}); + +it('resumes startup after Mongo creation instead of returning an unfinished auction', async () => { + const matchId = new mongoose.Types.ObjectId(), + communityId = new mongoose.Types.ObjectId(); + const captain = new mongoose.Types.ObjectId(), + player = new mongoose.Types.ObjectId(), + lateJoiner = new mongoose.Types.ObjectId(); + const auction = { + _id: new mongoose.Types.ObjectId(), + matchId, + auctionLifecycleId: 'life', + revision: 1, + phase: 'presenting', + startupCompleted: false, + startAuctionRequestId: 'same-request', + teams: [{ captainUserId: captain, members: [], teamIndex: 1 }], + order: [{ userId: player }], + }; + const match = { + communityId, + status: 'recruiting', + participants: [captain, player, lateJoiner].map((userId) => ({ + userId, + excluded: false, + })), + save: jest.fn(), + }; + const updateOne = jest.fn(); + const historyUpdate = jest.fn(); + const models: Record = { + auction: { findOne: async () => auction, updateOne }, + match: { findById: async () => match }, + auctionHistory: { updateOne: historyUpdate }, + }; + const hydrateState = jest.fn().mockResolvedValue({ hydrated: true }); + const service = new AuctionService( + { getModel: async (name: string) => models[name] } as never, + {} as never, + {} as never, + {} as never, + {} as never, + { hydrateState } as never, + {} as never, + ); + await service.start({ + matchId, + communityId, + startAuctionRequestId: 'same-request', + } as never); + expect(hydrateState).toHaveBeenCalledTimes(1); + expect(historyUpdate).toHaveBeenCalledTimes(1); + expect(match.save).toHaveBeenCalledTimes(1); + expect(match.status).toBe('team-built'); + expect(auction.startupCompleted).toBe(true); + expect(match.participants.map((entry) => entry.excluded)).toEqual([ + false, + false, + true, + ]); +}); + +it('does not open bidding until interrupted startup has been completed', async () => { + const service = new AuctionService( + { + getModel: async () => ({ + findOne: async () => ({ + auctionEngineVersion: 'redis-v1', + startupCompleted: false, + }), + }), + } as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + ); + await expect( + service.openBidding(new mongoose.Types.ObjectId()), + ).rejects.toThrow('startup is not complete'); +}); + +it('uses Redis completion state to schedule cleanup despite a delayed Mongo projection', async () => { + const matchId = new mongoose.Types.ObjectId(), + communityId = new mongoose.Types.ObjectId(); + const auction = { + _id: new mongoose.Types.ObjectId(), + matchId, + auctionEngineVersion: 'redis-v1', + auctionLifecycleId: 'life', + phase: 'sold', + revision: 9, + }; + const snapshot = { ...state, phase: 'bidding', revision: 5 }; + const models: Record = { + auction: { findOne: () => ({ session: async () => auction }) }, + match: { findById: () => ({ session: async () => ({ communityId }) }) }, + auctionHistory: { findOne: () => query({ snapshotBefore: snapshot }) }, + }; + const applyTransition = jest.fn().mockResolvedValue({ ok: true }); + const commands = { + reserveActionRequest: async () => ({ + reserved: true, + actionRequestScopeKey: 'scope', + auctionStateTransitionId: 'undo', + observedProcessingLeaseVersion: 1, + }), + applyTransition, + }; + const service = new AuctionService( + { + getModel: async (name: string) => models[name], + createSession: async () => session(), + } as never, + {} as never, + {} as never, + {} as never, + {} as never, + commands as never, + { + readAuctionStateView: async () => ({ + ...state, + phase: 'complete', + revision: 10, + }), + } as never, + ); + await service.undoTo({ matchId, communityId, sequence: 2 }); + const input = applyTransition.mock.calls[0][1] as { + durabilityPayloadJson: string; + }; + expect(JSON.parse(input.durabilityPayloadJson)).toMatchObject({ + matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' }, + }); +}); + +it.each(['deadline', 'durability'])( + 'continues %s ticks when lock release fails', + async (kind) => { + const redis = { + acquireTokenLock: jest.fn().mockResolvedValue(true), + releaseTokenLock: jest + .fn() + .mockRejectedValue(new Error('redis disconnected')), + }; + const scheduler = + kind === 'deadline' + ? new AuctionDeadlineScheduler( + redis as never, + {} as never, + {} as never, + {} as never, + ) + : new AuctionDurabilityScheduler({} as never, redis as never); + const work = jest.fn().mockResolvedValue({}); + Object.assign( + scheduler, + kind === 'deadline' + ? { processDueBidRoundDeadlines: work } + : { processDueDurabilityPayloads: work }, + ); + const internal = scheduler as unknown as { safeTick(): Promise }; + await internal.safeTick(); + await internal.safeTick(); + expect(work).toHaveBeenCalledTimes(2); + }, +); diff --git a/apps/api/src/modules/auction/auction-deadline.scheduler.ts b/apps/api/src/modules/auction/auction-deadline.scheduler.ts index 60f9b654..c0951e4e 100644 --- a/apps/api/src/modules/auction/auction-deadline.scheduler.ts +++ b/apps/api/src/modules/auction/auction-deadline.scheduler.ts @@ -88,10 +88,13 @@ export class AuctionDeadlineScheduler implements OnModuleInit, OnModuleDestroy { ); } finally { if (leaderLockAcquired) { - await this.auctionRedisService.releaseTokenLock( - BID_ROUND_DEADLINE_LEADER_LOCK_KEY, - leaderLockToken, - ); + await this.auctionRedisService + .releaseTokenLock(BID_ROUND_DEADLINE_LEADER_LOCK_KEY, leaderLockToken) + .catch((error: unknown) => { + this.logger.warn( + `Auction deadline lock release failed: ${String(error)}`, + ); + }); } this.tickInProgress = false; } diff --git a/apps/api/src/modules/auction/auction-durability.scheduler.spec.ts b/apps/api/src/modules/auction/auction-durability.scheduler.spec.ts index 4cceabd4..1cd482f4 100644 --- a/apps/api/src/modules/auction/auction-durability.scheduler.spec.ts +++ b/apps/api/src/modules/auction/auction-durability.scheduler.spec.ts @@ -1,3 +1,5 @@ +import { AuctionLifecycleLock } from './auction-lifecycle-lock'; +import { buildAuctionProjectionUnset } from './auction-projection'; import { AuctionDurabilityScheduler, buildAuctionHistoryUpsert, @@ -23,6 +25,12 @@ const payload = { } as const; describe('auction durability projection helpers', () => { + beforeEach(() => { + jest + .spyOn(AuctionLifecycleLock.prototype, 'run') + .mockImplementation(async (_id, work) => work()); + }); + afterEach(() => jest.restoreAllMocks()); it('builds idempotent history upsert by auction and transition id', () => { expect(buildAuctionHistoryUpsert(payload)).toEqual({ updateOne: { @@ -59,6 +67,9 @@ describe('auction durability projection helpers', () => { }, update: { $set: { phase: 'sold', revision: 9 }, + $unset: buildAuctionProjectionUnset( + payload.afterAuctionMongoProjection, + ), }, }); }); @@ -210,7 +221,12 @@ describe('auction durability projection helpers', () => { auctionLifecycleId: 'life-1', revision: { $lt: 9 }, }, - { $set: { phase: 'sold', revision: 9 } }, + { + $set: { phase: 'sold', revision: 9 }, + $unset: buildAuctionProjectionUnset( + payload.afterAuctionMongoProjection, + ), + }, { session }, ); expect(session.commitTransaction).toHaveBeenCalledTimes(1); @@ -370,7 +386,7 @@ describe('auction durability projection helpers', () => { expect(historyDeleteMany).toHaveBeenCalledWith( { auctionId: 'auction-1', - sequence: { $gt: 2 }, + sequence: { $gt: 2, $lt: 4 }, }, { session }, ); @@ -811,21 +827,14 @@ describe('auction durability projection helpers', () => { skipped: 0, }); expect(sendCommand).toHaveBeenNthCalledWith(5, [ - 'ZREM', + 'EVAL', + expect.stringContaining("redis.call('HEXISTS'"), + '4', + 'payload-1', 'auction:auctionDurabilityRetryQueue', - 'payload-1', - ]); - expect(sendCommand).toHaveBeenNthCalledWith(6, [ - 'HSET', - 'payload-1', - 'deadLetteredAtRedisTimestampMS', - '1000', - ]); - expect(sendCommand).toHaveBeenNthCalledWith(7, [ - 'ZADD', 'auction:auctionDurabilityDeadLetterQueue', + 'auction:deadLetterStatusQueue', '1000', - 'payload-1', ]); }); @@ -1030,11 +1039,20 @@ describe('auction durability projection helpers', () => { { _id: 'auction-1', auctionLifecycleId: 'life-1', - matchSideEffectRevision: 1, + $or: [ + { matchSideEffectTransitionId: 'transition-1' }, + { + matchSideEffectTransitionId: { $exists: false }, + matchSideEffectRevision: 1, + }, + ], }, { $inc: { matchSideEffectRevision: 1 }, - $set: { matchSideEffectStatus: 'processing' }, + $set: { + matchSideEffectStatus: 'processing', + matchSideEffectTransitionId: 'transition-1', + }, $unset: { matchSideEffectError: '' }, }, expect.objectContaining({ returnDocument: 'after' }), @@ -1045,34 +1063,42 @@ describe('auction durability projection helpers', () => { _id: 'auction-1', auctionLifecycleId: 'life-1', matchSideEffectRevision: 2, + matchSideEffectTransitionId: 'transition-1', }, { $inc: { matchSideEffectRevision: 1 }, - $set: { matchSideEffectStatus: 'completed' }, + $set: { + matchSideEffectStatus: 'completed', + matchSideEffectTransitionId: 'transition-1', + }, $unset: { matchSideEffectError: '' }, }, expect.objectContaining({ returnDocument: 'after' }), ); expect(mirrorMatchSideEffectStatus).toHaveBeenCalledWith({ auctionLifecycleId: 'life-1', + auctionStateTransitionId: 'transition-1', matchId: 'match-1', matchSideEffectRevision: 2, matchSideEffectStatus: 'processing', }); expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({ auctionLifecycleId: 'life-1', + auctionStateTransitionId: 'transition-1', matchId: 'match-1', matchSideEffectRevision: 3, matchSideEffectStatus: 'completed', }); expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(1, 'match-1', { auctionLifecycleId: 'life-1', + auctionStateTransitionId: 'transition-1', currentServerTimestampMS: 2000, matchSideEffectRevision: 2, matchSideEffectStatus: 'processing', }); expect(broadcastMatchSideEffect).toHaveBeenNthCalledWith(2, 'match-1', { auctionLifecycleId: 'life-1', + auctionStateTransitionId: 'transition-1', currentServerTimestampMS: 3000, matchSideEffectRevision: 3, matchSideEffectStatus: 'completed', @@ -1080,7 +1106,7 @@ describe('auction durability projection helpers', () => { expect(sendCommand).toHaveBeenCalledWith([ 'EVAL', expect.stringContaining("redis.call('EXPIRE'"), - '11', + '12', 'auction:match-1:liveAuctionState', 'auction:bidRoundDeadlineQueue', 'auction:match-1:auctionStructureState', @@ -1092,6 +1118,7 @@ describe('auction durability projection helpers', () => { 'auction:match-1:auctionActionRequestResultOrder', 'auction:match-1:rehydrateLock', 'auction:match-1:auctionStateTransitionLock', + 'auction:match-1:rejectedBidRequestResultOrder', 'match-1', 'life-1', String(7 * 24 * 60 * 60), @@ -1263,7 +1290,12 @@ describe('auction durability projection helpers', () => { const broadcastMatchSideEffect = jest.fn(); const scheduler = new AuctionDurabilityScheduler( { - createSession: jest.fn().mockResolvedValue(null), + createSession: jest.fn().mockResolvedValueOnce(null).mockResolvedValue({ + startTransaction: jest.fn(), + commitTransaction: jest.fn(), + abortTransaction: jest.fn(), + endSession: jest.fn(), + }), getModel: jest.fn((name: string) => { if (name === 'auctionHistory') { return { @@ -1282,7 +1314,11 @@ describe('auction durability projection helpers', () => { return { updateOne: sideEffectUpdateOne }; } if (name === 'match') - return { findById: jest.fn().mockReturnValue(match) }; + return { + findById: jest.fn().mockReturnValue({ + session: jest.fn().mockResolvedValue(match), + }), + }; return {}; }), } as never, @@ -1301,8 +1337,8 @@ describe('auction durability projection helpers', () => { await scheduler.applyDurabilityPayload(resetPayload); - expect(releaseFromMatch).toHaveBeenCalledWith(match); - expect(cancelPolling).toHaveBeenCalledWith(match); + expect(releaseFromMatch).toHaveBeenCalledWith(match, expect.any(Object)); + expect(cancelPolling).toHaveBeenCalledWith(match, expect.any(Object)); expect(match).toMatchObject({ gameSchedule: [], result: undefined, @@ -1328,12 +1364,14 @@ describe('auction durability projection helpers', () => { ); expect(mirrorMatchSideEffectStatus).toHaveBeenLastCalledWith({ auctionLifecycleId: 'life-1', + auctionStateTransitionId: 'transition-1', matchId: 'match-1', matchSideEffectRevision: 3, matchSideEffectStatus: 'completed', }); expect(broadcastMatchSideEffect).toHaveBeenLastCalledWith('match-1', { auctionLifecycleId: 'life-1', + auctionStateTransitionId: 'transition-1', currentServerTimestampMS: 3000, matchSideEffectRevision: 3, matchSideEffectStatus: 'completed', diff --git a/apps/api/src/modules/auction/auction-durability.scheduler.ts b/apps/api/src/modules/auction/auction-durability.scheduler.ts index 7163e204..ecae2efe 100644 --- a/apps/api/src/modules/auction/auction-durability.scheduler.ts +++ b/apps/api/src/modules/auction/auction-durability.scheduler.ts @@ -4,6 +4,7 @@ import { OnModuleDestroy, OnModuleInit, Optional, + ConflictException, } from '@nestjs/common'; import { createHash, randomUUID } from 'crypto'; import type mongoose from 'mongoose'; @@ -16,6 +17,10 @@ import type { import { TournamentCodeService } from '../tournament-code/tournament-code.service'; import { TournamentResultService } from '../tournament-result/tournament-result.service'; import { AuctionEventBusService } from './auction-event-bus.service'; +import { AuctionService } from './auction.service'; +import { AuctionBidArchiveService } from './auction-bid-archive.service'; +import { AuctionLifecycleLock } from './auction-lifecycle-lock'; +import { buildAuctionProjectionUnset } from './auction-projection'; import { AuctionSideEffectService, buildMatchSideEffectDelta, @@ -26,6 +31,7 @@ import { AuctionRedisService } from './redis/auction-redis.service'; import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service'; import { AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, AUCTION_DURABILITY_RETRY_QUEUE_KEY, buildAuctionRedisKeys, } from './redis/auction-redis.keys'; @@ -36,6 +42,8 @@ const AUCTION_DURABILITY_BATCH_LIMIT = 20; const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000; const AUCTION_DURABILITY_MAX_RETRY_DELAY_MS = 60_000; const AUCTION_DURABILITY_MAX_ATTEMPTS = 10; +const AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS = 120; +const AUCTION_DURABILITY_MAX_PENDING_DURATION_MS = 10 * 60 * 1000; const AUCTION_DURABILITY_ERROR_MESSAGE_MAX_LENGTH = 2000; const AUCTION_DURABILITY_LEADER_LOCK_KEY = 'auction:auctionDurabilityScheduler:leader'; @@ -45,6 +53,7 @@ const AUCTION_SIDE_EFFECT_LEASE_DURATION_MS = 30_000; const SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE = 'syncMatchTeamsAfterComplete'; const RESET_MATCH_SIDE_EFFECT_TYPE = 'resetMatchAfterAuctionReset'; const UNDO_COMPLETED_MATCH_SIDE_EFFECT_TYPE = 'undoCompletedAuctionSideEffects'; +class SideEffectPendingError extends Error {} export interface AuctionDurabilityPayload { actorUserId?: unknown; @@ -67,6 +76,7 @@ export interface AuctionDurabilityPayload { export interface AuctionProjectionIdentity { auctionLifecycleId?: string; revision?: number; + matchSideEffectTransitionId?: string; } type ParsedAuctionHistoryMutation = @@ -85,7 +95,7 @@ interface CleanupSideEffectMatchDocument { participants: Array<{ excluded?: boolean }>; result?: unknown; resultPollState?: unknown; - save(): Promise; + save(options?: { session: mongoose.ClientSession }): Promise; status: string; teamBuiltAt?: Date; teams?: unknown[]; @@ -99,6 +109,7 @@ export class AuctionDurabilityScheduler private readonly logger = new Logger(AuctionDurabilityScheduler.name); private tickTimer?: NodeJS.Timeout; private tickInProgress = false; + private deadLetterScanCursor = '0'; constructor( private readonly connectionService: ConnectionService, @@ -113,6 +124,9 @@ export class AuctionDurabilityScheduler private readonly auctionEventBusService?: AuctionEventBusService, @Optional() private readonly auctionRedisStateReaderService?: AuctionRedisStateReaderService, + @Optional() private readonly auctionService?: AuctionService, + @Optional() + private readonly auctionBidArchiveService?: AuctionBidArchiveService, ) {} public onModuleInit(): void { @@ -187,6 +201,23 @@ export class AuctionDurabilityScheduler }); await this.clearDurabilityPayload(payloadKey); } catch (error) { + if ( + error instanceof SideEffectPendingError || + error instanceof ConflictException + ) { + const disposition = await this.deferDurabilityPayload( + payloadKey, + currentRedisTimestampMS, + error, + ); + if (disposition === 'dead-lettered') { + failed += 1; + this.logger.warn( + `Auction durability payload exceeded its pending limit: ${payloadKey}`, + ); + } else skipped += 1; + continue; + } failed += 1; const failureDisposition = await this.recordDurabilityPayloadFailure( payloadKey, @@ -214,6 +245,14 @@ export class AuctionDurabilityScheduler const payloadJson = hash.payloadJson; if (!payloadJson) return null; const payload = JSON.parse(payloadJson) as AuctionDurabilityPayload; + const committedDeadline = Number(hash.committedBidRoundDeadlineTimestampMS); + if (Number.isFinite(committedDeadline) && committedDeadline > 0) { + payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS = + committedDeadline; + payload.afterAuctionMongoProjection.currentDeadline = new Date( + committedDeadline, + ).toISOString(); + } const committedAtRedisTimestampMS = Number( hash.committedAtRedisTimestampMS, ); @@ -240,12 +279,12 @@ export class AuctionDurabilityScheduler const sideEffectMutation = parseMatchSideEffectMutation( payload.matchSideEffectMutation, ); + let shouldUpdateProjection = true; + let shouldApplySideEffects = true; + let shouldApplyHistoryMutation = true; try { session?.startTransaction(); - const historyMutation = parseAuctionHistoryMutation( - payload.historyMutation, - ); if (typeof AuctionModel.findById === 'function') { const currentProjection = await readAuctionProjectionIdentityForMutation( @@ -256,37 +295,44 @@ export class AuctionDurabilityScheduler if ( !shouldProcessAuctionDurabilityPayload(currentProjection, payload) ) { - // 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다. - // 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다. if ( - historyMutation.type === 'append' && - currentProjection.auctionLifecycleId === payload.auctionLifecycleId + currentProjection.auctionLifecycleId !== payload.auctionLifecycleId ) { - await this.applyStaleAppendHistoryRecovery({ + if (session) await session.commitTransaction(); + return; + } + shouldApplyHistoryMutation = + !(await this.isAppendHistoryInvalidatedByLaterDestructiveMutation({ AuctionHistoryModel, payload, session, - }); - } - if (session) await session.commitTransaction(); - return; + })); + // History operations have their own sequence bounds. A newer ordinary + // projection must not discard an earlier undo's history cleanup. + shouldUpdateProjection = false; + shouldApplySideEffects = + Boolean(this.auctionRedisStateReaderService) || + currentProjection.matchSideEffectTransitionId === + payload.auctionStateTransitionId; } } - await this.applyAuctionHistoryMutation({ - AuctionHistoryModel, - AuctionModel, - payload, - session, - }); + if (shouldApplyHistoryMutation) + await this.applyAuctionHistoryMutation({ + AuctionHistoryModel, + AuctionModel, + payload, + session, + }); const projectionOperation = buildAuctionProjectionGuardedUpdate(payload); - await execMaybe( - AuctionModel.updateOne( - projectionOperation.filter as never, - projectionOperation.update as never, - { session } as never, - ), - ); - if (sideEffectMutation) { + if (shouldUpdateProjection) + await execMaybe( + AuctionModel.updateOne( + projectionOperation.filter as never, + projectionOperation.update as never, + { session } as never, + ), + ); + if (sideEffectMutation && shouldApplySideEffects) { const sideEffectOperation = buildSideEffectApplicationUpsert( payload, sideEffectMutation, @@ -312,7 +358,7 @@ export class AuctionDurabilityScheduler await this.broadcastLatestRedisAuctionStateIfRequested(payload); - if (sideEffectMutation) { + if (sideEffectMutation && shouldApplySideEffects) { await this.applyMatchSideEffect(payload, sideEffectMutation); } } @@ -352,6 +398,64 @@ export class AuctionDurabilityScheduler private async applyMatchSideEffect( payload: AuctionDurabilityPayload, mutation: ParsedMatchSideEffectMutation, + ): Promise { + return new AuctionLifecycleLock(this.auctionRedisService).run( + String(payload.matchId), + async () => { + if (this.auctionRedisStateReaderService) { + const state = + await this.auctionRedisStateReaderService.readAuctionStateView( + String(payload.matchId), + ); + if (!state) + throw new SideEffectPendingError( + 'Auction realtime state is unavailable.', + ); + if ( + state.auctionLifecycleId !== payload.auctionLifecycleId || + (state.matchSideEffectTransitionId && + state.matchSideEffectTransitionId !== + payload.auctionStateTransitionId) || + (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE && + state.phase !== 'complete') + ) + return; + // An ordinary later projection may have reached Mongo before this + // side-effect-bearing transition. Install its pending generation only + // while Redis still names it and Mongo has not advanced beyond it. + const AuctionModel = await this.connectionService.getModel('auction'); + const pendingRevision = Number( + payload.afterAuctionMongoProjection.matchSideEffectRevision ?? 0, + ); + await execMaybe( + AuctionModel.updateOne( + { + _id: payload.auctionId, + auctionLifecycleId: payload.auctionLifecycleId, + $or: [ + { matchSideEffectRevision: { $lt: pendingRevision } }, + { matchSideEffectRevision: { $exists: false } }, + ], + } as never, + { + $set: { + matchSideEffectTransitionId: payload.auctionStateTransitionId, + matchSideEffectRevision: pendingRevision, + matchSideEffectStatus: 'pending', + }, + $unset: { matchSideEffectError: '' }, + } as never, + ), + ); + } + await this.applyMatchSideEffectWhileLocked(payload, mutation); + }, + ); + } + + private async applyMatchSideEffectWhileLocked( + payload: AuctionDurabilityPayload, + mutation: ParsedMatchSideEffectMutation, ): Promise { if (mutation.type === SYNC_MATCH_TEAMS_SIDE_EFFECT_TYPE) { await this.applySyncMatchTeamsSideEffect(payload, mutation); @@ -407,7 +511,26 @@ export class AuctionDurabilityScheduler }) as never, ), ); - if (!hasMatchedDocument(leaseResult)) return; + if (!hasMatchedDocument(leaseResult)) { + const existing = await AuctionSideEffectApplicationModel.findOne( + identity as never, + ) + .lean() + .exec(); + if (existing?.state === 'completed') { + await this.persistAndBroadcastMatchSideEffectStatus({ + AuctionModel, + currentServerTimestampMS: currentRedisTimestampMS, + payload, + status: 'completed', + }); + await this.expireCompletedAuctionRedisRuntime(payload); + return; + } + throw new SideEffectPendingError( + 'Auction side effect is still owned by another worker.', + ); + } const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus( { @@ -420,7 +543,11 @@ export class AuctionDurabilityScheduler status: 'processing', }, ); - if (!processingDelta) return; + if (!processingDelta) { + throw new SideEffectPendingError( + 'Auction side-effect generation is not ready.', + ); + } const expectedMatchSideEffectRevision = processingDelta.matchSideEffectRevision; @@ -489,7 +616,11 @@ export class AuctionDurabilityScheduler await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( - identity as never, + { + ...identity, + processingOwnerToken: ownerToken, + state: 'processing', + } as never, { $set: { completedAtTimestampMS, @@ -518,7 +649,11 @@ export class AuctionDurabilityScheduler await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( - identity as never, + { + ...identity, + processingOwnerToken: ownerToken, + state: 'processing', + } as never, { $set: { lastError: error instanceof Error ? error.message : String(error), @@ -544,33 +679,6 @@ export class AuctionDurabilityScheduler } } - private async applyStaleAppendHistoryRecovery(params: { - AuctionHistoryModel: { - findOne?: (...args: never[]) => unknown; - updateOne: (...args: never[]) => unknown; - }; - payload: AuctionDurabilityPayload; - session: mongoose.ClientSession | null | undefined; - }): Promise { - const invalidated = - await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params); - if (invalidated) return; - - const historyOperation = buildAuctionHistoryUpsert( - params.payload, - ).updateOne; - await execMaybe( - params.AuctionHistoryModel.updateOne( - historyOperation.filter as never, - historyOperation.update as never, - { - session: params.session, - upsert: true, - } as never, - ) as Promise | { exec(): Promise }, - ); - } - private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: { AuctionHistoryModel: { findOne?: (...args: never[]) => unknown; @@ -622,9 +730,7 @@ export class AuctionDurabilityScheduler await this.applyMatchCleanupSideEffect({ mutation, payload, - updateMatch: async (match) => { - await this.tournamentCodeService?.releaseFromMatch(match as never); - await this.tournamentResultService?.cancelPolling(match as never); + updateMatch: (match) => { match.status = 'recruiting'; match.teams = []; match.tournamentCodeIds = []; @@ -635,7 +741,6 @@ export class AuctionDurabilityScheduler match.teamBuiltAt = undefined; match.result = undefined; match.resultPollState = undefined; - await match.save(); }, }); } @@ -647,13 +752,14 @@ export class AuctionDurabilityScheduler await this.applyMatchCleanupSideEffect({ mutation, payload, - updateMatch: async (match) => { - await this.tournamentCodeService?.releaseFromMatch(match as never); - await this.tournamentResultService?.cancelPolling(match as never); + updateMatch: (match) => { match.status = 'team-built'; + match.teams = []; + match.teamBuiltAt = undefined; + match.tournamentCodeIds = []; + match.gameSchedule = []; match.result = undefined; match.resultPollState = undefined; - await match.save(); }, }); } @@ -661,7 +767,7 @@ export class AuctionDurabilityScheduler private async applyMatchCleanupSideEffect(params: { mutation: ParsedMatchSideEffectMutation; payload: AuctionDurabilityPayload; - updateMatch: (match: CleanupSideEffectMatchDocument) => Promise; + updateMatch: (match: CleanupSideEffectMatchDocument) => void; }): Promise { if (!this.tournamentCodeService || !this.tournamentResultService) { throw new Error('Auction side-effect services are not available.'); @@ -700,7 +806,25 @@ export class AuctionDurabilityScheduler }) as never, ), ); - if (!hasMatchedDocument(leaseResult)) return; + if (!hasMatchedDocument(leaseResult)) { + const existing = await AuctionSideEffectApplicationModel.findOne( + identity as never, + ) + .lean() + .exec(); + if (existing?.state === 'completed') { + await this.persistAndBroadcastMatchSideEffectStatus({ + AuctionModel, + currentServerTimestampMS: currentRedisTimestampMS, + payload: params.payload, + status: 'completed', + }); + return; + } + throw new SideEffectPendingError( + 'Auction side effect is still owned by another worker.', + ); + } const processingDelta = await this.persistAndBroadcastMatchSideEffectStatus( { @@ -714,29 +838,56 @@ export class AuctionDurabilityScheduler status: 'processing', }, ); - if (!processingDelta) return; + if (!processingDelta) { + throw new SideEffectPendingError( + 'Auction side-effect generation is not ready.', + ); + } const expectedMatchSideEffectRevision = processingDelta.matchSideEffectRevision; try { - const match = (await execMaybe( - MatchModel.findById(params.payload.matchId) as never, - )) as CleanupSideEffectMatchDocument | null; - if (!match) { - throw new Error(`Match not found: ${String(params.payload.matchId)}`); - } - if (match.competitionId) { - throw new Error( - 'Cannot reset or undo Match teams after the immutable Competition has been created.', + const session = await this.connectionService.createSession(); + if (!session) + throw new Error('Failed to start Match cleanup transaction.'); + try { + session.startTransaction(); + const match = (await MatchModel.findById( + params.payload.matchId, + ).session(session)) as CleanupSideEffectMatchDocument | null; + if (!match) + throw new Error(`Match not found: ${String(params.payload.matchId)}`); + if (match.competitionId) + throw new Error( + 'Cannot reset or undo Match teams after the immutable Competition has been created.', + ); + await this.tournamentCodeService.releaseFromMatch( + match as never, + session, ); + await this.tournamentResultService.cancelPolling( + match as never, + session, + ); + params.updateMatch(match); + await match.save({ session }); + await session.commitTransaction(); + } catch (error) { + await session.abortTransaction(); + throw error; + } finally { + await session.endSession(); } - await params.updateMatch(match); const completedAtTimestampMS = await this.auctionRedisService.getCurrentRedisTimestampMS(); await execMaybe( AuctionSideEffectApplicationModel.updateOne( - identity as never, + { + ...identity, + processingOwnerToken: ownerToken, + state: 'processing', + } as never, { $set: { completedAtTimestampMS, @@ -766,7 +917,11 @@ export class AuctionDurabilityScheduler error instanceof Error ? error.message : String(error); await execMaybe( AuctionSideEffectApplicationModel.updateOne( - identity as never, + { + ...identity, + processingOwnerToken: ownerToken, + state: 'processing', + } as never, { $set: { lastError: errorMessage, @@ -828,7 +983,10 @@ export class AuctionDurabilityScheduler params.AuctionHistoryModel.deleteMany( { auctionId: params.payload.auctionId, - sequence: { $gt: mutation.targetHistorySequence }, + sequence: { + $gt: mutation.targetHistorySequence, + $lt: params.payload.auctionHistorySequence, + }, } as never, { session: params.session } as never, ) as Promise | { exec(): Promise }, @@ -842,6 +1000,7 @@ export class AuctionDurabilityScheduler params.AuctionHistoryModel.deleteMany( { auctionId: params.payload.auctionId, + sequence: { $lt: params.payload.auctionHistorySequence }, } as never, { session: params.session } as never, ) as Promise | { exec(): Promise }, @@ -874,16 +1033,43 @@ export class AuctionDurabilityScheduler }; currentServerTimestampMS: number; errorMessage?: string; - expectedMatchSideEffectRevision: number; + expectedMatchSideEffectRevision?: number; payload: AuctionDurabilityPayload; status: AuctionMatchSideEffectStatus; + onlyUnfinished?: boolean; }): Promise { const updatedAuction = (await execMaybe( params.AuctionModel.findOneAndUpdate( { _id: params.payload.auctionId, auctionLifecycleId: params.payload.auctionLifecycleId, - matchSideEffectRevision: params.expectedMatchSideEffectRevision, + ...(params.onlyUnfinished + ? { matchSideEffectStatus: { $ne: 'completed' } } + : {}), + ...(params.status === 'processing' + ? { + $or: [ + { + matchSideEffectTransitionId: + params.payload.auctionStateTransitionId, + }, + { + matchSideEffectTransitionId: { $exists: false }, + matchSideEffectRevision: + params.expectedMatchSideEffectRevision, + }, + ], + } + : { + matchSideEffectTransitionId: + params.payload.auctionStateTransitionId, + ...(params.expectedMatchSideEffectRevision === undefined + ? {} + : { + matchSideEffectRevision: + params.expectedMatchSideEffectRevision, + }), + }), } as never, { $inc: { matchSideEffectRevision: 1 }, @@ -892,6 +1078,8 @@ export class AuctionDurabilityScheduler ? { matchSideEffectError: params.errorMessage } : {}), matchSideEffectStatus: params.status, + matchSideEffectTransitionId: + params.payload.auctionStateTransitionId, }, ...(params.errorMessage ? {} @@ -915,6 +1103,7 @@ export class AuctionDurabilityScheduler const delta = buildMatchSideEffectDelta({ auctionLifecycleId: updatedAuction.auctionLifecycleId, + auctionStateTransitionId: params.payload.auctionStateTransitionId, currentServerTimestampMS: params.currentServerTimestampMS, matchSideEffectRevision: Number( updatedAuction.matchSideEffectRevision ?? 0, @@ -923,19 +1112,170 @@ export class AuctionDurabilityScheduler updatedAuction.matchSideEffectStatus ?? params.status, }); - await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({ - auctionLifecycleId: delta.auctionLifecycleId, - matchId: String(params.payload.matchId), - matchSideEffectRevision: delta.matchSideEffectRevision, - matchSideEffectStatus: delta.matchSideEffectStatus, - }); - this.auctionEventBusService?.broadcastMatchSideEffect( - String(params.payload.matchId), - delta, - ); + const mirrored = + await this.auctionSideEffectService?.mirrorMatchSideEffectStatus({ + auctionLifecycleId: delta.auctionLifecycleId, + auctionStateTransitionId: params.payload.auctionStateTransitionId, + matchId: String(params.payload.matchId), + matchSideEffectRevision: delta.matchSideEffectRevision, + matchSideEffectStatus: delta.matchSideEffectStatus, + }); + if (mirrored !== false) { + this.auctionEventBusService?.broadcastMatchSideEffect( + String(params.payload.matchId), + delta, + ); + } return delta; } + public async reconcileDeadLetterStatuses(limit = 20): Promise { + const client = this.auctionRedisService.raw(); + const now = await this.auctionRedisService.getCurrentRedisTimestampMS(); + // Also reconcile entries dead-lettered before status notifications existed. + const scanned = await client.sendCommand([ + 'ZSCAN', + AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, + this.deadLetterScanCursor, + 'COUNT', + String(limit), + ]); + if (Array.isArray(scanned) && Array.isArray(scanned[1])) { + this.deadLetterScanCursor = String(scanned[0]); + for (let index = 0; index < scanned[1].length; index += 2) { + const key = String(scanned[1][index]); + if ( + !(await client.sendCommand([ + 'HGET', + key, + 'deadLetterStatusReconciledAtTimestampMS', + ])) + ) { + await client.sendCommand([ + 'ZADD', + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, + 'NX', + String(now), + key, + ]); + } + } + } + const reply = await client.sendCommand([ + 'ZRANGEBYSCORE', + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, + '-inf', + String(now), + 'LIMIT', + '0', + String(limit), + ]); + if (!Array.isArray(reply)) return; + for (const key of reply.map(String)) { + try { + const payload = await this.readDurabilityPayload(key); + if ( + payload && + parseMatchSideEffectMutation(payload.matchSideEffectMutation) + ) { + const state = + await this.auctionRedisStateReaderService?.readAuctionStateView( + String(payload.matchId), + ); + // A terminal notification for a previous generation must not change + // a new completion/undo, even when Mongo is behind Redis. + const superseded = + state && + (state.auctionLifecycleId !== payload.auctionLifecycleId || + state.matchSideEffectStatus === 'completed' || + (state.matchSideEffectTransitionId && + state.matchSideEffectTransitionId !== + payload.auctionStateTransitionId)); + if (!superseded) { + const AuctionModel = + await this.connectionService.getModel('auction'); + if ( + state?.matchSideEffectTransitionId === + payload.auctionStateTransitionId + ) { + const revision = Number( + payload.afterAuctionMongoProjection.matchSideEffectRevision ?? + 0, + ); + await AuctionModel.updateOne( + { + _id: payload.auctionId, + auctionLifecycleId: payload.auctionLifecycleId, + $or: [ + { matchSideEffectRevision: { $lt: revision } }, + { matchSideEffectRevision: { $exists: false } }, + ], + } as never, + { + $set: { + matchSideEffectTransitionId: + payload.auctionStateTransitionId, + matchSideEffectRevision: revision, + matchSideEffectStatus: 'pending', + }, + }, + ); + // Heal a same-generation revision mismatch before sending the + // terminal status; otherwise Redis could ignore it as older. + await AuctionModel.updateOne( + { + _id: payload.auctionId, + auctionLifecycleId: payload.auctionLifecycleId, + matchSideEffectTransitionId: payload.auctionStateTransitionId, + matchSideEffectStatus: { $ne: 'completed' }, + } as never, + { + $max: { + matchSideEffectRevision: Number( + state.matchSideEffectRevision ?? revision, + ), + }, + }, + ); + } + const error = await client.sendCommand(['HGET', key, 'lastError']); + await this.persistAndBroadcastMatchSideEffectStatus({ + AuctionModel: AuctionModel as never, + payload, + currentServerTimestampMS: now, + status: 'failed', + onlyUnfinished: true, + errorMessage: + typeof error === 'string' + ? error + : 'Auction side effect exhausted its retry limit.', + }); + } + } + await client.sendCommand([ + 'EVAL', + "if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 1 then redis.call('HSET', KEYS[1], 'deadLetterStatusReconciledAtTimestampMS', ARGV[1]) end redis.call('ZREM', KEYS[2], KEYS[1]); return 1", + '2', + key, + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, + String(now), + ]); + } catch (error) { + // Mongo or Redis mirroring may be unavailable at dead-letter time. Keep + // the notification durable until the failure can be shown to viewers. + await client.sendCommand([ + 'ZADD', + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, + String(now + 5000), + key, + ]); + this.logger.warn( + `Auction terminal status will retry for ${key}: ${String(error)}`, + ); + } + } + } + private async safeTick(): Promise { if (this.tickInProgress) return; this.tickInProgress = true; @@ -948,7 +1288,22 @@ export class AuctionDurabilityScheduler AUCTION_DURABILITY_LEADER_LOCK_TTL_MS, ); if (!leaderLockAcquired) return; - await this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT); + // A failed maintenance task must not starve the other durable queues. + for (const work of [ + () => this.processDueDurabilityPayloads(AUCTION_DURABILITY_BATCH_LIMIT), + () => this.reconcileDeadLetterStatuses(AUCTION_DURABILITY_BATCH_LIMIT), + () => this.auctionBidArchiveService?.persistPendingBids(), + () => + this.auctionService?.retryPendingCancellations( + AUCTION_DURABILITY_BATCH_LIMIT, + ), + ]) { + try { + await work(); + } catch (error) { + this.logger.warn(`Auction maintenance task failed: ${String(error)}`); + } + } } catch (error) { this.logger.warn( `Auction durability tick failed: ${ @@ -957,10 +1312,13 @@ export class AuctionDurabilityScheduler ); } finally { if (leaderLockAcquired) { - await this.auctionRedisService.releaseTokenLock( - AUCTION_DURABILITY_LEADER_LOCK_KEY, - leaderLockToken, - ); + await this.auctionRedisService + .releaseTokenLock(AUCTION_DURABILITY_LEADER_LOCK_KEY, leaderLockToken) + .catch((error: unknown) => { + this.logger.warn( + `Auction durability lock release failed: ${String(error)}`, + ); + }); } this.tickInProgress = false; } @@ -987,6 +1345,51 @@ export class AuctionDurabilityScheduler ]); } + private async deferDurabilityPayload( + payloadKey: string, + timestampMS: number, + error: unknown, + ): Promise<'gone' | 'requeued' | 'dead-lettered'> { + // Keep coordination waits separate from failed work, but bound them across + // restarts too. Updating the payload and queue atomically avoids orphan jobs. + const result = await this.auctionRedisService + .raw() + .sendCommand([ + 'EVAL', + [ + "if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end", + "local count = redis.call('HINCRBY', KEYS[1], 'pendingAttempts', 1)", + "redis.call('HSETNX', KEYS[1], 'pendingSinceTimestampMS', ARGV[1])", + "local since = tonumber(redis.call('HGET', KEYS[1], 'pendingSinceTimestampMS'))", + "redis.call('HSET', KEYS[1], 'lastPendingReason', ARGV[5], 'lastPendingAtTimestampMS', ARGV[1])", + 'if count >= tonumber(ARGV[3]) or tonumber(ARGV[1]) - since >= tonumber(ARGV[4]) then', + " redis.call('ZREM', KEYS[2], KEYS[1])", + " redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1], 'lastError', ARGV[5])", + " redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1])", + " redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1])", + ' return 2', + 'end', + "redis.call('ZADD', KEYS[2], tonumber(ARGV[1]) + tonumber(ARGV[2]), KEYS[1])", + 'return 1', + ].join('\n'), + '4', + payloadKey, + AUCTION_DURABILITY_RETRY_QUEUE_KEY, + AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, + String(timestampMS), + String(AUCTION_DURABILITY_RETRY_DELAY_MS), + String(AUCTION_DURABILITY_MAX_PENDING_ATTEMPTS), + String(AUCTION_DURABILITY_MAX_PENDING_DURATION_MS), + formatErrorMessageForRedis(error), + ]); + return Number(result) === 2 + ? 'dead-lettered' + : Number(result) === 1 + ? 'requeued' + : 'gone'; + } + private async recordDurabilityPayloadFailure( payloadKey: string, currentRedisTimestampMS: number, @@ -1012,24 +1415,17 @@ export class AuctionDurabilityScheduler Number.isFinite(attempts) && attempts >= AUCTION_DURABILITY_MAX_ATTEMPTS ) { - await this.auctionRedisService - .raw() - .sendCommand(['ZREM', AUCTION_DURABILITY_RETRY_QUEUE_KEY, payloadKey]); await this.auctionRedisService .raw() .sendCommand([ - 'HSET', + 'EVAL', + "if redis.call('HEXISTS', KEYS[1], 'payloadJson') == 0 then return 0 end redis.call('HSET', KEYS[1], 'deadLetteredAtRedisTimestampMS', ARGV[1]); redis.call('ZREM', KEYS[2], KEYS[1]); redis.call('ZADD', KEYS[3], ARGV[1], KEYS[1]); redis.call('ZADD', KEYS[4], ARGV[1], KEYS[1]); return 1", + '4', payloadKey, - 'deadLetteredAtRedisTimestampMS', - String(currentRedisTimestampMS), - ]); - await this.auctionRedisService - .raw() - .sendCommand([ - 'ZADD', + AUCTION_DURABILITY_RETRY_QUEUE_KEY, AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, String(currentRedisTimestampMS), - payloadKey, ]); return 'dead-lettered'; } @@ -1064,7 +1460,7 @@ export class AuctionDurabilityScheduler 'end', 'return 1', ].join('\n'), - '11', + '12', keys.liveStateKey, keys.deadlineQueueKey, keys.structureStateKey, @@ -1076,6 +1472,7 @@ export class AuctionDurabilityScheduler keys.actionRequestResultOrderKey, keys.rehydrateLockKey, keys.stateTransitionLockKey, + keys.rejectedBidRequestResultOrderKey, matchId, payload.auctionLifecycleId, String(AUCTION_REDIS_COMPLETED_RETAIN_SECONDS), @@ -1159,6 +1556,7 @@ async function readAuctionProjectionIdentityForMutation( ? (query as { select: (...args: never[]) => unknown }).select({ auctionLifecycleId: 1, revision: 1, + matchSideEffectTransitionId: 1, } as never) : query; const scoped = @@ -1178,6 +1576,10 @@ async function readAuctionProjectionIdentityForMutation( if (!projection || typeof projection !== 'object') return {}; const record = projection as Record; return { + matchSideEffectTransitionId: + typeof record.matchSideEffectTransitionId === 'string' + ? record.matchSideEffectTransitionId + : undefined, auctionLifecycleId: typeof record.auctionLifecycleId === 'string' ? record.auctionLifecycleId @@ -1200,6 +1602,7 @@ export function buildAuctionProjectionGuardedUpdate( }, update: { $set: payload.afterAuctionMongoProjection, + $unset: buildAuctionProjectionUnset(payload.afterAuctionMongoProjection), }, }; } diff --git a/apps/api/src/modules/auction/auction-event-bus.service.ts b/apps/api/src/modules/auction/auction-event-bus.service.ts index f617de51..93b70908 100644 --- a/apps/api/src/modules/auction/auction-event-bus.service.ts +++ b/apps/api/src/modules/auction/auction-event-bus.service.ts @@ -24,4 +24,10 @@ export class AuctionEventBusService { ?.to(matchId) .emit('state', serializeRealtimeAuction(state)); } + + public broadcastReset(matchId: string, auctionLifecycleId: string): void { + this.auctionNamespace + ?.to(matchId) + .emit('reset', { matchId, auctionLifecycleId }); + } } diff --git a/apps/api/src/modules/auction/auction-lifecycle-lock.spec.ts b/apps/api/src/modules/auction/auction-lifecycle-lock.spec.ts new file mode 100644 index 00000000..8d240b18 --- /dev/null +++ b/apps/api/src/modules/auction/auction-lifecycle-lock.spec.ts @@ -0,0 +1,32 @@ +import { AuctionLifecycleLock } from './auction-lifecycle-lock'; + +afterEach(() => jest.useRealTimers()); + +it('renews ownership and handles renewal/release failures without unhandled rejection', async () => { + jest.useFakeTimers(); + const sendCommand = jest + .fn() + .mockRejectedValue(new Error('redis disconnected')); + const releaseTokenLock = jest + .fn() + .mockRejectedValue(new Error('release failed')); + const lock = new AuctionLifecycleLock({ + acquireTokenLock: jest.fn().mockResolvedValue(true), + releaseTokenLock, + raw: () => ({ sendCommand }), + } as never); + const result = lock.run('match', async () => { + await jest.advanceTimersByTimeAsync(10000); + }); + await expect(result).rejects.toThrow('redis disconnected'); + expect(releaseTokenLock).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); +}); + +it('preserves an operation result if only lock release fails', async () => { + const lock = new AuctionLifecycleLock({ + acquireTokenLock: jest.fn().mockResolvedValue(true), + releaseTokenLock: jest.fn().mockRejectedValue(new Error('release failed')), + } as never); + await expect(lock.run('match', async () => 42)).resolves.toBe(42); +}); diff --git a/apps/api/src/modules/auction/auction-lifecycle-lock.ts b/apps/api/src/modules/auction/auction-lifecycle-lock.ts new file mode 100644 index 00000000..aeabc1c8 --- /dev/null +++ b/apps/api/src/modules/auction/auction-lifecycle-lock.ts @@ -0,0 +1,72 @@ +import { ConflictException, Logger } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import { AuctionRedisService } from './redis/auction-redis.service'; + +export class AuctionLifecycleBusyError extends ConflictException { + constructor() { + super({ + code: 'AUCTION_LIFECYCLE_BUSY', + message: '다른 경매 작업을 처리 중입니다. 잠시 후 다시 시도해 주세요.', + }); + } +} + +/** Serializes startup, cancellation, undo and Match side effects across nodes. */ +export class AuctionLifecycleLock { + private readonly logger = new Logger(AuctionLifecycleLock.name); + + constructor(private readonly redis: AuctionRedisService) {} + + async run(matchId: string, operation: () => Promise): Promise { + // Kept outside runtime cleanup: cancelling an auction must not release a + // lock still held by its cancellation/recovery operation. + const key = `auction:${matchId}:lifecycleMutationLock`; + const token = randomUUID(); + if (!(await this.redis.acquireTokenLock(key, token, 30_000))) { + throw new AuctionLifecycleBusyError(); + } + let renewal: Promise | undefined; + let ownershipError: Error | undefined; + const timer = setInterval(() => { + if (renewal) return; + renewal = this.redis + .raw() + .sendCommand([ + 'SET', + key, + token, + 'IFEQ', + token, + 'PX', + '30000', + ]) + .then((result) => { + if (result !== 'OK') + throw new Error('Auction lifecycle lock ownership was lost.'); + }) + .catch((error: unknown) => { + ownershipError = + error instanceof Error ? error : new Error(String(error)); + }) + .finally(() => { + renewal = undefined; + }); + }, 10_000); + timer.unref?.(); + try { + const result = await operation(); + clearInterval(timer); + await renewal; + if (ownershipError) throw ownershipError; + return result; + } finally { + clearInterval(timer); + await renewal; + await this.redis.releaseTokenLock(key, token).catch((error: unknown) => { + this.logger.warn( + `Auction lifecycle lock release failed: ${String(error)}`, + ); + }); + } + } +} diff --git a/apps/api/src/modules/auction/auction-projection.ts b/apps/api/src/modules/auction/auction-projection.ts new file mode 100644 index 00000000..773e8200 --- /dev/null +++ b/apps/api/src/modules/auction/auction-projection.ts @@ -0,0 +1,20 @@ +/** Missing optional fields in a complete projection mean deletion, not retention. */ +export const AUCTION_OPTIONAL_PROJECTION_FIELDS = [ + 'completedAt', + 'auctionCompletedAtTimestampMS', + 'currentBidderTeamIndex', + 'currentHighestBidderTeamIndex', + 'currentDeadline', + 'currentBidRoundDeadlineTimestampMS', + 'currentBidRoundId', +] as const; + +export function buildAuctionProjectionUnset( + projection: Record, +) { + return Object.fromEntries( + AUCTION_OPTIONAL_PROJECTION_FIELDS.filter( + (field) => projection[field] === undefined, + ).map((field) => [field, '']), + ); +} diff --git a/apps/api/src/modules/auction/auction-rejection.spec.ts b/apps/api/src/modules/auction/auction-rejection.spec.ts index aaf89184..cbc45659 100644 --- a/apps/api/src/modules/auction/auction-rejection.spec.ts +++ b/apps/api/src/modules/auction/auction-rejection.spec.ts @@ -5,6 +5,11 @@ import { } from './auction-rejection'; describe('resolveAuctionRejectionReason', () => { + it('distinguishes an outdated client from a stale round and does not suggest a state-only resync', () => { + const reason = resolveAuctionRejectionReason('BID_CLIENT_UPDATE_REQUIRED'); + expect(reason).toBe('BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED'); + expect(isAuctionResyncRecommended(reason)).toBe(false); + }); it('reports roster validation failures without recommending a retry', () => { const reason = resolveAuctionRejectionReason( 'AUCTION_ROSTER_INVALID', diff --git a/apps/api/src/modules/auction/auction-rejection.ts b/apps/api/src/modules/auction/auction-rejection.ts index e29ce974..f681e443 100644 --- a/apps/api/src/modules/auction/auction-rejection.ts +++ b/apps/api/src/modules/auction/auction-rejection.ts @@ -46,6 +46,8 @@ export const AUCTION_REJECTION_REASON = { 'AUCTION_TRANSITION_REJECTED_BECAUSE_REQUEST_COORDINATION_RETRYABLE', // === 입찰(bid) === + BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED: + 'BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED', /** 입찰 라운드가 이미 다음 선수/다음 라운드로 넘어감(내부 `STALE_BID_ROUND`). */ BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED: 'BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED', @@ -86,6 +88,8 @@ export type AuctionRejectionReason = /** 내부 짧은 코드 → 설명적 reason. */ const INTERNAL_CODE_TO_REASON: Record = { + BID_CLIENT_UPDATE_REQUIRED: + AUCTION_REJECTION_REASON.BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED, AUCTION_ROSTER_INVALID: AUCTION_REJECTION_REASON.AUCTION_TRANSITION_REJECTED_BECAUSE_ROSTER_INVALID, CONFLICT: diff --git a/apps/api/src/modules/auction/auction-side-effect.service.spec.ts b/apps/api/src/modules/auction/auction-side-effect.service.spec.ts index ffdac4c0..9eae86fa 100644 --- a/apps/api/src/modules/auction/auction-side-effect.service.spec.ts +++ b/apps/api/src/modules/auction/auction-side-effect.service.spec.ts @@ -96,6 +96,7 @@ describe('auction side effect helpers', () => { 'life-1', '2', 'completed', + '', ]); }); }); diff --git a/apps/api/src/modules/auction/auction-side-effect.service.ts b/apps/api/src/modules/auction/auction-side-effect.service.ts index 950ccef9..1798164f 100644 --- a/apps/api/src/modules/auction/auction-side-effect.service.ts +++ b/apps/api/src/modules/auction/auction-side-effect.service.ts @@ -25,6 +25,7 @@ export interface BuildSideEffectLeaseAcquireUpdateParams { export interface MatchSideEffectDelta { auctionLifecycleId: string; + auctionStateTransitionId?: string; currentServerTimestampMS: number; matchSideEffectRevision: number; matchSideEffectStatus: AuctionMatchSideEffectStatus; @@ -36,6 +37,7 @@ export class AuctionSideEffectService { public async mirrorMatchSideEffectStatus(params: { auctionLifecycleId: string; + auctionStateTransitionId?: string; matchId: string; matchSideEffectRevision: number; matchSideEffectStatus: AuctionMatchSideEffectStatus; @@ -47,6 +49,8 @@ export class AuctionSideEffectService { 'EVAL', [ "if redis.call('HGET', KEYS[1], 'auctionLifecycleId') ~= ARGV[1] then return 0 end", + "local generation = redis.call('HGET', KEYS[1], 'matchSideEffectTransitionId')", + "if ARGV[4] ~= '' and generation and generation ~= ARGV[4] then return 0 end", "local currentRevision = tonumber(redis.call('HGET', KEYS[1], 'matchSideEffectRevision') or '-1')", 'local nextRevision = tonumber(ARGV[2])', 'if currentRevision >= nextRevision then return 0 end', @@ -58,6 +62,7 @@ export class AuctionSideEffectService { params.auctionLifecycleId, String(params.matchSideEffectRevision), params.matchSideEffectStatus, + params.auctionStateTransitionId ?? '', ]); return Number(result) === 1; } @@ -104,6 +109,9 @@ export function buildMatchSideEffectDelta( ): MatchSideEffectDelta { return { auctionLifecycleId: params.auctionLifecycleId, + ...(params.auctionStateTransitionId + ? { auctionStateTransitionId: params.auctionStateTransitionId } + : {}), currentServerTimestampMS: params.currentServerTimestampMS, matchSideEffectRevision: params.matchSideEffectRevision, matchSideEffectStatus: params.matchSideEffectStatus, diff --git a/apps/api/src/modules/auction/auction.controller.ts b/apps/api/src/modules/auction/auction.controller.ts index f6570d50..0f7e55a2 100644 --- a/apps/api/src/modules/auction/auction.controller.ts +++ b/apps/api/src/modules/auction/auction.controller.ts @@ -9,6 +9,7 @@ import { Param, Post, UseGuards, + Query, } from '@nestjs/common'; import mongoose from 'mongoose'; import { AllowPublicAccess } from '../auth/allow-public-access.decorator'; @@ -19,6 +20,7 @@ import { ParseObjectIdPipe } from '../common/parse-object-id.pipe'; import type { UserDocument } from '../global/database/schema'; import { AuctionService } from './auction.service'; import { AuctionGateway } from './auction.gateway'; +import { AuctionBidArchiveService } from './auction-bid-archive.service'; import { NextAuctionDto, OpenBiddingDto, @@ -35,6 +37,7 @@ export class AuctionController { constructor( private readonly auctionService: AuctionService, private readonly auctionGateway: AuctionGateway, + private readonly auctionBidArchiveService: AuctionBidArchiveService, ) {} @Post() @@ -78,6 +81,20 @@ export class AuctionController { } // 공개 — 팀장 2명만 쓸 엔드포인트(트러스트 기반, 10명 동시 관전). + @Get('bids/history') + @AllowPublicAccess() + public async getBidHistory( + @Param('matchId', ParseObjectIdPipe) matchId: mongoose.Types.ObjectId, + @Query('afterRevision') afterRevision?: string, + @Query('limit') limit?: string, + ) { + return this.auctionBidArchiveService.list( + matchId, + afterRevision === undefined ? -1 : Number(afterRevision), + limit === undefined ? 100 : Number(limit), + ); + } + @Post('open-bidding') @AllowPublicAccess() @HttpCode(HttpStatus.OK) @@ -100,6 +117,7 @@ export class AuctionController { teamIndex: body.bidderTeamIndex, amount: body.bidPointAmount, bidRequestId: body.bidRequestId, + clientObservedBidRoundId: body.clientObservedBidRoundId, clientObservedAuctionLifecycleId: body.clientObservedAuctionLifecycleId, clientObservedCurrentBidRoundPlayerUserId: body.clientObservedCurrentBidRoundPlayerUserId, @@ -213,7 +231,10 @@ export class AuctionController { clientObservedPhase: body.clientObservedPhase, clientObservedRevision: body.clientObservedRevision, }); - this.auctionGateway.broadcastReset(matchId.toString()); + this.auctionGateway.broadcastReset( + matchId.toString(), + body.clientObservedAuctionLifecycleId, + ); } private assertSameCommunity( diff --git a/apps/api/src/modules/auction/auction.gateway.spec.ts b/apps/api/src/modules/auction/auction.gateway.spec.ts index 7a8ba603..03b204b5 100644 --- a/apps/api/src/modules/auction/auction.gateway.spec.ts +++ b/apps/api/src/modules/auction/auction.gateway.spec.ts @@ -1,4 +1,5 @@ import mongoose from 'mongoose'; +import { ServiceUnavailableException } from '@nestjs/common'; jest.mock('../auth/jwt.service', () => ({ JwtService: class JwtService {}, @@ -7,6 +8,76 @@ jest.mock('../auth/jwt.service', () => ({ import { AuctionGateway } from './auction.gateway'; describe('AuctionGateway', () => { + it('signals pending initial state so the client can retry privately after recovery', async () => { + const matchId = new mongoose.Types.ObjectId().toString(); + const gateway = new AuctionGateway( + { + getByMatchId: async () => ({}), + getStateByMatchId: async () => { + throw new ServiceUnavailableException({ + code: 'AUCTION_STATE_RECOVERING', + message: 'recovering', + }); + }, + } as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + ); + const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() }; + await gateway.handleJoin(socket as never, { matchId }); + expect(socket.emit).toHaveBeenCalledTimes(1); + expect(socket.emit).toHaveBeenCalledWith('state-pending', { + retryAfterMS: 500, + }); + }); + it('sends join and resync snapshots only to the requesting socket and coalesces in-flight resyncs', async () => { + const matchId = new mongoose.Types.ObjectId().toString(); + const getStateByMatchId = jest.fn().mockResolvedValue({ revision: 3 }); + const gateway = new AuctionGateway( + { + getByMatchId: async () => ({ chatMessages: [] }), + getStateByMatchId, + } as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + ); + const broadcast = jest.fn(); + (gateway as unknown as { server: unknown }).server = { + to: () => ({ emit: broadcast }), + }; + const socket = { data: { matchId }, join: jest.fn(), emit: jest.fn() }; + await gateway.handleJoin(socket as never, { matchId }); + expect(socket.emit).toHaveBeenCalledWith('state', { revision: 3 }); + expect(broadcast).not.toHaveBeenCalled(); + let finish!: (value: unknown) => void; + getStateByMatchId.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const ack = jest.fn(), + busyAck = jest.fn(); + const pending = gateway.handleResync(socket as never, {}, ack); + await gateway.handleResync(socket as never, {}, busyAck); + expect(busyAck).toHaveBeenCalledWith( + expect.objectContaining({ code: 'RESYNC_BUSY' }), + ); + finish({ revision: 4 }); + await pending; + expect(ack).toHaveBeenCalledWith({ ok: true, state: { revision: 4 } }); + expect(getStateByMatchId).toHaveBeenCalledTimes(2); + expect(broadcast).not.toHaveBeenCalled(); + }); + function createGateway(params?: { getCurrentRedisTimestampMS?: jest.Mock; placeBidDelta?: jest.Mock; diff --git a/apps/api/src/modules/auction/auction.gateway.ts b/apps/api/src/modules/auction/auction.gateway.ts index fc5951f9..df2527e9 100644 --- a/apps/api/src/modules/auction/auction.gateway.ts +++ b/apps/api/src/modules/auction/auction.gateway.ts @@ -44,6 +44,7 @@ interface JoinAuctionPayload { } interface BidPayload { + clientObservedBidRoundId?: string; bidderTeamIndex: number; bidPointAmount: number; bidRequestId?: string; @@ -79,6 +80,8 @@ interface BidRateLimitState { } interface AuctionSocketData { + auctionResyncAtTimestampMS?: number; + auctionResyncInProgress?: boolean; auctionBidRateLimit?: BidRateLimitState; identity?: IdentityDocument; matchId?: string; @@ -162,9 +165,55 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit { new mongoose.Types.ObjectId(payload.matchId), ); this.emitChatHistory(socket, auction); - this.broadcastState(payload.matchId, auctionState); + socket.emit('state', serializeRealtimeAuction(auctionState)); } catch (error) { - this.emitKnownError(socket, error); + const rejection = this.resolveErrorPayload(error); + if (rejection.code === 'AUCTION_STATE_RECOVERING') { + socket.emit('state-pending', { retryAfterMS: 500 }); + } else { + this.emitKnownError(socket, error); + } + } + } + + @SubscribeMessage('resync') + public async handleResync( + @ConnectedSocket() socket: AuctionSocket, + @MessageBody() _payload: unknown, + @Ack() ack?: (response: Record) => void, + ) { + const matchId = this.requireJoinedMatchId(socket); + if (!matchId) { + ack?.({ ok: false, code: 'JOIN_REQUIRED' }); + return; + } + const now = Date.now(); + const retryAfterMS = + 500 - (now - (socket.data.auctionResyncAtTimestampMS ?? 0)); + if (socket.data.auctionResyncInProgress || retryAfterMS > 0) { + ack?.({ + ok: false, + code: 'RESYNC_BUSY', + retryAfterMS: Math.max(500, retryAfterMS), + }); + return; + } + socket.data.auctionResyncInProgress = true; + socket.data.auctionResyncAtTimestampMS = now; + try { + const state = await this.auctionService.getStateByMatchId( + new mongoose.Types.ObjectId(matchId), + ); + // Only the requesting socket receives the full snapshot. + ack?.({ ok: true, state: serializeRealtimeAuction(state) }); + } catch (error) { + ack?.({ + ...this.resolveErrorPayload(error), + ok: false, + retryAfterMS: 500, + }); + } finally { + socket.data.auctionResyncInProgress = false; } } @@ -196,6 +245,7 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit { teamIndex: payload.bidderTeamIndex, amount: payload.bidPointAmount, bidRequestId, + clientObservedBidRoundId: payload.clientObservedBidRoundId, clientObservedAuctionLifecycleId: payload.clientObservedAuctionLifecycleId, clientObservedCurrentBidRoundPlayerUserId: @@ -609,8 +659,11 @@ export class AuctionGateway implements OnGatewayConnection, OnGatewayInit { this.server.to(matchId).emit('bid', delta); } - public broadcastReset(matchId: string) { - this.server.to(matchId).emit('reset', { matchId }); + public broadcastReset(matchId: string, auctionLifecycleId?: string) { + this.server.to(matchId).emit('reset', { + matchId, + ...(auctionLifecycleId ? { auctionLifecycleId } : {}), + }); } public broadcastMatchSideEffect( diff --git a/apps/api/src/modules/auction/auction.module.ts b/apps/api/src/modules/auction/auction.module.ts index cdf7c595..8b0dbf9f 100644 --- a/apps/api/src/modules/auction/auction.module.ts +++ b/apps/api/src/modules/auction/auction.module.ts @@ -10,6 +10,7 @@ import { AuctionEventBusService } from './auction-event-bus.service'; import { AuctionRedisModule } from './redis/auction-redis.module'; import { AuctionSideEffectService } from './auction-side-effect.service'; import { AuctionService } from './auction.service'; +import { AuctionBidArchiveService } from './auction-bid-archive.service'; @Module({ imports: [ @@ -21,6 +22,7 @@ import { AuctionService } from './auction.service'; controllers: [AuctionController], providers: [ AuctionService, + AuctionBidArchiveService, AuctionGateway, AuctionEventBusService, AuctionDeadlineScheduler, diff --git a/apps/api/src/modules/auction/auction.service.spec.ts b/apps/api/src/modules/auction/auction.service.spec.ts index 9a8a6fad..80368dc2 100644 --- a/apps/api/src/modules/auction/auction.service.spec.ts +++ b/apps/api/src/modules/auction/auction.service.spec.ts @@ -45,6 +45,10 @@ describe('AuctionService', () => { const allModels = { auctionHistory: auctionHistoryModel, ...models, + auction: { + updateOne: jest.fn().mockResolvedValue({ matchedCount: 1 }), + ...((models.auction as object) ?? {}), + }, }; const connectionService = { getModel: jest.fn((name: string) => allModels[name]), @@ -697,7 +701,7 @@ describe('AuctionService', () => { expect(auctionRedisService.raw().sendCommand).toHaveBeenCalledWith([ 'EVAL', expect.stringContaining('currentAuctionLifecycleId'), - '11', + '12', `auction:${matchId.toString()}:liveAuctionState`, 'auction:bidRoundDeadlineQueue', `auction:${matchId.toString()}:auctionStructureState`, @@ -709,6 +713,7 @@ describe('AuctionService', () => { `auction:${matchId.toString()}:auctionActionRequestResultOrder`, `auction:${matchId.toString()}:rehydrateLock`, `auction:${matchId.toString()}:auctionStateTransitionLock`, + `auction:${matchId.toString()}:rejectedBidRequestResultOrder`, matchId.toString(), 'old-life-1', ]); @@ -909,9 +914,9 @@ describe('AuctionService', () => { shouldForceAuctionStateResync: true, }); expect(auctionRedisService.acquireTokenLock).toHaveBeenCalledWith( - `auction:${matchId.toString()}:rehydrateLock`, + `auction:${matchId.toString()}:lifecycleMutationLock`, expect.any(String), - 5000, + 30000, ); expect(auctionRedisCommandService.hydrateState).toHaveBeenCalledWith( matchId.toString(), @@ -2621,8 +2626,14 @@ describe('AuctionService', () => { auctionRedisCommandService.reserveActionRequest, ).not.toHaveBeenCalled(); expect(auctionRedisCommandService.applyTransition).not.toHaveBeenCalled(); - expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match); - expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match); + expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith( + match, + session, + ); + expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith( + match, + session, + ); expect(match).toMatchObject({ gameSchedule: [], result: undefined, @@ -2642,13 +2653,12 @@ describe('AuctionService', () => { auctionId, }); expect(auction.deleteOne).toHaveBeenCalledWith({ session }); - expect(redisClient.sendCommand).toHaveBeenCalledWith( - expect.arrayContaining([ - 'auction:bidRoundDeadlineQueue', - matchId.toString(), - 'life-1', - ]), - ); + expect(redisClient.sendCommand).not.toHaveBeenCalled(); + expect(match).toMatchObject({ + lastCancelledAuctionLifecycleId: 'life-1', + auctionCancellationCleanupPending: true, + auctionCancellationCleanupRetryAt: expect.any(Date), + }); expect(session.commitTransaction).toHaveBeenCalledTimes(1); }); @@ -2732,8 +2742,14 @@ describe('AuctionService', () => { service.undoTo({ matchId, communityId, sequence: 3 }), ).resolves.toBe(restored); - expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith(match); - expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith(match); + expect(tournamentCodeService.releaseFromMatch).toHaveBeenCalledWith( + match, + session, + ); + expect(tournamentResultService.cancelPolling).toHaveBeenCalledWith( + match, + session, + ); expect(match.status).toBe('team-built'); expect(match.result).toBeUndefined(); expect(auctionReplaceOne).toHaveBeenCalledWith( diff --git a/apps/api/src/modules/auction/auction.service.ts b/apps/api/src/modules/auction/auction.service.ts index e9e2bf81..103f992e 100644 --- a/apps/api/src/modules/auction/auction.service.ts +++ b/apps/api/src/modules/auction/auction.service.ts @@ -5,8 +5,16 @@ import { InternalServerErrorException, Logger, NotFoundException, + ServiceUnavailableException, + Optional, } from '@nestjs/common'; import { randomUUID } from 'crypto'; +import { + AuctionLifecycleLock, + AuctionLifecycleBusyError, +} from './auction-lifecycle-lock'; +import { AUCTION_OPTIONAL_PROJECTION_FIELDS } from './auction-projection'; +import { AuctionEventBusService } from './auction-event-bus.service'; import { performance } from 'perf_hooks'; import mongoose from 'mongoose'; import { ConnectionService } from '../global/database/connection.service'; @@ -58,8 +66,6 @@ const AUCTION_BID_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000; const AUCTION_MAX_STORED_ACTION_REQUEST_RESULT_COUNT = 500; const AUCTION_ACTION_REQUEST_REPLAY_WINDOW_MS = 5 * 60 * 1000; const AUCTION_ACTION_REQUEST_LEASE_DURATION_MS = 15 * 1000; -const AUCTION_REHYDRATE_LOCK_TTL_MS = 5 * 1000; -const AUCTION_REHYDRATE_LOCK_WAIT_MS = 50; const AUCTION_HISTORY_LIST_LIMIT = 100; const AUCTION_HISTORY_LIST_CACHE_TTL_MS = 2 * 1000; const AUCTION_HISTORY_LIST_CACHE_MAX_ENTRIES = 256; @@ -82,6 +88,7 @@ interface StartAuctionParams { } interface PlaceBidParams { + clientObservedBidRoundId?: string; matchId: mongoose.Types.ObjectId; teamIndex: number; amount: number; @@ -160,6 +167,7 @@ interface AuctionHistoryListCacheEntry { export type AuctionStateResponse = AuctionDocument | Record; export interface AuctionBidDeltaResponse { + currentBidRoundId?: string; auctionBidHistoryEntry: Record; auctionLifecycleId?: string; bidServerTiming?: AuctionBidServerTiming; @@ -212,9 +220,20 @@ export class AuctionService { private readonly auctionRedisService: AuctionRedisService, private readonly auctionRedisCommandService: AuctionRedisCommandService, private readonly auctionRedisStateReaderService: AuctionRedisStateReaderService, + @Optional() + private readonly auctionEventBusService?: AuctionEventBusService, ) {} public async start(params: StartAuctionParams): Promise { + return new AuctionLifecycleLock(this.auctionRedisService).run( + params.matchId.toString(), + () => this.startLocked(params), + ); + } + + private async startLocked( + params: StartAuctionParams, + ): Promise { const MatchModel = await this.connectionService.getModel('match'); const UserModel = await this.connectionService.getModel('user'); const AuctionModel = await this.connectionService.getModel('auction'); @@ -231,7 +250,7 @@ export class AuctionService { const existing = await AuctionModel.findOne({ matchId: params.matchId }); if (existing) { if (existing.startAuctionRequestId === params.startAuctionRequestId) { - return existing; + return this.finishAuctionStartup(existing, match); } if (this.isRestartableResetRedisAuction(existing)) { await this.clearRedisAuctionRuntimeForLifecycle( @@ -249,6 +268,19 @@ export class AuctionService { } } + // No Mongo auction owns this match. Remove runtime left by an interrupted + // cancellation before creating a new lifecycle (under the lifecycle lock). + const orphanedRuntime = + await this.auctionRedisStateReaderService.readLiveStateIdentity( + params.matchId.toString(), + ); + if (orphanedRuntime?.auctionLifecycleId) { + await this.clearRedisAuctionRuntimeForLifecycle( + params.matchId.toString(), + orphanedRuntime.auctionLifecycleId, + ); + } + const activeParticipants = this.applyAuctionCut(match); const desiredAuctionCount = activeParticipants.length; if (desiredAuctionCount < 10) { @@ -381,6 +413,7 @@ export class AuctionService { auctionStartedByUserId: params.actorUserId, matchSideEffectStatus: 'none', matchSideEffectRevision: 0, + startupCompleted: false, teams: legacyTeams, order: prioritizedOrder, passedQueue: [], @@ -419,12 +452,12 @@ export class AuctionService { matchId: params.matchId, }); if (raceWinner?.startAuctionRequestId === params.startAuctionRequestId) { - return raceWinner; + return this.finishAuctionStartup(raceWinner, match); } throw new BadRequestException('Auction already exists for this match.'); } await this.recordSnapshot(auction, 'start'); - await this.auctionRedisCommandService.hydrateState( + const hydration = await this.auctionRedisCommandService.hydrateState( params.matchId.toString(), { auctionHistorySequence: 1, @@ -437,9 +470,18 @@ export class AuctionService { structureStateJson, }, ); + if ('rejected' in hydration) + throw new ConflictException( + `Auction initialization failed: ${hydration.code}.`, + ); match.status = 'team-built'; await match.save(); + auction.startupCompleted = true; + await AuctionModel.updateOne( + { _id: auction._id }, + { $set: { startupCompleted: true } }, + ); this.enqueueBackgroundOrderEnrichment({ auctionId: auction._id, targets: enrichTargets, @@ -450,6 +492,61 @@ export class AuctionService { return auction; } + private async finishAuctionStartup( + auction: AuctionDocument, + match: MatchDocument, + ): Promise { + // Documents predating the startup marker already completed the old path. + if (auction.startupCompleted !== false) return auction; + // Retry the original cut; recruiting may have changed since Mongo creation. + const roster = new Set([ + ...auction.teams.map((team) => team.captainUserId.toString()), + ...auction.order.map((player) => player.userId.toString()), + ]); + const participants = new Set( + match.participants.map((entry) => entry.userId.toString()), + ); + if ([...roster].some((id) => !participants.has(id))) { + throw new ConflictException( + 'Auction startup roster has changed. Cancel and restart the auction.', + ); + } + const History = await this.connectionService.getModel('auctionHistory'); + await History.updateOne( + { auctionId: auction._id, sequence: 1 }, + { + $setOnInsert: { + auctionId: auction._id, + matchId: auction.matchId, + sequence: 1, + action: 'start', + snapshotBefore: this.cloneJson( + typeof auction.toObject === 'function' + ? auction.toObject() + : auction, + ), + takenAt: auction.startedAt ?? new Date(), + }, + }, + { upsert: true }, + ); + await this.hydrateRedisStateFromAuctionDocument(auction); + if (match.status === 'recruiting') { + match.participants.forEach((participant) => { + participant.excluded = !roster.has(participant.userId.toString()); + }); + match.status = 'team-built'; + await match.save(); + } + const AuctionModel = await this.connectionService.getModel('auction'); + await AuctionModel.updateOne( + { _id: auction._id }, + { $set: { startupCompleted: true } }, + ); + auction.startupCompleted = true; + return auction; + } + public async getByMatchId( matchId: mongoose.Types.ObjectId, ): Promise { @@ -462,18 +559,46 @@ export class AuctionService { public async getStateByMatchId( matchId: mongoose.Types.ObjectId, ): Promise { - const auction = await this.getByMatchId(matchId); + for (let attempt = 0; ; attempt++) { + try { + return await this.getStateByMatchIdOnce(matchId); + } catch (error) { + if (!(error instanceof AuctionLifecycleBusyError)) throw error; + if (attempt >= 4) + throw new ServiceUnavailableException({ + code: 'AUCTION_STATE_RECOVERING', + retryAfterMS: 500, + message: '경매 상태를 복구 중입니다. 잠시 후 다시 시도해 주세요.', + }); + await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1))); + } + } + } + + private async getStateByMatchIdOnce( + matchId: mongoose.Types.ObjectId, + ): Promise { + let auction = await this.getByMatchId(matchId); if (!this.isRedisV1Auction(auction)) return auction; + if (auction.startupCompleted === false) { + auction = await new AuctionLifecycleLock(this.auctionRedisService).run( + matchId.toString(), + async () => { + const current = await this.getByMatchId(matchId); + const MatchModel = await this.connectionService.getModel('match'); + const match = await MatchModel.findById(matchId); + if (!match) throw new NotFoundException('Match not found.'); + return this.finishAuctionStartup(current, match); + }, + ); + } + const redisState = await this.readRedisAuctionStateOrRehydrate(auction); if (!redisState) return auction; - const mongoState = - typeof auction.toObject === 'function' - ? auction.toObject({ depopulate: true }) - : auction; return { - ...this.cloneJson(mongoState), + ...this.buildAuctionSnapshotFromRedisState(auction, redisState), ...redisState, chatMessages: auction.chatMessages ?? [], }; @@ -485,12 +610,15 @@ export class AuctionService { const matchId = auction.matchId.toString(); const mongoRevision = auction.revision ?? 0; const mongoAuctionLifecycleId = auction.auctionLifecycleId; - if (!mongoAuctionLifecycleId || auction.phase === 'complete') { + if (!mongoAuctionLifecycleId) { return this.auctionRedisStateReaderService.readAuctionStateView(matchId); } const liveIdentity = await this.auctionRedisStateReaderService.readLiveStateIdentity(matchId); + // Completed runtimes expire after the retention period. Mongo remains the + // archive; viewing it must not create an immortal Redis runtime again. + if (!liveIdentity && auction.phase === 'complete') return null; const shouldHydrate = shouldHydrateRedisAuctionState({ live: liveIdentity, mongo: { @@ -503,29 +631,32 @@ export class AuctionService { return this.auctionRedisStateReaderService.readAuctionStateView(matchId); } - const keys = buildAuctionRedisKeys(matchId); - const lockToken = randomUUID(); - const lockAcquired = await this.auctionRedisService.acquireTokenLock( - keys.rehydrateLockKey, - lockToken, - AUCTION_REHYDRATE_LOCK_TTL_MS, + await new AuctionLifecycleLock(this.auctionRedisService).run( + matchId, + async () => { + const currentAuction = await this.getByMatchId(auction.matchId); + if (currentAuction.auctionLifecycleId !== mongoAuctionLifecycleId) { + throw new ConflictException( + 'Auction lifecycle has changed. Refresh the auction.', + ); + } + const currentLive = + await this.auctionRedisStateReaderService.readLiveStateIdentity( + matchId, + ); + if ( + currentLive?.auctionLifecycleId && + currentLive.auctionLifecycleId !== mongoAuctionLifecycleId + ) { + await this.clearRedisAuctionRuntimeForLifecycle( + matchId, + currentLive.auctionLifecycleId, + ); + } + await this.hydrateRedisStateFromAuctionDocument(currentAuction); + }, ); - if (lockAcquired) { - try { - await this.hydrateRedisStateFromAuctionDocument(auction); - } finally { - await this.auctionRedisService.releaseTokenLock( - keys.rehydrateLockKey, - lockToken, - ); - } - } else { - await new Promise((resolve) => - setTimeout(resolve, AUCTION_REHYDRATE_LOCK_WAIT_MS), - ); - } - return this.auctionRedisStateReaderService.readAuctionStateView(matchId, { shouldForceAuctionStateResync: true, }); @@ -814,6 +945,7 @@ export class AuctionService { bidderTeamIndex: params.teamIndex, bidderUserId: bidContext.bidderTeamCaptainUserId, bidRequestId: params.bidRequestId ?? randomUUID(), + clientObservedBidRoundId: params.clientObservedBidRoundId, clientObservedAuctionLifecycleId: auctionLifecycleId, clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId, matchId, @@ -890,6 +1022,7 @@ export class AuctionService { bidderTeamIndex: params.teamIndex, bidderUserId: this.stringFromState(team.captainUserId), bidRequestId: params.bidRequestId ?? randomUUID(), + clientObservedBidRoundId: params.clientObservedBidRoundId, clientObservedAuctionLifecycleId: auctionLifecycleId, clientObservedCurrentBidRoundPlayerUserId: currentBidRoundPlayerUserId, matchId, @@ -923,6 +1056,10 @@ export class AuctionService { ): AuctionBidDeltaResponse { return { auctionBidHistoryEntry: result.auctionBidHistoryEntry, + currentBidRoundId: + typeof result.auctionBidHistoryEntry.bidRoundId === 'string' + ? result.auctionBidHistoryEntry.bidRoundId + : undefined, auctionLifecycleId: this.stringFromState(state.auctionLifecycleId), bidRequestId: result.bidRequestId, currentBidRoundDeadlineTimestampMS: @@ -1408,6 +1545,34 @@ export class AuctionService { matchId: mongoose.Types.ObjectId; communityId: mongoose.Types.ObjectId; }): Promise { + let committed = false; + try { + await new AuctionLifecycleLock(this.auctionRedisService).run( + params.matchId.toString(), + () => + this.resetLocked(params, () => { + committed = true; + }), + ); + } catch (error) { + if (!committed) throw error; + // A committed cancellation is successful even if lock renewal/teardown + // fails afterward. Cleanup is durably queued in the same Mongo transaction. + this.logger.warn( + `Auction cancellation committed; post-commit coordination failed: ${String(error)}`, + ); + } + } + + private async resetLocked( + params: { + matchId: mongoose.Types.ObjectId; + communityId: mongoose.Types.ObjectId; + clientObservedAuctionLifecycleId?: string; + clientObservedRevision?: number; + }, + onCommitted: () => void, + ): Promise { const AuctionModel = await this.connectionService.getModel('auction'); const AuctionHistoryModel = await this.connectionService.getModel('auctionHistory'); @@ -1419,70 +1584,155 @@ export class AuctionService { if (!session) { throw new InternalServerErrorException('Failed to start DB session.'); } - let redisRuntimeToClear: { - auctionLifecycleId?: string; - matchId: string; - } | null = null; try { session.startTransaction(); const auction = await AuctionModel.findOne({ matchId: params.matchId, }).session(session); - if (!auction) throw new NotFoundException('Auction not found.'); - - const match = await MatchModel.findById(params.matchId).session(session); - if (!match) throw new NotFoundException('Match not found.'); - if (!match.communityId.equals(params.communityId)) { - throw new BadRequestException('Community mismatch.'); - } - this.assertCompetitionLifecycleMutable(match, 'reset the auction'); - - if (this.isRedisV1Auction(auction)) { - redisRuntimeToClear = { - auctionLifecycleId: auction.auctionLifecycleId, - matchId: params.matchId.toString(), - }; - await this.resetMatchAfterAuctionCancellation(match, session); - await AuctionSideEffectApplicationModel.deleteMany({ - auctionId: auction._id, - }).session(session); - await AuctionHistoryModel.deleteMany({ - auctionId: auction._id, - }).session(session); - await auction.deleteOne({ session }); + if (!auction) { + // 이미 커밋된 취소의 재전송이면 성공으로 응답하고 남은 Redis 정리만 재시도한다. + const cancelledMatch = params.clientObservedAuctionLifecycleId + ? await MatchModel.findById(params.matchId).session(session) + : null; + if ( + !cancelledMatch || + cancelledMatch.lastCancelledAuctionLifecycleId !== + params.clientObservedAuctionLifecycleId + ) { + throw new NotFoundException('Auction not found.'); + } + if (!cancelledMatch.communityId.equals(params.communityId)) { + throw new BadRequestException('Community mismatch.'); + } + cancelledMatch.auctionCancellationCleanupPending = true; + cancelledMatch.auctionCancellationCleanupRetryAt = new Date(); + await cancelledMatch.save({ session }); } else { - await auction.deleteOne({ session }); - await AuctionHistoryModel.deleteMany({ - auctionId: auction._id, - }).session(session); - if (match.status === 'team-built') { - await this.tournamentCodeService.releaseFromMatch(match); - match.status = 'recruiting'; - match.teams = []; - match.tournamentCodeIds = []; - match.gameSchedule = []; - match.participants.forEach((participant) => { - participant.excluded = false; - }); - match.teamBuiltAt = undefined; - match.result = undefined; - match.resultPollState = undefined; - await match.save({ session }); + if ( + this.isRedisV1Auction(auction) && + (!params.clientObservedAuctionLifecycleId || + params.clientObservedAuctionLifecycleId !== + auction.auctionLifecycleId) + ) { + throw new ConflictException( + 'Auction lifecycle has changed. Refresh before cancelling.', + ); + } + + const match = await MatchModel.findById(params.matchId).session( + session, + ); + if (!match) throw new NotFoundException('Match not found.'); + if (!match.communityId.equals(params.communityId)) { + throw new BadRequestException('Community mismatch.'); + } + this.assertCompetitionLifecycleMutable(match, 'reset the auction'); + + if (this.isRedisV1Auction(auction)) { + // 같은 트랜잭션에서 취소 완료를 기록해, 재전송을 멱등 처리한다. + match.lastCancelledAuctionLifecycleId = auction.auctionLifecycleId; + match.auctionCancellationCleanupPending = true; + match.auctionCancellationCleanupRetryAt = new Date(); + match.auctionCancellationCleanupAttempts = 0; + match.auctionCancellationCleanupError = undefined; + await this.resetMatchAfterAuctionCancellation(match, session); + await AuctionSideEffectApplicationModel.deleteMany({ + auctionId: auction._id, + }).session(session); + await AuctionHistoryModel.deleteMany({ + auctionId: auction._id, + }).session(session); + await auction.deleteOne({ session }); + } else { + await auction.deleteOne({ session }); + await AuctionHistoryModel.deleteMany({ + auctionId: auction._id, + }).session(session); + if (match.status === 'team-built') { + await this.tournamentCodeService.releaseFromMatch(match, session); + match.status = 'recruiting'; + match.teams = []; + match.tournamentCodeIds = []; + match.gameSchedule = []; + match.participants.forEach((participant) => { + participant.excluded = false; + }); + match.teamBuiltAt = undefined; + match.result = undefined; + match.resultPollState = undefined; + await match.save({ session }); + } } } await session.commitTransaction(); + onCommitted(); } catch (err) { await session.abortTransaction(); throw err; } finally { await session.endSession(); } + } - if (redisRuntimeToClear) { - await this.clearRedisAuctionRuntimeForLifecycle( - redisRuntimeToClear.matchId, - redisRuntimeToClear.auctionLifecycleId, - ); + public async retryPendingCancellations(batchLimit = 20): Promise { + const MatchModel = await this.connectionService.getModel('match'); + const pending = await MatchModel.find({ + auctionCancellationCleanupPending: true, + auctionCancellationCleanupRetryAt: { $lte: new Date() }, + }) + .sort({ auctionCancellationCleanupRetryAt: 1 }) + .limit(batchLimit) + .lean() + .exec(); + for (const match of pending) { + const lifecycle = match.lastCancelledAuctionLifecycleId; + if (!lifecycle) continue; + const filter = { + _id: match._id, + lastCancelledAuctionLifecycleId: lifecycle, + }; + try { + await new AuctionLifecycleLock(this.auctionRedisService).run( + match._id.toString(), + async () => { + // Compare-and-delete in Redis also protects a newly started lifecycle. + await this.clearRedisAuctionRuntimeForLifecycle( + match._id.toString(), + lifecycle, + ); + const AuctionModel = + await this.connectionService.getModel('auction'); + if (!(await AuctionModel.exists({ matchId: match._id }))) { + this.auctionEventBusService?.broadcastReset( + match._id.toString(), + lifecycle, + ); + } + await MatchModel.updateOne(filter, { + $set: { auctionCancellationCleanupPending: false }, + $unset: { + auctionCancellationCleanupRetryAt: '', + auctionCancellationCleanupError: '', + }, + }); + }, + ); + } catch (error) { + const attempts = (match.auctionCancellationCleanupAttempts ?? 0) + 1; + await MatchModel.updateOne(filter, { + $set: { + auctionCancellationCleanupPending: true, + auctionCancellationCleanupRetryAt: new Date( + Date.now() + Math.min(60000, 1000 * 2 ** Math.min(attempts, 6)), + ), + auctionCancellationCleanupError: String(error).slice(0, 2000), + }, + $inc: { auctionCancellationCleanupAttempts: 1 }, + }); + this.logger.warn( + `Auction cancellation cleanup will retry for ${match._id.toString()}: ${String(error)}`, + ); + } } } @@ -1497,6 +1747,19 @@ export class AuctionService { communityId: mongoose.Types.ObjectId; sequence: number; }): Promise { + return new AuctionLifecycleLock(this.auctionRedisService).run( + params.matchId.toString(), + () => this.undoToLocked(params), + ); + } + + private async undoToLocked( + params: AuctionStructureActionRequestParams & { + matchId: mongoose.Types.ObjectId; + communityId: mongoose.Types.ObjectId; + sequence: number; + }, + ): Promise { const AuctionModel = await this.connectionService.getModel('auction'); const AuctionHistoryModel = await this.connectionService.getModel('auctionHistory'); @@ -1508,6 +1771,7 @@ export class AuctionService { } let redisUndoContext: { + state?: Record; actionRequestParams?: AuctionStructureActionRequestParams; auction: AuctionDocument; snapshot: Record; @@ -1543,8 +1807,12 @@ export class AuctionService { throw new BadRequestException('Auction history snapshot is invalid.'); } - const undoingCompletedAuction = - Boolean(auction.completedAt) && !('completedAt' in snapshot); + const liveState = this.isRedisV1Auction(auction) + ? await this.requireRedisAuctionState(params.matchId.toString()) + : undefined; + const undoingCompletedAuction = liveState + ? liveState.phase === 'complete' && snapshot.phase !== 'complete' + : Boolean(auction.completedAt) && !('completedAt' in snapshot); if (undoingCompletedAuction) { this.assertCompetitionLifecycleMutable( match, @@ -1553,6 +1821,7 @@ export class AuctionService { } if (this.isRedisV1Auction(auction)) { redisUndoContext = { + state: liveState, auction, actionRequestParams: params, snapshot: snapshot, @@ -1562,8 +1831,8 @@ export class AuctionService { await session.commitTransaction(); } else { if (undoingCompletedAuction) { - await this.tournamentCodeService.releaseFromMatch(match); - await this.tournamentResultService.cancelPolling(match); + await this.tournamentCodeService.releaseFromMatch(match, session); + await this.tournamentResultService.cancelPolling(match, session); match.status = 'team-built'; match.result = undefined; await match.save({ session }); @@ -1758,6 +2027,14 @@ export class AuctionService { currentAuctionHistorySequence: auctionHistorySequence, lastAppliedAuctionStateTransitionId: auctionStateTransitionId, revision: expectedRevision + 1, + currentBidRoundId: + params.deadline.mode === 'set-relative' + ? auctionStateTransitionId + : params.state.currentBidRoundId, + currentBidRoundDeadlineTimestampMS: + params.deadline.mode === 'remove' + ? 0 + : params.state.currentBidRoundDeadlineTimestampMS, }; const afterAuctionMongoProjection = this.buildAuctionMongoProjectionFromState(params.auction, afterState); @@ -1769,8 +2046,18 @@ export class AuctionService { if (matchSideEffectMutation.type !== 'none') { afterAuctionMongoProjection.matchSideEffectStatus = 'pending'; afterAuctionMongoProjection.matchSideEffectRevision = - (params.auction.matchSideEffectRevision ?? 0) + 1; + Math.max( + params.auction.matchSideEffectRevision ?? 0, + this.numberFromState(params.state.matchSideEffectRevision), + ) + 1; afterAuctionMongoProjection.matchSideEffectError = undefined; + afterAuctionMongoProjection.matchSideEffectTransitionId = + auctionStateTransitionId; + params.liveStatePatch.matchSideEffectTransitionId = + auctionStateTransitionId; + params.liveStatePatch.matchSideEffectStatus = 'pending'; + params.liveStatePatch.matchSideEffectRevision = + afterAuctionMongoProjection.matchSideEffectRevision as number; } const durabilityPayload = { @@ -1925,6 +2212,7 @@ export class AuctionService { } private async undoRedisAuction(params: { + state?: Record; actionRequestParams?: AuctionStructureActionRequestParams; auction: AuctionDocument; snapshot: Record; @@ -1932,7 +2220,8 @@ export class AuctionService { undoingCompletedAuction: boolean; }): Promise { const matchId = params.auction.matchId.toString(); - const state = await this.requireRedisAuctionState(matchId); + const state = + params.state ?? (await this.requireRedisAuctionState(matchId)); const actionRequest = await this.reserveRedisStructureActionRequest({ action: 'undo', matchId, @@ -2118,8 +2407,11 @@ export class AuctionService { typeof auction.toObject === 'function' ? (auction.toObject({ depopulate: true }) as Record) : (auction as unknown as Record); + const snapshotBase = this.cloneJson(base); + for (const field of AUCTION_OPTIONAL_PROJECTION_FIELDS) + delete snapshotBase[field]; return { - ...this.cloneJson(base), + ...snapshotBase, ...this.buildAuctionMongoProjectionFromState(auction, state), _id: auction._id, matchId: auction.matchId, @@ -2145,6 +2437,7 @@ export class AuctionService { ); const phase = this.stringFromState(state.phase) || 'presenting'; const projection: Record = { + currentBidRoundId: state.currentBidRoundId || undefined, auctionBidHistory, auctionCompletedAtTimestampMS: phase === 'complete' && @@ -2285,6 +2578,8 @@ export class AuctionService { private formatRedisBidRejectCode(code: string): string { switch (code) { + case 'BID_CLIENT_UPDATE_REQUIRED': + return '경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.'; case 'PHASE': return 'Auction is not in bidding phase.'; case 'STALE_BID_ROUND': @@ -2701,8 +2996,8 @@ export class AuctionService { match: MatchDocument, session: mongoose.ClientSession, ): Promise { - await this.tournamentCodeService.releaseFromMatch(match); - await this.tournamentResultService.cancelPolling(match); + await this.tournamentCodeService.releaseFromMatch(match, session); + await this.tournamentResultService.cancelPolling(match, session); match.status = 'recruiting'; match.teams = []; match.tournamentCodeIds = []; @@ -2717,6 +3012,11 @@ export class AuctionService { } private assertLiveAuctionOperationSupported(auction: AuctionDocument): void { + if (auction.startupCompleted === false) { + throw new ConflictException( + 'Auction startup is not complete. Refresh to resume initialization.', + ); + } if (auction.auctionEngineVersion && !this.isRedisV1Auction(auction)) { throw new BadRequestException( 'Legacy auction live operations are not supported.', @@ -2881,7 +3181,7 @@ export class AuctionService { auction.topTierThreshold ?? 'NONE', }; - await this.auctionRedisCommandService.hydrateState( + const hydration = await this.auctionRedisCommandService.hydrateState( auction.matchId.toString(), { auctionHistorySequence: auction.currentAuctionHistorySequence ?? 0, @@ -2889,6 +3189,10 @@ export class AuctionService { auction.auctionLifecycleId ?? `auction:${auction._id.toString()}:rehydrated`, liveStatePatch: { + currentBidRoundId: auction.phase === 'bidding' ? randomUUID() : '', + matchSideEffectTransitionId: auction.matchSideEffectTransitionId, + matchSideEffectRevision: auction.matchSideEffectRevision, + matchSideEffectStatus: auction.matchSideEffectStatus, ...buildAuctionRedisLiveStatePatchFromStructure(structureInput), currentHighestBidPointAmount: auction.currentHighestBidPointAmount ?? auction.currentBid ?? 0, @@ -2905,6 +3209,10 @@ export class AuctionService { structureStateJson: stringifyAuctionRedisStructureState(structureInput), }, ); + if ('rejected' in hydration) + throw new ConflictException( + `Auction recovery failed: ${hydration.code}.`, + ); } private async clearRedisAuctionRuntimeForLifecycle( @@ -2925,10 +3233,10 @@ export class AuctionService { ' end', 'end', "redis.call('ZREM', KEYS[2], ARGV[1])", - "redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11])", + "redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9], KEYS[10], KEYS[11], KEYS[12])", 'return 1', ].join('\n'), - '11', + '12', keys.liveStateKey, keys.deadlineQueueKey, keys.structureStateKey, @@ -2940,6 +3248,7 @@ export class AuctionService { keys.actionRequestResultOrderKey, keys.rehydrateLockKey, keys.stateTransitionLockKey, + keys.rejectedBidRequestResultOrderKey, matchId, expectedAuctionLifecycleId ?? '', ]); diff --git a/apps/api/src/modules/auction/dto/auction.dto.ts b/apps/api/src/modules/auction/dto/auction.dto.ts index d7a58e7f..48c512ed 100644 --- a/apps/api/src/modules/auction/dto/auction.dto.ts +++ b/apps/api/src/modules/auction/dto/auction.dto.ts @@ -72,6 +72,10 @@ export class StartAuctionDto { } export class PlaceBidDto { + @IsOptional() + @IsString() + clientObservedBidRoundId?: string; + @IsOptional() @IsUUID() bidRequestId?: string; diff --git a/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.spec.ts b/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.spec.ts index ed605a36..3a9eeea7 100644 --- a/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.spec.ts +++ b/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.spec.ts @@ -7,7 +7,13 @@ function makeService( ) { const sendCommand = jest .fn() - .mockImplementation(() => Promise.resolve(commandReplies.shift())); + .mockImplementation((command: string[]) => + Promise.resolve( + command[0] === 'EVAL' + ? [currentServerTimestampMS, ...commandReplies.splice(0, 3)] + : commandReplies.shift(), + ), + ); const auctionRedisService = { getCurrentRedisTimestampMS: jest .fn() @@ -125,19 +131,14 @@ describe('AuctionRedisStateReaderService', () => { }); expect(view).not.toHaveProperty('team:2:remainingBidPointAmount'); expect(view).not.toHaveProperty('bidRoundExpiryDecision'); - expect(sendCommand).toHaveBeenNthCalledWith(1, [ - 'HGETALL', + expect(sendCommand).toHaveBeenCalledTimes(1); + expect(sendCommand).toHaveBeenCalledWith([ + 'EVAL', + expect.stringContaining("redis.call('TIME')"), + '3', 'auction:match-1:liveAuctionState', - ]); - expect(sendCommand).toHaveBeenNthCalledWith(2, [ - 'GET', 'auction:match-1:auctionStructureState', - ]); - expect(sendCommand).toHaveBeenNthCalledWith(3, [ - 'LRANGE', 'auction:match-1:auctionBidHistory', - '0', - '-1', ]); }); diff --git a/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.ts b/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.ts index cff926bf..09acdbcc 100644 --- a/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.ts +++ b/apps/api/src/modules/auction/redis/auction-redis-state-reader.service.ts @@ -111,18 +111,11 @@ export class AuctionRedisStateReaderService { liveReply, structureStateJson, bidEntries, - ] = await Promise.all([ - this.auctionRedisService.getCurrentRedisTimestampMS(), - this.auctionRedisService - .raw() - .sendCommand(['HGETALL', keys.liveStateKey]), - this.auctionRedisService - .raw() - .sendCommand(['GET', keys.structureStateKey]), - this.auctionRedisService - .raw() - .sendCommand(['LRANGE', keys.bidHistoryKey, '0', '-1']), - ]); + ] = await this.auctionRedisService + .raw() + .sendCommand< + unknown[] + >(['EVAL', ["local now = redis.call('TIME')", 'local timestamp = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)', "return {tostring(timestamp), redis.call('HGETALL', KEYS[1]), redis.call('GET', KEYS[2]), redis.call('LRANGE', KEYS[3], 0, -1)}"].join('\n'), '3', keys.liveStateKey, keys.structureStateKey, keys.bidHistoryKey]); const liveState = parseHashReply(liveReply, { includeInternalLiveStateFields: @@ -144,7 +137,7 @@ export class AuctionRedisStateReaderService { auctionBidHistoryEntries: Array.isArray(bidEntries) ? bidEntries.map(String) : [], - currentServerTimestampMS, + currentServerTimestampMS: Number(currentServerTimestampMS), liveState: liveState as Record & { auctionLifecycleId: string; phase: string; diff --git a/apps/api/src/modules/auction/redis/auction-redis.apply-transition.spec.ts b/apps/api/src/modules/auction/redis/auction-redis.apply-transition.spec.ts index 239a222a..50611785 100644 --- a/apps/api/src/modules/auction/redis/auction-redis.apply-transition.spec.ts +++ b/apps/api/src/modules/auction/redis/auction-redis.apply-transition.spec.ts @@ -46,6 +46,7 @@ describe('auction apply transition redis script contract', () => { 'auction:match-1:bidRequestResults', 'auction:match-1:bidRequestResultOrder', 'auction:bidRoundDeadlineQueue', + 'auction:match-1:rejectedBidRequestResultOrder', ]); expect(invocation.values.slice(0, 19)).toEqual([ 'match-1', diff --git a/apps/api/src/modules/auction/redis/auction-redis.apply-transition.ts b/apps/api/src/modules/auction/redis/auction-redis.apply-transition.ts index 4cdbb153..4c29f33d 100644 --- a/apps/api/src/modules/auction/redis/auction-redis.apply-transition.ts +++ b/apps/api/src/modules/auction/redis/auction-redis.apply-transition.ts @@ -79,6 +79,7 @@ export function buildAuctionApplyTransitionScriptInvocation( keys.bidRequestResultsKey, keys.bidRequestResultOrderKey, keys.deadlineQueueKey, + keys.rejectedBidRequestResultOrderKey, ], script: AUCTION_APPLY_TRANSITION_SCRIPT, values: [ @@ -128,6 +129,7 @@ local auctionBidHistoryKey = KEYS[8] local bidRequestResultsKey = KEYS[9] local bidRequestResultOrderKey = KEYS[10] local bidRoundDeadlineQueueKey = KEYS[11] +local rejectedBidRequestResultOrderKey = KEYS[12] local matchId = ARGV[1] local auctionLifecycleId = ARGV[2] @@ -270,16 +272,27 @@ for i = 1, liveStatePatchPairCount do end redis.call('HSET', unpack(hsetArgs)) +local committedBidRoundDeadlineTimestampMS = '' if deadlineMode == 'set-relative' then local currentBidRoundDeadlineTimestampMS = currentRedisTimestampMS + (deadlineDurationSeconds * 1000) + committedBidRoundDeadlineTimestampMS = tostring(currentBidRoundDeadlineTimestampMS) redis.call('HSET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS', tostring(currentBidRoundDeadlineTimestampMS)) + redis.call('HSET', liveAuctionStateKey, 'currentBidRoundId', auctionStateTransitionId) redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId) elseif deadlineMode == 'remove' then redis.call('ZREM', bidRoundDeadlineQueueKey, matchId) + redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') +end + +-- A resumed auction must not disappear when its former completion TTL elapses. +if redis.call('HGET', liveAuctionStateKey, 'phase') ~= 'complete' then + for _, index in ipairs({1, 2, 3, 6, 7, 8, 9, 10, 12}) do + redis.call('PERSIST', KEYS[index]) + end end if shouldResetBidRuntime == '1' then - redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey) + redis.call('DEL', auctionBidHistoryKey, bidRequestResultsKey, bidRequestResultOrderKey, rejectedBidRequestResultOrderKey) end redis.call('SADD', appliedAuctionStateTransitionIdsKey, transitionDedupeMember) @@ -288,6 +301,8 @@ redis.call( durabilityPayloadKey, 'payloadJson', durabilityPayloadJson, + 'committedBidRoundDeadlineTimestampMS', + committedBidRoundDeadlineTimestampMS, 'preparedAtRedisTimestampMS', tostring(currentRedisTimestampMS), 'committedAtRedisTimestampMS', diff --git a/apps/api/src/modules/auction/redis/auction-redis.bid.spec.ts b/apps/api/src/modules/auction/redis/auction-redis.bid.spec.ts index 9a71e098..d8e6537f 100644 --- a/apps/api/src/modules/auction/redis/auction-redis.bid.spec.ts +++ b/apps/api/src/modules/auction/redis/auction-redis.bid.spec.ts @@ -30,6 +30,9 @@ describe('auction bid redis script contract', () => { 'auction:match-1:bidRequestResults', 'auction:match-1:bidRequestResultOrder', 'auction:bidRoundDeadlineQueue', + 'auction:match-1:rejectedBidRequestResultOrder', + 'auction:bidArchiveQueue', + 'auction:bidArchivePayloads', ]); expect(invocation.values).toEqual([ 'match-1', @@ -45,6 +48,7 @@ describe('auction bid redis script contract', () => { '100', '60000', '200', + '', ]); }); diff --git a/apps/api/src/modules/auction/redis/auction-redis.bid.ts b/apps/api/src/modules/auction/redis/auction-redis.bid.ts index 882fce34..15f1e829 100644 --- a/apps/api/src/modules/auction/redis/auction-redis.bid.ts +++ b/apps/api/src/modules/auction/redis/auction-redis.bid.ts @@ -1,4 +1,8 @@ import type { AuctionRedisKeys } from './auction-redis.keys'; +import { + AUCTION_BID_ARCHIVE_QUEUE_KEY, + AUCTION_BID_ARCHIVE_PAYLOADS_KEY, +} from './auction-redis.keys'; import { defineAuctionRedisScript } from './auction-redis.scripts'; export interface AuctionBidScriptInput { @@ -8,6 +12,7 @@ export interface AuctionBidScriptInput { bidderUserId: string; clientObservedAuctionLifecycleId: string; clientObservedCurrentBidRoundPlayerUserId: string; + clientObservedBidRoundId?: string; maximumAuctionTeamMemberCount: number; maximumStoredAuctionBidHistoryCount: number; maximumStoredBidRequestResultCount: number; @@ -40,15 +45,19 @@ export function buildBidRequestPayloadFingerprint(params: { bidPointAmount: number; clientObservedAuctionLifecycleId: string; clientObservedCurrentBidRoundPlayerUserId: string; + clientObservedBidRoundId?: string; matchId: string; }): string { - return [ + const parts = [ params.matchId, params.clientObservedAuctionLifecycleId, params.clientObservedCurrentBidRoundPlayerUserId, params.bidderTeamIndex, params.bidPointAmount, - ].join('|'); + ]; + if (params.clientObservedBidRoundId) + parts.push(params.clientObservedBidRoundId); + return parts.join('|'); } export function buildAuctionBidScriptInvocation( @@ -62,6 +71,9 @@ export function buildAuctionBidScriptInvocation( keys.bidRequestResultsKey, keys.bidRequestResultOrderKey, keys.deadlineQueueKey, + keys.rejectedBidRequestResultOrderKey, + AUCTION_BID_ARCHIVE_QUEUE_KEY, + AUCTION_BID_ARCHIVE_PAYLOADS_KEY, ], script: AUCTION_BID_SCRIPT, values: [ @@ -78,6 +90,7 @@ export function buildAuctionBidScriptInvocation( String(input.maximumStoredBidRequestResultCount), String(input.minimumBidRequestResultReplayWindowMS), String(input.maximumStoredAuctionBidHistoryCount), + input.clientObservedBidRoundId ?? '', ], }; } @@ -90,6 +103,9 @@ local auctionBidHistoryKey = KEYS[2] local bidRequestResultsKey = KEYS[3] local bidRequestResultOrderKey = KEYS[4] local bidRoundDeadlineQueueKey = KEYS[5] +local rejectedBidRequestResultOrderKey = KEYS[6] +local bidArchiveQueueKey = KEYS[7] +local bidArchivePayloadsKey = KEYS[8] local matchId = ARGV[1] local bidderTeamIndex = ARGV[2] @@ -104,6 +120,7 @@ local bidRequestPayloadFingerprint = ARGV[10] local maximumStoredBidRequestResultCount = tonumber(ARGV[11]) local minimumBidRequestResultReplayWindowMS = tonumber(ARGV[12]) local maximumStoredAuctionBidHistoryCount = tonumber(ARGV[13]) +local clientObservedBidRoundId = ARGV[14] or '' local redisTime = redis.call('TIME') local currentRedisTimestampMS = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000) @@ -168,12 +185,23 @@ if existingFingerprint then return replayExistingRequest() end -trimOldBidRequestResults() -if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then - return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)} +-- Upgrade the old shared queue once, preserving every accepted replay record. +if redis.call('HSETNX', bidRequestResultsKey, '_acceptedOnlyOrderV1', '1') == 1 then + for _, oldRequestId in ipairs(redis.call('LRANGE', bidRequestResultOrderKey, 0, -1)) do + if redis.call('HGET', bidRequestResultsKey, oldRequestId .. ':status') == 'rejected' then + redis.call('LREM', bidRequestResultOrderKey, 0, oldRequestId) + hdelBidRequest(oldRequestId) + end + end end local function storeRejectedRequest(rejectCode) + -- Rejections have their own bounded retention. They must never consume the + -- protected replay capacity of accepted bids. + while redis.call('LLEN', rejectedBidRequestResultOrderKey) >= maximumStoredBidRequestResultCount do + local oldest = redis.call('LPOP', rejectedBidRequestResultOrderKey) + if oldest then hdelBidRequest(oldest) end + end local rejectedCurrentBidRoundDeadlineTimestampMS = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or '' redis.call( 'HSET', @@ -191,7 +219,7 @@ local function storeRejectedRequest(rejectCode) requestField('createdAtTimestampMS'), tostring(currentRedisTimestampMS) ) - redis.call('RPUSH', bidRequestResultOrderKey, bidRequestId) + redis.call('RPUSH', rejectedBidRequestResultOrderKey, bidRequestId) return {'REJECTED', rejectCode, bidRequestId, tostring(currentRedisTimestampMS), rejectedCurrentBidRoundDeadlineTimestampMS} end @@ -202,9 +230,16 @@ end local auctionLifecycleId = redis.call('HGET', liveAuctionStateKey, 'auctionLifecycleId') local currentBidRoundPlayerUserId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundPlayerUserId') +local currentBidRoundId = redis.call('HGET', liveAuctionStateKey, 'currentBidRoundId') or '' +if currentBidRoundId ~= '' and clientObservedBidRoundId == '' then + return storeRejectedRequest('BID_CLIENT_UPDATE_REQUIRED') +end if auctionLifecycleId ~= clientObservedAuctionLifecycleId or currentBidRoundPlayerUserId ~= clientObservedCurrentBidRoundPlayerUserId then return storeRejectedRequest('STALE_BID_ROUND') end +if currentBidRoundId ~= clientObservedBidRoundId then + return storeRejectedRequest('STALE_BID_ROUND') +end local currentBidRoundDeadlineTimestampMS = tonumber(redis.call('HGET', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') or '0') if currentBidRoundDeadlineTimestampMS <= currentRedisTimestampMS then @@ -235,8 +270,9 @@ if bidPointAmount + requiredReserveAfterBid > remainingBidPointAmount then return storeRejectedRequest('INSUFFICIENT') end -if redis.call('LLEN', auctionBidHistoryKey) >= maximumStoredAuctionBidHistoryCount then - return storeRejectedRequest('BID_HISTORY_LIMIT') +trimOldBidRequestResults() +if redis.call('LLEN', bidRequestResultOrderKey) >= maximumStoredBidRequestResultCount then + return {'REJECTED', 'BID_REQUEST_RESULT_LIMIT', bidRequestId, tostring(currentRedisTimestampMS)} end local maximumBidRoundDurationSeconds = tonumber(redis.call('HGET', liveAuctionStateKey, 'maximumBidRoundDurationSeconds') or '0') @@ -254,14 +290,36 @@ end local nextBidRoundDeadlineTimestampMS = currentRedisTimestampMS + nextRemainingBidRoundTimeMS local revision = redis.call('HINCRBY', liveAuctionStateKey, 'revision', 1) -local auctionBidHistoryEntryJson = - '{"bidderTeamIndex":' .. bidderTeamIndex .. - ',"bidPointAmount":' .. tostring(bidPointAmount) .. - ',"bidderUserId":"' .. bidderUserId .. - '","bidPlacedAtTimestampMS":' .. tostring(currentRedisTimestampMS) .. - '}' +local auctionBidHistoryEntryJson = cjson.encode({ + bidRequestId = bidRequestId, + revision = revision, + bidderTeamIndex = tonumber(bidderTeamIndex), + bidPointAmount = bidPointAmount, + bidderUserId = bidderUserId, + playerUserId = currentBidRoundPlayerUserId, + bidRoundId = currentBidRoundId, + bidPlacedAtTimestampMS = currentRedisTimestampMS +}) +-- Accepted replay returns above this point. Persist every accepted event before +-- trimming the recent view; the durable archive worker removes only saved events. +local archiveId = matchId .. ':' .. auctionLifecycleId .. ':' .. string.format('%016d', revision) +redis.call('HSET', bidArchivePayloadsKey, archiveId, cjson.encode({ + _id = archiveId, + matchId = matchId, + auctionLifecycleId = auctionLifecycleId, + revision = revision, + bidRequestId = bidRequestId, + bidderTeamIndex = tonumber(bidderTeamIndex), + bidPointAmount = bidPointAmount, + bidderUserId = bidderUserId, + playerUserId = currentBidRoundPlayerUserId, + bidRoundId = currentBidRoundId, + bidPlacedAtTimestampMS = currentRedisTimestampMS +})) +redis.call('ZADD', bidArchiveQueueKey, currentRedisTimestampMS, archiveId) redis.call('RPUSH', auctionBidHistoryKey, auctionBidHistoryEntryJson) +redis.call('LTRIM', auctionBidHistoryKey, -maximumStoredAuctionBidHistoryCount, -1) redis.call( 'HSET', liveAuctionStateKey, diff --git a/apps/api/src/modules/auction/redis/auction-redis.keys.ts b/apps/api/src/modules/auction/redis/auction-redis.keys.ts index 0ef9a636..06826511 100644 --- a/apps/api/src/modules/auction/redis/auction-redis.keys.ts +++ b/apps/api/src/modules/auction/redis/auction-redis.keys.ts @@ -4,6 +4,14 @@ export const AUCTION_DURABILITY_RETRY_QUEUE_KEY = 'auction:auctionDurabilityRetryQueue'; export const AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY = 'auction:auctionDurabilityDeadLetterQueue'; +export const AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY = + 'auction:deadLetterStatusQueue'; +export const AUCTION_BID_ARCHIVE_QUEUE_KEY = 'auction:bidArchiveQueue'; +export const AUCTION_BID_ARCHIVE_PAYLOADS_KEY = 'auction:bidArchivePayloads'; +export const AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY = + 'auction:bidArchiveDeadLetterQueue'; +export const AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY = + 'auction:bidArchiveDeadLetterPayloads'; export interface AuctionRedisKeys { actionRequestResultOrderKey: string; @@ -12,6 +20,7 @@ export interface AuctionRedisKeys { bidHistoryKey: string; bidRequestResultOrderKey: string; bidRequestResultsKey: string; + rejectedBidRequestResultOrderKey: string; deadlineQueueKey: string; durabilityDeadLetterQueueKey: string; durabilityPayloadKey: (auctionStateTransitionId: string) => string; @@ -33,6 +42,7 @@ export function buildAuctionRedisKeys(matchId: string): AuctionRedisKeys { bidHistoryKey: `${prefix}:auctionBidHistory`, bidRequestResultOrderKey: `${prefix}:bidRequestResultOrder`, bidRequestResultsKey: `${prefix}:bidRequestResults`, + rejectedBidRequestResultOrderKey: `${prefix}:rejectedBidRequestResultOrder`, deadlineQueueKey: AUCTION_BID_ROUND_DEADLINE_QUEUE_KEY, durabilityDeadLetterQueueKey: AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, durabilityPayloadKey: (auctionStateTransitionId: string) => diff --git a/apps/api/src/modules/auction/redis/auction-redis.state.spec.ts b/apps/api/src/modules/auction/redis/auction-redis.state.spec.ts index 537fc5a2..7d4c3682 100644 --- a/apps/api/src/modules/auction/redis/auction-redis.state.spec.ts +++ b/apps/api/src/modules/auction/redis/auction-redis.state.spec.ts @@ -24,7 +24,7 @@ describe('auction redis state assembly', () => { mongo: { auctionLifecycleId: 'new-life', revision: 2 }, }, { - expected: false, + expected: true, live: { auctionLifecycleId: 'redis-life', revision: 3 }, mongo: { auctionLifecycleId: 'mongo-life', revision: 2 }, }, diff --git a/apps/api/src/modules/auction/redis/auction-redis.state.ts b/apps/api/src/modules/auction/redis/auction-redis.state.ts index 8e6aad6f..85ff6b17 100644 --- a/apps/api/src/modules/auction/redis/auction-redis.state.ts +++ b/apps/api/src/modules/auction/redis/auction-redis.state.ts @@ -62,13 +62,11 @@ export function shouldHydrateRedisAuctionState( params: ShouldHydrateRedisAuctionStateParams, ): boolean { if (!params.live) return true; + if (params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId) + return true; const redisRevision = params.live.revision ?? -1; const mongoRevision = params.mongo.revision; - if (mongoRevision > redisRevision) return true; - return ( - mongoRevision === redisRevision && - params.mongo.auctionLifecycleId !== params.live.auctionLifecycleId - ); + return mongoRevision > redisRevision; } export function assembleAuctionViewFromRedisState( @@ -140,13 +138,17 @@ if not liveStatePatchPairCount or #ARGV ~= 8 + (liveStatePatchPairCount * 2) the return {'REJECTED', 'INVALID_PATCH_ARITY', tostring(currentRedisTimestampMS)} end +-- A lifecycle replacement requires Mongo authority and an explicit cleanup +-- under the lifecycle lock. Revision numbers from different runs are unrelated. +if redisAuctionLifecycleId and redisAuctionLifecycleId ~= mongoAuctionLifecycleId then + return {'REJECTED', 'LIFECYCLE_CONFLICT', tostring(currentRedisTimestampMS)} +end + local shouldHydrate = false if redisRevision < 0 then shouldHydrate = true elseif mongoRevision > redisRevision then shouldHydrate = true -elseif mongoRevision == redisRevision and mongoAuctionLifecycleId ~= redisAuctionLifecycleId then - shouldHydrate = true end if not shouldHydrate then @@ -184,6 +186,7 @@ if phase == 'bidding' then redis.call('ZADD', bidRoundDeadlineQueueKey, currentBidRoundDeadlineTimestampMS, matchId) else redis.call('ZREM', bidRoundDeadlineQueueKey, matchId) + redis.call('HDEL', liveAuctionStateKey, 'currentBidRoundDeadlineTimestampMS') end return {'OK', tostring(hydratedRevision), tostring(currentRedisTimestampMS)} diff --git a/apps/api/src/modules/global/database/auction-cancellation-index.ts b/apps/api/src/modules/global/database/auction-cancellation-index.ts new file mode 100644 index 00000000..1e8ab2bc --- /dev/null +++ b/apps/api/src/modules/global/database/auction-cancellation-index.ts @@ -0,0 +1,7 @@ +export const AUCTION_CANCELLATION_CLEANUP_INDEX = { + key: { auctionCancellationCleanupRetryAt: 1 as const }, + options: { + name: 'auction_cancellation_cleanup_due', + partialFilterExpression: { auctionCancellationCleanupPending: true }, + }, +}; diff --git a/apps/api/src/modules/global/database/migrations/20261109-auction-cancellation-index.ts b/apps/api/src/modules/global/database/migrations/20261109-auction-cancellation-index.ts new file mode 100644 index 00000000..73945cde --- /dev/null +++ b/apps/api/src/modules/global/database/migrations/20261109-auction-cancellation-index.ts @@ -0,0 +1,39 @@ +import type { mongo } from 'mongoose'; +import { AUCTION_CANCELLATION_CLEANUP_INDEX } from '../auction-cancellation-index'; + +/** Creates only this additive index; never drops or rebuilds other indexes. */ +export async function migrateAuctionCancellationIndex( + db: mongo.Db, + { dryRun = true }: { dryRun?: boolean } = {}, +) { + const { key, options } = AUCTION_CANCELLATION_CLEANUP_INDEX; + const collection = db.collection('match'); + const indexes = await collection + .listIndexes() + .toArray() + .catch((error: unknown) => { + if (error instanceof Error && 'code' in error && error.code === 26) + return []; + throw error; + }); + const existing = indexes.find((index) => index.name === options.name); + if ( + existing && + (JSON.stringify(existing.key) !== JSON.stringify(key) || + JSON.stringify(existing.partialFilterExpression) !== + JSON.stringify(options.partialFilterExpression) || + existing.unique === true || + existing.sparse === true || + existing.hidden === true) + ) { + throw new Error( + `Index ${options.name} already exists with a different definition.`, + ); + } + if (!dryRun) await collection.createIndex(key, options); + return { + dryRun, + alreadyPresent: Boolean(existing), + index: { key, ...options }, + }; +} diff --git a/apps/api/src/modules/global/database/migrations/run-auction-cancellation-index.ts b/apps/api/src/modules/global/database/migrations/run-auction-cancellation-index.ts new file mode 100644 index 00000000..9997592b --- /dev/null +++ b/apps/api/src/modules/global/database/migrations/run-auction-cancellation-index.ts @@ -0,0 +1,105 @@ +import 'reflect-metadata'; +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { NestFactory } from '@nestjs/core'; +import { readFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { ConnectionService, resolveMongoDbName } from '../connection.service'; +import { SecretModule } from '../../secret/secret.module'; +import { migrateAuctionCancellationIndex } from './20261109-auction-cancellation-index'; + +@Module({ + imports: [ + ConfigModule.forRoot({ + isGlobal: true, + envFilePath: [ + resolve(__dirname, '../../../../../../project.env'), + resolve(process.cwd(), '../project.env'), + resolve(process.cwd(), 'apps/project.env'), + resolve(process.cwd(), 'project.env'), + ], + }), + SecretModule, + ], + providers: [ConnectionService], +}) +class MigrationModule {} + +export function parseAuctionIndexOptions(argv: string[]) { + let dryRun = true; + let help = false; + let nodeEnv = process.env.NODE_ENV ?? 'dev'; + for (let index = 0; index < argv.length; index++) { + const argument = argv[index]; + if (argument === '--') continue; + if (argument === '--apply') dryRun = false; + else if (argument === '--dry-run') dryRun = true; + else if (argument === '--help' || argument === '-h') help = true; + else if (argument === '--env') nodeEnv = argv[++index] ?? ''; + else if (argument.startsWith('--env=')) nodeEnv = argument.slice(6); + else throw new Error(`Unknown argument: ${argument}`); + } + if (!help && !['dev', 'staging', 'prod'].includes(nodeEnv)) { + throw new Error('--env must be dev, staging, or prod.'); + } + return { dryRun, help, nodeEnv }; +} + +async function main(): Promise { + const options = parseAuctionIndexOptions(process.argv.slice(2)); + if (options.help) { + console.log(`Usage: + pnpm --filter api auction:indexes -- --env staging --dry-run + pnpm --filter api auction:indexes -- --env prod --apply + +Defaults to dry-run. --env accepts dev, staging, or prod (NODE_ENV or dev by default). +Creates the Match cancellation cleanup partial index; does not modify documents.`); + return; + } + // Use the same optional machine environment as the existing migration runners. + const machineHome = process.env.HOME ?? process.env.USERPROFILE; + if (machineHome) { + const contents = await readFile( + resolve(machineHome, 'source/global-env-vars.txt'), + 'utf8', + ).catch(() => ''); + for (const line of contents.split(/\r?\n/)) { + const match = line.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/); + if (match) process.env[match[1]] = match[2].replace(/^["']|["']$/g, ''); + } + } + process.env.NODE_ENV = options.nodeEnv; + // Even a dry-run with auto-index enabled in the environment must not write. + process.env.MONGODB_AUTO_INDEX = 'false'; + const app = await NestFactory.createApplicationContext(MigrationModule, { + logger: ['warn', 'error'], + }); + try { + const connection = await app.get(ConnectionService).getConnection(); + if (!connection.db) throw new Error('MongoDB connection has no db handle.'); + const result = await migrateAuctionCancellationIndex( + connection.db, + options, + ); + console.log( + JSON.stringify( + { + env: options.nodeEnv, + db: resolveMongoDbName(options.nodeEnv), + ...result, + }, + null, + 2, + ), + ); + } finally { + await app.close(); + } +} + +if (require.main === module) { + void main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/apps/api/src/modules/global/database/schema.ts b/apps/api/src/modules/global/database/schema.ts index 24201db6..867a3abb 100644 --- a/apps/api/src/modules/global/database/schema.ts +++ b/apps/api/src/modules/global/database/schema.ts @@ -12,6 +12,7 @@ import { competitionModelDefinitions, } from '../../competition/competition.schema'; import { ModelDefinition } from './types'; +import { AUCTION_CANCELLATION_CLEANUP_INDEX } from './auction-cancellation-index'; export type Position = 'TOP' | 'JGL' | 'MID' | 'ADC' | 'SUP'; export const POSITIONS: Position[] = ['TOP', 'JGL', 'MID', 'ADC', 'SUP']; @@ -700,6 +701,12 @@ export interface Match extends MongooseTimestamps { teams: MatchTeam[]; result?: MatchResult; resultPollState?: MatchResultPollState; + /** 마지막으로 취소된 Redis 경매 lifecycle. 취소 재전송을 멱등 처리한다. */ + lastCancelledAuctionLifecycleId?: string; + auctionCancellationCleanupPending?: boolean; + auctionCancellationCleanupRetryAt?: Date; + auctionCancellationCleanupAttempts?: number; + auctionCancellationCleanupError?: string; doubleElimination?: DoubleEliminationBracket; legacyImported?: boolean; legacyImportSource?: { @@ -804,12 +811,41 @@ export interface AuctionBid { } export interface AuctionBidHistoryEntry { + bidRequestId?: string; + revision?: number; + bidRoundId?: string; + playerUserId?: mongoose.Types.ObjectId; bidderTeamIndex: number; bidPointAmount: number; bidderUserId: mongoose.Types.ObjectId; bidPlacedAtTimestampMS: number; } +export interface AuctionBidRecord extends AuctionBidHistoryEntry { + _id: string; + matchId: mongoose.Types.ObjectId; + auctionLifecycleId: string; + revision: number; + bidRequestId: string; +} + +export const AuctionBidRecordSchema = new Schema( + { + _id: { type: String, required: true }, + matchId: { type: Schema.Types.ObjectId, required: true }, + auctionLifecycleId: { type: String, required: true }, + revision: { type: Number, required: true }, + bidRequestId: { type: String, required: true }, + bidRoundId: { type: String }, + playerUserId: { type: Schema.Types.ObjectId }, + bidderUserId: { type: Schema.Types.ObjectId, required: true }, + bidderTeamIndex: { type: Number, required: true }, + bidPointAmount: { type: Number, required: true }, + bidPlacedAtTimestampMS: { type: Number, required: true }, + }, + { collection: 'auctionBidRecords', versionKey: false }, +); + export interface AuctionChatMessage { id: string; userId?: mongoose.Types.ObjectId; @@ -832,6 +868,8 @@ export interface Auction extends MongooseTimestamps { matchSideEffectStatus: AuctionMatchSideEffectStatus; matchSideEffectRevision: number; matchSideEffectError?: string; + matchSideEffectTransitionId?: string; + startupCompleted?: boolean; teams: AuctionTeam[]; order: AuctionPlayerSnapshot[]; passedQueue: AuctionPlayerSnapshot[]; @@ -858,6 +896,7 @@ export interface Auction extends MongooseTimestamps { currentHighestBidPointAmount?: number; currentHighestBidderTeamIndex?: number; currentBidRoundDeadlineTimestampMS?: number; + currentBidRoundId?: string; maximumBidRoundDurationSeconds?: number; bidRoundExtensionSeconds?: number; initialTeamBidPointAmount?: number; @@ -1971,6 +2010,11 @@ export const MatchSchema = new Schema( type: MatchResultPollStateSubSchema, required: false, }, + lastCancelledAuctionLifecycleId: { type: String, required: false }, + auctionCancellationCleanupPending: { type: Boolean }, + auctionCancellationCleanupRetryAt: { type: Date }, + auctionCancellationCleanupAttempts: { type: Number }, + auctionCancellationCleanupError: { type: String }, doubleElimination: { type: DoubleEliminationBracketSubSchema, required: false, @@ -1995,6 +2039,10 @@ export const MatchSchema = new Schema( ); MatchSchema.index({ communityId: 1, scheduledAt: -1 }); +MatchSchema.index( + AUCTION_CANCELLATION_CLEANUP_INDEX.key, + AUCTION_CANCELLATION_CLEANUP_INDEX.options, +); applyCompetitionDomainInvariantIndexes(MatchSchema, 'match'); MatchSchema.index({ communityId: 1, status: 1, scheduledAt: -1 }); MatchSchema.index( @@ -2174,6 +2222,8 @@ const AuctionBidSubSchema = new Schema( const AuctionBidHistoryEntrySubSchema = new Schema( { + bidRoundId: { type: String }, + playerUserId: { type: Schema.Types.ObjectId, ref: 'user' }, bidderTeamIndex: { type: Number, required: true, min: 1 }, bidPointAmount: { type: Number, required: true }, bidderUserId: { type: Schema.Types.ObjectId, ref: 'user', required: true }, @@ -2195,6 +2245,9 @@ const AuctionChatMessageSubSchema = new Schema( export const AuctionSchema = new Schema( { + startupCompleted: { type: Boolean }, + matchSideEffectTransitionId: { type: String }, + currentBidRoundId: { type: String }, matchId: { type: Schema.Types.ObjectId, ref: 'match', @@ -2520,6 +2573,11 @@ export const tournamentModelDefinitions: Array< }, { name: 'match', schema: MatchSchema, schemaName: 'match' }, { name: 'auction', schema: AuctionSchema, schemaName: 'auction' }, + { + name: 'auctionBidRecord', + schema: AuctionBidRecordSchema, + schemaName: 'auctionBidRecords', + }, { name: 'auctionHistory', schema: AuctionHistoryEntrySchema, @@ -2553,6 +2611,7 @@ export interface TournamentModelTypes { legacyImportJob: LegacyImportJob; match: Match; auction: Auction; + auctionBidRecord: AuctionBidRecord; auctionHistory: AuctionHistoryEntry; auctionSideEffectApplication: AuctionSideEffectApplication; matchStats: MatchStats; diff --git a/apps/api/src/modules/tournament-code/tournament-code.service.spec.ts b/apps/api/src/modules/tournament-code/tournament-code.service.spec.ts index f2ebd890..9286febd 100644 --- a/apps/api/src/modules/tournament-code/tournament-code.service.spec.ts +++ b/apps/api/src/modules/tournament-code/tournament-code.service.spec.ts @@ -90,6 +90,33 @@ function createMatch(overrides: MatchOverrides): MatchDocument { } as unknown as MatchDocument; } +it('releases reserved codes inside the caller transaction during auction cancellation', async () => { + const session = createSession(); + const match = createMatch({ + tournamentCodeIds: [new mongoose.Types.ObjectId()], + }); + const matchQuery = mongoQuery(match); + const codeQuery = mongoQuery({ modifiedCount: 1 }); + const createSessionSpy = jest.fn(); + const service = new TournamentCodeService( + { + createSession: createSessionSpy, + getModel: async (name: string) => + name === 'match' + ? { findById: () => matchQuery } + : { updateMany: () => codeQuery }, + } as never, + createAuctionRedisService() as never, + ); + await service.releaseFromMatch(match, session as never); + expect(createSessionSpy).not.toHaveBeenCalled(); + expect(matchQuery.session).not.toHaveBeenCalled(); + expect(codeQuery.session).toHaveBeenCalledWith(session); + expect(match.save).toHaveBeenCalledWith({ session }); + expect(session.commitTransaction).not.toHaveBeenCalled(); + expect(match.tournamentCodeIds).toEqual([]); +}); + describe('TournamentCodeService round-robin scheduling', () => { it('keeps pair slot calculation unchanged', () => { const service = createService(); diff --git a/apps/api/src/modules/tournament-code/tournament-code.service.ts b/apps/api/src/modules/tournament-code/tournament-code.service.ts index 45c677b8..6c91e931 100644 --- a/apps/api/src/modules/tournament-code/tournament-code.service.ts +++ b/apps/api/src/modules/tournament-code/tournament-code.service.ts @@ -329,23 +329,31 @@ export class TournamentCodeService { public async releaseFromMatch( matchOrId: MatchDocument | mongoose.Types.ObjectId, + session?: mongoose.ClientSession, ): Promise { const matchId = this.resolveMatchId(matchOrId); + // Auction cancellation owns the surrounding transaction. Committing a + // separate release would survive its rollback and conflict with Match.save. + if (session) return this.releaseFromMatchInTransaction(matchOrId, session); await this.withLockedMatchTransaction(matchId, (session) => this.releaseFromMatchInTransaction(matchId, session), ); } private async releaseFromMatchInTransaction( - matchId: mongoose.Types.ObjectId, + matchOrId: MatchDocument | mongoose.Types.ObjectId, session: mongoose.ClientSession, ): Promise { const MatchModel = await this.connectionService.getModel('match'); const TournamentCodeModel = await this.connectionService.getModel('tournamentCode'); - const match = await this.execQuery( - this.withQuerySession(MatchModel.findById(matchId), session), - ); + // Preserve the caller's document and __v when participating in its session. + const match = + matchOrId instanceof mongoose.Types.ObjectId + ? await this.execQuery( + this.withQuerySession(MatchModel.findById(matchOrId), session), + ) + : matchOrId; if (!match) return; if (match.tournamentCodeIds.length === 0) return; diff --git a/apps/api/src/modules/tournament-result/tournament-result.service.ts b/apps/api/src/modules/tournament-result/tournament-result.service.ts index 1e8cd46a..c900b987 100644 --- a/apps/api/src/modules/tournament-result/tournament-result.service.ts +++ b/apps/api/src/modules/tournament-result/tournament-result.service.ts @@ -240,10 +240,15 @@ export class TournamentResultService { public async cancelPolling( matchOrId: MatchDocument | mongoose.Types.ObjectId, + session?: mongoose.ClientSession, ): Promise { - const match = await this.loadMatch(matchOrId); + const match = + matchOrId instanceof mongoose.Types.ObjectId + ? await this.loadMatch(matchOrId, session) + : matchOrId; match.resultPollState = undefined; - await match.save(); + if (session) await match.save({ session }); + else await match.save(); } public async processPollJob(job: PollTournamentResultJob): Promise { @@ -908,12 +913,14 @@ export class TournamentResultService { private async loadMatch( matchOrId: MatchDocument | mongoose.Types.ObjectId, + session?: mongoose.ClientSession, ): Promise { const MatchModel = await this.connectionService.getModel('match'); - const match = + const query = matchOrId instanceof mongoose.Types.ObjectId - ? await MatchModel.findById(matchOrId) - : await MatchModel.findById(matchOrId._id); + ? MatchModel.findById(matchOrId) + : MatchModel.findById(matchOrId._id); + const match = await (session ? query.session(session) : query); if (!match) throw new NotFoundException('Match not found.'); return match; } diff --git a/apps/api/src/scripts/auction-redis-rehearsal.ts b/apps/api/src/scripts/auction-redis-rehearsal.ts index 026b6f15..22d4e565 100644 --- a/apps/api/src/scripts/auction-redis-rehearsal.ts +++ b/apps/api/src/scripts/auction-redis-rehearsal.ts @@ -446,6 +446,13 @@ async function main(): Promise { const bidResult = await auctionService.placeBid({ amount: 10, bidRequestId: randomUUID(), + clientObservedBidRoundId: String( + ( + await auctionRedisStateReaderService.readAuctionStateView( + resources.matchId.toString(), + ) + )?.currentBidRoundId, + ), matchId: resources.matchId, teamIndex: 1, }); @@ -537,6 +544,13 @@ async function main(): Promise { await auctionService.reset({ auctionActionRequestId: randomUUID(), + clientObservedAuctionLifecycleId: String( + ( + await auctionRedisStateReaderService.readAuctionStateView( + resources.matchId.toString(), + ) + )?.auctionLifecycleId, + ), communityId: resources.communityId, matchId: resources.matchId, }); diff --git a/apps/api/test/auction-bid-archive.integration-spec.ts b/apps/api/test/auction-bid-archive.integration-spec.ts new file mode 100644 index 00000000..84cdd9c1 --- /dev/null +++ b/apps/api/test/auction-bid-archive.integration-spec.ts @@ -0,0 +1,180 @@ +import { createClient } from 'redis'; +import mongoose from 'mongoose'; +import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service'; +import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service'; +import { + AUCTION_BID_ARCHIVE_QUEUE_KEY as queue, + AUCTION_BID_ARCHIVE_PAYLOADS_KEY as payloads, + AUCTION_BID_ARCHIVE_DEAD_LETTER_QUEUE_KEY as deadQueue, + AUCTION_BID_ARCHIVE_DEAD_LETTER_PAYLOADS_KEY as deadPayloads, +} from '../src/modules/auction/redis/auction-redis.keys'; + +const client = createClient({ + url: process.env.REDIS_TEST_URL, + socket: { reconnectStrategy: false }, +}); +const redis = new AuctionRedisService(client); +const matchId = new mongoose.Types.ObjectId().toString(); +function event(revision: number) { + return { + _id: `${matchId}:life:${String(revision).padStart(16, '0')}`, + matchId, + auctionLifecycleId: 'life', + revision, + bidRequestId: `bid-${revision}`, + bidderUserId: new mongoose.Types.ObjectId().toString(), + bidderTeamIndex: 1, + bidPointAmount: revision, + bidPlacedAtTimestampMS: Date.now(), + }; +} +function fixture() { + const bulkWrite = jest.fn(async (operations: unknown[]) => ({ + matchedCount: 0, + upsertedCount: operations.length, + })); + const service = new AuctionBidArchiveService( + { getModel: async () => ({ bulkWrite }) } as never, + redis, + ); + return { service, bulkWrite }; +} +async function enqueue(id: string, raw: string | null) { + await client.zAdd(queue, { score: 1, value: id }); + if (raw !== null) await client.hSet(payloads, id, raw); +} +beforeAll(async () => { + if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) + throw new Error('Use the isolated Redis test runner.'); + client.on('error', () => {}); + await client.connect(); +}); +beforeEach(async () => { + await client.flushDb(); +}); +afterEach(() => jest.restoreAllMocks()); +afterAll(async () => { + if (client.isOpen) await client.quit(); +}); + +it('isolates malformed, missing and invalid payloads while saving valid bids and retaining the originals', async () => { + const { service, bulkWrite } = fixture(); + const good = event(10); + const invalid = [ + ['broken', '{broken'], + ['empty', ''], + ['null', 'null'], + ['array', '[]'], + ['missing', null], + [event(1)._id, JSON.stringify({ ...event(1), revision: -1 })], + [event(2)._id, JSON.stringify({ ...event(2), bidderUserId: 'bad' })], + [event(3)._id, JSON.stringify({ ...event(3), bidderUserId: undefined })], + [event(4)._id, JSON.stringify({ ...event(4), bidPointAmount: '4' })], + ] as Array<[string, string | null]>; + for (const [id, raw] of invalid) await enqueue(id, raw); + await enqueue(good._id, JSON.stringify(good)); + await expect(service.persistPendingBids()).resolves.toBe(1); + expect(bulkWrite.mock.calls[0][0]).toHaveLength(1); + expect(await client.zCard(queue)).toBe(0); + expect(await client.hLen(payloads)).toBe(0); + expect(await client.zCard(deadQueue)).toBe(invalid.length); + for (const [id, raw] of invalid) { + expect(JSON.parse((await client.hGet(deadPayloads, id))!)).toMatchObject({ + id, + rawPayload: raw, + error: expect.any(String), + quarantinedAtTimestampMS: expect.any(Number), + }); + } + expect(await client.ttl(deadPayloads)).toBe(-1); + await expect(service.persistPendingBids()).resolves.toBe(0); +}); + +it('retains valid bids during a Mongo outage, after moving malformed entries out of the pending queue', async () => { + const { service, bulkWrite } = fixture(); + const good = event(1); + await enqueue('broken', '{broken'); + await enqueue(good._id, JSON.stringify(good)); + bulkWrite.mockRejectedValueOnce(new Error('Mongo offline')); + await expect(service.persistPendingBids()).rejects.toThrow('Mongo offline'); + expect(await client.zRange(queue, 0, -1)).toEqual([good._id]); + expect(await client.hGet(payloads, good._id)).toBe(JSON.stringify(good)); + expect(await client.zCard(deadQueue)).toBe(1); + await expect(service.persistPendingBids()).resolves.toBe(1); + expect(await client.zCard(queue)).toBe(0); +}); + +it('does not quarantine a payload repaired concurrently after HMGET', async () => { + const { service } = fixture(); + const good = event(1); + await enqueue(good._id, '{broken'); + const send = client.sendCommand.bind(client); + jest.spyOn(client, 'sendCommand').mockImplementation(async (command) => { + const result = await send(command); + if (command[0] === 'HMGET') + await client.hSet(payloads, good._id, JSON.stringify(good)); + return result; + }); + await expect(service.persistPendingBids()).resolves.toBe(0); + expect(await client.zCard(deadQueue)).toBe(0); + expect(await client.zRange(queue, 0, -1)).toEqual([good._id]); + await expect(service.persistPendingBids()).resolves.toBe(1); +}); + +it('preserves pending payloads if quarantine cannot be written', async () => { + const { service, bulkWrite } = fixture(); + const good = event(1); + await enqueue('broken', '{broken'); + await enqueue(good._id, JSON.stringify(good)); + await client.set(deadPayloads, 'wrong-type'); + await expect(service.persistPendingBids()).rejects.toThrow('WRONGTYPE'); + expect(await client.zCard(queue)).toBe(2); + expect(await client.hGet(payloads, 'broken')).toBe('{broken'); + expect(bulkWrite).not.toHaveBeenCalled(); + await client.del(deadPayloads); + await expect(service.persistPendingBids()).resolves.toBe(1); +}); + +it('does not acknowledge a payload changed while Mongo was saving', async () => { + const { service, bulkWrite } = fixture(); + const good = event(1); + await enqueue(good._id, JSON.stringify(good)); + bulkWrite.mockImplementationOnce(async () => { + await client.hSet(payloads, good._id, '{replaced'); + return { matchedCount: 0, upsertedCount: 1 }; + }); + await expect(service.persistPendingBids()).resolves.toBe(1); + expect(await client.hGet(payloads, good._id)).toBe('{replaced'); + expect(await client.zCard(queue)).toBe(1); +}); + +it.each([ + ['bidRoundId', null], + ['bidRoundId', 1], + ['bidRoundId', {}], + ['bidRoundId', []], + ['bidRoundId', ' '], + ['playerUserId', ''], +])('continues quarantining invalid %s=%j', async (field, value) => { + const { service, bulkWrite } = fixture(); + const bad = { ...event(1), [field]: value }; + await enqueue(bad._id, JSON.stringify(bad)); + await expect(service.persistPendingBids()).resolves.toBe(0); + expect(bulkWrite).not.toHaveBeenCalled(); + expect(await client.zCard(deadQueue)).toBe(1); +}); + +it('archives a pre-upgrade bid with an empty round ID instead of quarantining it', async () => { + const { service, bulkWrite } = fixture(); + const legacy = { ...event(1), bidRoundId: '' }; + await enqueue(legacy._id, JSON.stringify(legacy)); + await expect(service.persistPendingBids()).resolves.toBe(1); + expect(await client.zCard(deadQueue)).toBe(0); + expect(await client.zCard(queue)).toBe(0); + const [[operations]] = bulkWrite.mock.calls as unknown as [ + [Array<{ updateOne: { update: { $setOnInsert: object } } }>], + ]; + expect(operations[0].updateOne.update.$setOnInsert).not.toHaveProperty( + 'bidRoundId', + ); +}); diff --git a/apps/api/test/auction-cancellation.mongo-spec.ts b/apps/api/test/auction-cancellation.mongo-spec.ts new file mode 100644 index 00000000..7548ef05 --- /dev/null +++ b/apps/api/test/auction-cancellation.mongo-spec.ts @@ -0,0 +1,769 @@ +import mongoose from 'mongoose'; +import { AuctionService } from '../src/modules/auction/auction.service'; +import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler'; +import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service'; +import { migrateAuctionCancellationIndex } from '../src/modules/global/database/migrations/20261109-auction-cancellation-index'; +import { AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY } from '../src/modules/auction/redis/auction-redis.keys'; +import { TournamentCodeService } from '../src/modules/tournament-code/tournament-code.service'; +import { TournamentResultService } from '../src/modules/tournament-result/tournament-result.service'; +import { + MatchSchema, + AuctionSchema, + AuctionHistoryEntrySchema, + AuctionSideEffectApplicationSchema, + TournamentCodeSchema, + AuctionBidRecordSchema, +} from '../src/modules/global/database/schema'; + +let connection: mongoose.Connection; +let models: Record>; +const redis = { + acquireTokenLock: async () => true, + releaseTokenLock: async () => true, + getCurrentRedisTimestampMS: async () => Date.now(), + raw: () => ({ + sendCommand: async (command: string[]) => + command[0] === 'SCAN' ? ['0', []] : 1, + }), +}; + +beforeAll(async () => { + if ( + process.env.MONGO_TEST_ISOLATED !== '1' || + !process.env.MONGO_TEST_URL?.startsWith('mongodb://127.0.0.1:') + ) + throw new Error('Use test:mongodb with an isolated local replica set.'); + connection = await mongoose + .createConnection(process.env.MONGO_TEST_URL, { + autoIndex: false, + autoCreate: false, + }) + .asPromise(); + models = { + match: connection.model('match', MatchSchema), + auction: connection.model('auction', AuctionSchema), + auctionHistory: connection.model( + 'auctionHistory', + AuctionHistoryEntrySchema, + ), + auctionSideEffectApplication: connection.model( + 'auctionSideEffectApplication', + AuctionSideEffectApplicationSchema, + ), + tournamentCode: connection.model('tournamentCode', TournamentCodeSchema), + auctionBidRecord: connection.model( + 'auctionBidRecord', + AuctionBidRecordSchema, + ), + }; + await Promise.all( + Object.values(models).map((model) => model.createCollection()), + ); +}); +beforeEach(async () => { + await Promise.all(Object.values(models).map((model) => model.deleteMany({}))); +}); +afterEach(() => jest.restoreAllMocks()); +afterAll(async () => { + await connection?.close(); +}); + +async function fixture(reserved: boolean) { + const matchId = new mongoose.Types.ObjectId(), + communityId = new mongoose.Types.ObjectId(), + auctionId = new mongoose.Types.ObjectId(), + codeId = new mongoose.Types.ObjectId(); + await models.match.create({ + _id: matchId, + communityId, + title: 'Isolated auction test', + scheduledAt: new Date(), + status: 'team-built', + teamBuiltAt: new Date(), + teams: [], + participants: [], + tournamentCodeIds: reserved ? [codeId] : [], + gameSchedule: [], + __v: 0, + }); + await models.auction.collection.insertOne({ + _id: auctionId, + matchId, + auctionEngineVersion: 'redis-v1', + auctionLifecycleId: 'life', + phase: 'complete', + revision: 10, + matchSideEffectRevision: 0, + matchSideEffectStatus: 'none', + }); + if (reserved) + await models.tournamentCode.collection.insertOne({ + _id: codeId, + code: 'ISOLATED-TEST', + state: 'reserved', + matchId, + }); + const database = { + getModel: async (name: string) => models[name], + createSession: () => connection.startSession(), + }; + const codes = new TournamentCodeService(database as never, redis as never); + const results = new TournamentResultService( + database as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + ); + const service = new AuctionService( + database as never, + {} as never, + codes, + results, + redis as never, + {} as never, + {} as never, + ); + return { + matchId, + communityId, + auctionId, + codeId, + service, + database, + codes, + results, + }; +} + +it.each([false, true])( + 'cancels the auction atomically with reserved codes=%s', + async (reserved) => { + const f = await fixture(reserved); + await f.service.reset({ + matchId: f.matchId, + communityId: f.communityId, + clientObservedAuctionLifecycleId: 'life', + }); + const match = await models.match.findById(f.matchId).lean(); + expect(match).toMatchObject({ + status: 'recruiting', + teams: [], + tournamentCodeIds: [], + gameSchedule: [], + }); + expect(match?.teamBuiltAt).toBeUndefined(); + expect(await models.auction.findById(f.auctionId)).toBeNull(); + if (reserved) + expect( + (await models.tournamentCode.findById(f.codeId).lean())?.state, + ).toBe('released'); + }, +); + +it('rolls back code release and Match changes if cancellation later fails', async () => { + const f = await fixture(true); + jest.spyOn(models.auctionHistory, 'deleteMany').mockImplementation(() => { + throw new Error('forced history failure'); + }); + await expect( + f.service.reset({ + matchId: f.matchId, + communityId: f.communityId, + clientObservedAuctionLifecycleId: 'life', + }), + ).rejects.toThrow('forced history failure'); + expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe( + 'reserved', + ); + expect(await models.match.findById(f.matchId).lean()).toMatchObject({ + status: 'team-built', + __v: 0, + tournamentCodeIds: [f.codeId], + }); + expect(await models.auction.findById(f.auctionId)).not.toBeNull(); +}); + +it('retries cancellation cleanup from Mongo after Redis fails, without another cancellation request', async () => { + const f = await fixture(true); + const cleanup = jest + .spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle') + .mockRejectedValueOnce(new Error('Redis offline')) + .mockResolvedValue(undefined); + await f.service.reset({ + matchId: f.matchId, + communityId: f.communityId, + clientObservedAuctionLifecycleId: 'life', + }); + expect(cleanup).not.toHaveBeenCalled(); + await f.service.retryPendingCancellations(); + expect(await models.match.findById(f.matchId).lean()).toMatchObject({ + auctionCancellationCleanupPending: true, + auctionCancellationCleanupAttempts: 1, + }); + await models.match.updateOne( + { _id: f.matchId }, + { $set: { auctionCancellationCleanupRetryAt: new Date(0) } }, + ); + await f.service.retryPendingCancellations(); + expect(cleanup).toHaveBeenCalledTimes(2); + expect( + (await models.match.findById(f.matchId).lean()) + ?.auctionCancellationCleanupPending, + ).toBe(false); +}); + +it('keeps all 1005 Mongo bid records and idempotently retries when Redis acknowledgement fails', async () => { + const f = await fixture(false); + const events = Array.from({ length: 1005 }, (_, i) => ({ + _id: `${f.matchId.toString()}:life:${String(i + 1).padStart(16, '0')}`, + matchId: f.matchId.toString(), + auctionLifecycleId: 'life', + revision: i + 1, + bidRequestId: `bid:${i}`, + bidRoundId: 'round', + bidderTeamIndex: 1, + bidPointAmount: i + 1, + bidderUserId: new mongoose.Types.ObjectId().toString(), + playerUserId: new mongoose.Types.ObjectId().toString(), + bidPlacedAtTimestampMS: Date.now(), + })); + let failAck = true; + const sendCommand = jest.fn(async (command: string[]) => { + if (command[0] === 'ZRANGE') return events.map((e) => e._id); + if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e)); + if (failAck) throw new Error('Redis acknowledgement failed'); + return 1; + }); + const archive = new AuctionBidArchiveService( + f.database as never, + { raw: () => ({ sendCommand }) } as never, + ); + await expect(archive.persistPendingBids(2000)).rejects.toThrow( + 'acknowledgement failed', + ); + expect(await models.auctionBidRecord.countDocuments()).toBe(1005); + failAck = false; + await expect(archive.persistPendingBids(2000)).resolves.toBe(1005); + expect(await models.auctionBidRecord.countDocuments()).toBe(1005); + const page = await archive.list(f.matchId, 1000, 100); + expect(page.items.map((item) => item.revision)).toEqual([ + 1001, 1002, 1003, 1004, 1005, + ]); + expect(page.hasMore).toBe(false); +}); + +it('migrates the cleanup index idempotently and examines only due pending matches', async () => { + const f = await fixture(false); + const documents = Array.from({ length: 10000 }, (_, i) => ({ + _id: new mongoose.Types.ObjectId(), + lastCancelledAuctionLifecycleId: `life-${i}`, + ...(i < 5 + ? { + auctionCancellationCleanupPending: true, + auctionCancellationCleanupRetryAt: new Date(0), + } + : i === 5 + ? { + auctionCancellationCleanupPending: true, + auctionCancellationCleanupRetryAt: new Date(Date.now() + 3600000), + } + : {}), + })); + await models.match.collection.insertMany(documents); + const find = jest.spyOn(models.match, 'find'); + const cleanup = jest + .spyOn(f.service as any, 'clearRedisAuctionRuntimeForLifecycle') + .mockResolvedValue(undefined); + await f.service.retryPendingCancellations(); + expect(cleanup).toHaveBeenCalledTimes(5); + const query = find.mock.calls[0][0] as unknown as Record; + // Restore only the five due jobs so explain measures the actual worker query. + await models.match.updateMany( + { _id: { $in: documents.slice(0, 5).map((d) => d._id) } }, + { + $set: { + auctionCancellationCleanupPending: true, + auctionCancellationCleanupRetryAt: new Date(0), + }, + }, + ); + const explain = () => + models.match.collection + .find(query) + .sort({ auctionCancellationCleanupRetryAt: 1 }) + .limit(20) + .explain('executionStats'); + expect( + (await explain()).executionStats.totalDocsExamined, + ).toBeGreaterThanOrEqual(10000); + expect(await migrateAuctionCancellationIndex(connection.db!)).toMatchObject({ + dryRun: true, + alreadyPresent: false, + }); + expect( + (await models.match.collection.listIndexes().toArray()).some( + (i) => i.name === 'auction_cancellation_cleanup_due', + ), + ).toBe(false); + await migrateAuctionCancellationIndex(connection.db!, { dryRun: false }); + expect( + await migrateAuctionCancellationIndex(connection.db!, { dryRun: false }), + ).toMatchObject({ alreadyPresent: true }); + const plan = await explain(); + expect(plan.executionStats.nReturned).toBe(5); + expect(plan.executionStats.totalDocsExamined).toBe(5); + expect(JSON.stringify(plan.queryPlanner.winningPlan)).toContain('IXSCAN'); +}); + +function archiveEvent(matchId: mongoose.Types.ObjectId, revision: number) { + return { + _id: `${matchId.toString()}:life:${String(revision).padStart(16, '0')}`, + matchId: matchId.toString(), + auctionLifecycleId: 'life', + revision, + bidRequestId: `bid-${revision}`, + bidRoundId: 'round', + bidderUserId: new mongoose.Types.ObjectId().toString(), + playerUserId: new mongoose.Types.ObjectId().toString(), + bidderTeamIndex: 1, + bidPointAmount: revision, + bidPlacedAtTimestampMS: Date.now(), + }; +} + +it.each(['', undefined, 'round'])( + 'persists and lists bids with optional round ID %s without duplicating retries', + async (bidRoundId) => { + const f = await fixture(false); + const event = { ...archiveEvent(f.matchId, 1), bidRoundId }; + const sendCommand = jest.fn(async (command: string[]) => { + if (command[0] === 'ZRANGE') return [event._id]; + if (command[0] === 'HMGET') return [JSON.stringify(event)]; + return 1; + }); + const archive = new AuctionBidArchiveService( + f.database as never, + { raw: () => ({ sendCommand }) } as never, + ); + await expect(archive.persistPendingBids()).resolves.toBe(1); + await expect(archive.persistPendingBids()).resolves.toBe(1); + expect(await models.auctionBidRecord.countDocuments()).toBe(1); + const page = await archive.list(f.matchId); + expect(page.items).toHaveLength(1); + expect(String(page.items[0].playerUserId)).toBe(event.playerUserId); + if (bidRoundId) expect(page.items[0].bidRoundId).toBe(bidRoundId); + else expect(page.items[0]).not.toHaveProperty('bidRoundId'); + expect( + sendCommand.mock.calls + .filter(([command]) => command[0] === 'EVAL') + .every(([command]) => command[2] === '2'), + ).toBe(true); + }, +); + +it.each([ + 'json', + 'revision', + 'object-id', + 'required-field', + 'identity', + 'missing-payload', +])( + 'quarantines an invalid %s without losing the valid Mongo bid', + async (kind) => { + const f = await fixture(false); + const bad = archiveEvent(f.matchId, 1), + good = archiveEvent(f.matchId, 2); + const invalid: Record = { ...bad }; + if (kind === 'revision') invalid.revision = -1; + if (kind === 'object-id') invalid.bidderUserId = 'not-an-object-id'; + if (kind === 'required-field') delete invalid.bidderUserId; + if (kind === 'identity') + invalid.matchId = new mongoose.Types.ObjectId().toString(); + const raw = + kind === 'json' + ? '{broken' + : kind === 'missing-payload' + ? null + : JSON.stringify(invalid); + const sendCommand = jest.fn(async (command: string[]) => { + if (command[0] === 'ZRANGE') return [bad._id, good._id]; + if (command[0] === 'HMGET') return [raw, JSON.stringify(good)]; + return 1; + }); + const archive = new AuctionBidArchiveService( + f.database as never, + { raw: () => ({ sendCommand }) } as never, + ); + await expect(archive.persistPendingBids()).resolves.toBe(1); + expect(await models.auctionBidRecord.countDocuments()).toBe(1); + expect( + await models.auctionBidRecord.findById(good._id).lean(), + ).toMatchObject({ revision: 2 }); + expect(await models.auctionBidRecord.findById(bad._id)).toBeNull(); + const quarantine = sendCommand.mock.calls + .map((c) => c[0]) + .find((c) => c[0] === 'EVAL' && c[2] === '4')!; + expect(JSON.parse(quarantine[11])).toMatchObject({ + id: bad._id, + rawPayload: raw, + error: expect.any(String), + }); + const ack = sendCommand.mock.calls + .map((c) => c[0]) + .find((c) => c[0] === 'EVAL' && c[2] === '2')!; + expect(ack.slice(5)).toEqual([good._id, JSON.stringify(good)]); + }, +); + +it.each(['partial-write-error', 'partial-acknowledgement'])( + 'retains the batch after %s and retries Mongo upserts without duplicates', + async (failure) => { + const f = await fixture(false); + const events = [archiveEvent(f.matchId, 1), archiveEvent(f.matchId, 2)]; + const sendCommand = jest.fn(async (command: string[]) => { + if (command[0] === 'ZRANGE') return events.map((e) => e._id); + if (command[0] === 'HMGET') return events.map((e) => JSON.stringify(e)); + return 1; + }); + const archive = new AuctionBidArchiveService( + f.database as never, + { raw: () => ({ sendCommand }) } as never, + ); + const write = models.auctionBidRecord.bulkWrite.bind( + models.auctionBidRecord, + ); + jest + .spyOn(models.auctionBidRecord, 'bulkWrite') + .mockImplementationOnce(async (operations, options) => { + const result = await write(operations.slice(0, 1), options); + if (failure === 'partial-write-error') + throw new Error('Lost Mongo acknowledgement'); + return result; + }); + await expect(archive.persistPendingBids()).rejects.toThrow(); + expect(await models.auctionBidRecord.countDocuments()).toBe(1); + expect( + sendCommand.mock.calls.filter(([c]) => c[0] === 'EVAL'), + ).toHaveLength(0); + await expect(archive.persistPendingBids()).resolves.toBe(2); + expect(await models.auctionBidRecord.countDocuments()).toBe(2); + }, +); + +it.each(['Mongo', 'Redis mirroring'])( + 'reconciles a terminal status after %s recovers', + async (outage) => { + const f = await fixture(false); + const payload = { + auctionId: f.auctionId.toString(), + matchId: f.matchId.toString(), + auctionLifecycleId: 'life', + auctionStateTransitionId: 'complete-10', + matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' }, + afterAuctionMongoProjection: { matchSideEffectRevision: 1 }, + }; + await models.auction.updateOne( + { _id: f.auctionId }, + { + $set: { + matchSideEffectTransitionId: 'complete-10', + matchSideEffectRevision: 1, + matchSideEffectStatus: 'pending', + }, + }, + ); + let unavailable = outage === 'Mongo'; + const sendCommand = jest.fn(async (command: string[]) => { + if (command[0] === 'ZRANGEBYSCORE') return ['dead-letter']; + if (command[0] === 'HGETALL') + return { payloadJson: JSON.stringify(payload) }; + if (command[0] === 'HGET') return 'retry limit exceeded'; + return 1; + }); + const mirror = jest.fn().mockResolvedValue(true), + broadcast = jest.fn(); + if (outage === 'Redis mirroring') + mirror.mockRejectedValueOnce(new Error('Redis unavailable')); + const scheduler = new AuctionDurabilityScheduler( + { + getModel: async (name: string) => { + if (unavailable) throw new Error('Mongo offline'); + return models[name]; + }, + } as never, + { + raw: () => ({ sendCommand }), + getCurrentRedisTimestampMS: async () => 1000, + } as never, + undefined, + undefined, + { mirrorMatchSideEffectStatus: mirror } as never, + { broadcastMatchSideEffect: broadcast } as never, + ); + await scheduler.reconcileDeadLetterStatuses(); + expect(sendCommand).toHaveBeenCalledWith([ + 'ZADD', + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, + '6000', + 'dead-letter', + ]); + unavailable = false; + await scheduler.reconcileDeadLetterStatuses(); + expect( + (await models.auction.findById(f.auctionId).lean()) + ?.matchSideEffectStatus, + ).toBe('failed'); + expect(mirror).toHaveBeenCalledWith( + expect.objectContaining({ + auctionStateTransitionId: 'complete-10', + matchSideEffectStatus: 'failed', + }), + ); + expect(broadcast).toHaveBeenCalledTimes(1); + // Replaying the old notification must not overwrite a new generation. + await models.auction.updateOne( + { _id: f.auctionId }, + { + $set: { + matchSideEffectTransitionId: 'new', + matchSideEffectStatus: 'processing', + matchSideEffectRevision: 20, + }, + }, + ); + await scheduler.reconcileDeadLetterStatuses(); + expect( + (await models.auction.findById(f.auctionId).lean()) + ?.matchSideEffectStatus, + ).toBe('processing'); + expect(broadcast).toHaveBeenCalledTimes(1); + }, +); + +it('cleans up reserved codes and Match metadata after undoing completion', async () => { + const f = await fixture(true); + const scheduler = new AuctionDurabilityScheduler( + f.database as never, + redis as never, + f.codes, + f.results, + undefined, + undefined, + { + readAuctionStateView: async () => ({ + auctionLifecycleId: 'life', + phase: 'bidding', + matchSideEffectTransitionId: 'undo', + }), + } as never, + ); + await scheduler.applyDurabilityPayload({ + auctionId: f.auctionId, + matchId: f.matchId, + auctionLifecycleId: 'life', + revision: 11, + auctionHistorySequence: 11, + auctionActionType: 'undo', + auctionStateTransitionId: 'undo', + snapshotBefore: {}, + historyMutation: { type: 'append' }, + matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' }, + afterAuctionMongoProjection: { + phase: 'bidding', + revision: 11, + matchSideEffectTransitionId: 'undo', + matchSideEffectRevision: 1, + matchSideEffectStatus: 'pending', + }, + }); + expect(await models.match.findById(f.matchId).lean()).toMatchObject({ + status: 'team-built', + tournamentCodeIds: [], + gameSchedule: [], + teams: [], + }); + expect( + (await models.match.findById(f.matchId).lean())?.teamBuiltAt, + ).toBeUndefined(); + expect( + (await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus, + ).toBe('completed'); +}); + +it('reads an uncommitted Match in the provided session when cancelling polling by id', async () => { + const f = await fixture(false); + const session = await connection.startSession(); + try { + session.startTransaction(); + const [created] = await models.match.create( + [ + { + communityId: f.communityId, + title: 'Uncommitted', + scheduledAt: new Date(), + }, + ], + { session }, + ); + expect(await models.match.findById(created._id)).toBeNull(); + await f.results.cancelPolling( + created._id as mongoose.Types.ObjectId, + session, + ); + expect( + await models.match.findById(created._id).session(session), + ).not.toBeNull(); + await session.abortTransaction(); + } finally { + await session.endSession(); + } +}); + +it('rolls back failed Undo cleanup and successfully retries with fresh Match versions', async () => { + const f = await fixture(true); + const scheduler = new AuctionDurabilityScheduler( + f.database as never, + redis as never, + f.codes, + f.results, + undefined, + undefined, + { + readAuctionStateView: async () => ({ + auctionLifecycleId: 'life', + phase: 'bidding', + matchSideEffectTransitionId: 'undo-retry', + }), + } as never, + ); + const payload = { + auctionId: f.auctionId, + matchId: f.matchId, + auctionLifecycleId: 'life', + revision: 11, + auctionHistorySequence: 11, + auctionActionType: 'undo', + auctionStateTransitionId: 'undo-retry', + snapshotBefore: {}, + historyMutation: { type: 'append' }, + matchSideEffectMutation: { type: 'undoCompletedAuctionSideEffects' }, + afterAuctionMongoProjection: { + phase: 'bidding', + revision: 11, + matchSideEffectTransitionId: 'undo-retry', + matchSideEffectRevision: 1, + matchSideEffectStatus: 'pending', + }, + }; + jest + .spyOn(f.results, 'cancelPolling') + .mockRejectedValueOnce(new Error('temporary poll cleanup failure')); + await expect(scheduler.applyDurabilityPayload(payload)).rejects.toThrow( + 'temporary poll cleanup failure', + ); + expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe( + 'reserved', + ); + expect(await models.match.findById(f.matchId).lean()).toMatchObject({ + __v: 0, + tournamentCodeIds: [f.codeId], + }); + await scheduler.applyDurabilityPayload(payload); + expect((await models.tournamentCode.findById(f.codeId).lean())?.state).toBe( + 'released', + ); + expect( + (await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus, + ).toBe('completed'); +}); + +it('answers a retransmitted cancellation of the same lifecycle instead of 404', async () => { + const f = await fixture(true); + const request = { + matchId: f.matchId, + communityId: f.communityId, + clientObservedAuctionLifecycleId: 'life', + }; + await f.service.reset(request); + await expect(f.service.reset(request)).resolves.toBeUndefined(); + expect( + (await models.match.findById(f.matchId).lean()) + ?.lastCancelledAuctionLifecycleId, + ).toBe('life'); + await expect( + f.service.reset({ ...request, clientObservedAuctionLifecycleId: 'other' }), + ).rejects.toThrow('Auction not found.'); +}); + +it.each([ + 'new-lifecycle', + 'new-transition', + 'completed', + 'revision-ahead', + 'mirror-superseded', +])('fences terminal notifications against %s', async (scenario) => { + const f = await fixture(false); + await models.auction.updateOne( + { _id: f.auctionId }, + { + $set: { + matchSideEffectTransitionId: 'target', + matchSideEffectRevision: 1, + matchSideEffectStatus: 'pending', + }, + }, + ); + const payload = { + auctionId: f.auctionId.toString(), + matchId: f.matchId.toString(), + auctionLifecycleId: 'life', + auctionStateTransitionId: 'target', + matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' }, + afterAuctionMongoProjection: { matchSideEffectRevision: 1 }, + }; + const state = { + auctionLifecycleId: scenario === 'new-lifecycle' ? 'new-life' : 'life', + matchSideEffectTransitionId: + scenario === 'new-transition' ? 'new' : 'target', + matchSideEffectStatus: scenario === 'completed' ? 'completed' : 'pending', + matchSideEffectRevision: 40, + }; + const sendCommand = jest.fn(async (command: string[]) => { + if (command[0] === 'ZRANGEBYSCORE') return ['terminal']; + if (command[0] === 'HGETALL') + return { payloadJson: JSON.stringify(payload) }; + if (command[0] === 'HGET') return 'exhausted'; + return 1; + }); + const broadcast = jest.fn(); + const mirror = jest.fn().mockResolvedValue(scenario !== 'mirror-superseded'); + const scheduler = new AuctionDurabilityScheduler( + f.database as never, + { + raw: () => ({ sendCommand }), + getCurrentRedisTimestampMS: async () => 1000, + } as never, + undefined, + undefined, + { mirrorMatchSideEffectStatus: mirror } as never, + { broadcastMatchSideEffect: broadcast } as never, + { readAuctionStateView: async () => state } as never, + ); + await scheduler.reconcileDeadLetterStatuses(); + const changed = ['revision-ahead', 'mirror-superseded'].includes(scenario); + expect( + (await models.auction.findById(f.auctionId).lean())?.matchSideEffectStatus, + ).toBe(changed ? 'failed' : 'pending'); + if (changed) + expect(mirror).toHaveBeenCalledWith( + expect.objectContaining({ matchSideEffectRevision: 41 }), + ); + expect(broadcast).toHaveBeenCalledTimes( + scenario === 'revision-ahead' ? 1 : 0, + ); +}); diff --git a/apps/api/test/auction-correctness.integration-spec.ts b/apps/api/test/auction-correctness.integration-spec.ts new file mode 100644 index 00000000..fbf97a49 --- /dev/null +++ b/apps/api/test/auction-correctness.integration-spec.ts @@ -0,0 +1,598 @@ +import { createClient } from 'redis'; +import mongoose from 'mongoose'; +import { AuctionService } from '../src/modules/auction/auction.service'; +import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service'; +import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service'; +import { AuctionRedisStateReaderService } from '../src/modules/auction/redis/auction-redis-state-reader.service'; +import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys'; +import { + AuctionLifecycleLock, + AuctionLifecycleBusyError, +} from '../src/modules/auction/auction-lifecycle-lock'; +import { AuctionDurabilityScheduler } from '../src/modules/auction/auction-durability.scheduler'; +import { AuctionSideEffectService } from '../src/modules/auction/auction-side-effect.service'; +import { AuctionBidArchiveService } from '../src/modules/auction/auction-bid-archive.service'; +import { + AUCTION_DURABILITY_RETRY_QUEUE_KEY, + AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, + AUCTION_BID_ARCHIVE_QUEUE_KEY, + AUCTION_BID_ARCHIVE_PAYLOADS_KEY, + AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, +} from '../src/modules/auction/redis/auction-redis.keys'; + +const client = createClient({ + url: process.env.REDIS_TEST_URL, + socket: { reconnectStrategy: false }, +}); +const redis = new AuctionRedisService(client); +const commands = new AuctionRedisCommandService(redis); +const reader = new AuctionRedisStateReaderService(redis); + +beforeAll(async () => { + if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) { + throw new Error('Use test:redis to start an isolated local Redis server.'); + } + client.on('error', () => {}); + await client.connect(); +}); +beforeEach(async () => { + await client.flushDb(); +}); +afterAll(async () => { + if (client.isOpen) await client.quit(); +}); + +it.each(['count', 'elapsed'])( + 'dead-letters permanently pending side effects by %s while retaining the payload', + async (limit) => { + const key = 'review:pending'; + const now = Date.now(); + await client.hSet(key, { + payloadJson: '{}', + pendingAttempts: limit === 'count' ? '119' : '1', + pendingSinceTimestampMS: String(limit === 'elapsed' ? now - 600000 : now), + }); + await client.zAdd(AUCTION_DURABILITY_RETRY_QUEUE_KEY, { + score: now, + value: key, + }); + const scheduler = new AuctionDurabilityScheduler({} as never, redis); + const internal = scheduler as unknown as { + deferDurabilityPayload( + key: string, + time: number, + error: unknown, + ): Promise; + }; + expect( + await internal.deferDurabilityPayload( + key, + now, + new Error('state unavailable'), + ), + ).toBe('dead-lettered'); + expect( + await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key), + ).toBeNull(); + expect( + await client.zScore(AUCTION_DURABILITY_DEAD_LETTER_QUEUE_KEY, key), + ).toBe(now); + expect(await client.hGet(key, 'payloadJson')).toBe('{}'); + expect(await client.hGet(key, 'attempts')).toBeNull(); + expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe( + now, + ); + }, +); + +it('defers a temporary coordination wait without consuming failed-work attempts or recreating removed payloads', async () => { + const key = 'review:pending'; + const now = Date.now(); + const scheduler = new AuctionDurabilityScheduler({} as never, redis); + const internal = scheduler as unknown as { + deferDurabilityPayload( + key: string, + time: number, + error: unknown, + ): Promise; + }; + expect( + await internal.deferDurabilityPayload(key, now, new Error('busy')), + ).toBe('gone'); + expect(await client.exists(key)).toBe(0); + await client.hSet(key, 'payloadJson', '{}'); + expect( + await internal.deferDurabilityPayload(key, now, new Error('busy')), + ).toBe('requeued'); + expect(await client.zScore(AUCTION_DURABILITY_RETRY_QUEUE_KEY, key)).toBe( + now + 5000, + ); + expect(await client.hGet(key, 'attempts')).toBeNull(); +}); + +async function fixture() { + const matchId = new mongoose.Types.ObjectId(); + const id = matchId.toString(); + const lifecycle = `life:${id}`; + const playerA = new mongoose.Types.ObjectId().toString(); + const playerB = new mongoose.Types.ObjectId().toString(); + const captain = new mongoose.Types.ObjectId().toString(); + const initialRound = `initial:${id}`; + const structure = { + auctionPlayerOrder: [{ userId: playerA }, { userId: playerB }], + auctionTeams: [ + { + teamIndex: 1, + captainUserId: captain, + remainingBidPointAmount: 100000, + members: [], + }, + ], + currentAuctionPlayerOrderIndex: 0, + currentAuctionRound: 1, + passedAuctionPlayerQueue: [], + maximumBidRoundDurationSeconds: 30, + bidRoundExtensionSeconds: 5, + initialTeamBidPointAmount: 100000, + topTierAuctionPriorityThreshold: 'NONE', + manualTopTierAuctionPriorityUserIds: [], + }; + await commands.hydrateState(id, { + matchId: id, + auctionLifecycleId: lifecycle, + revision: 0, + auctionHistorySequence: 1, + phase: 'bidding', + maximumBidRoundDurationSeconds: 30, + structureStateJson: JSON.stringify(structure), + liveStatePatch: { + currentAuctionPlayerOrderIndex: 0, + currentBidRoundPlayerUserId: playerA, + currentBidRoundId: initialRound, + maximumBidRoundDurationSeconds: 30, + bidRoundExtensionSeconds: 5, + 'team:1:captainUserId': captain, + 'team:1:remainingBidPointAmount': 100000, + 'team:1:assignedAuctionTeamMemberCount': 1, + }, + }); + const auction = { + _id: new mongoose.Types.ObjectId(), + matchId, + auctionEngineVersion: 'redis-v1', + auctionLifecycleId: lifecycle, + phase: 'bidding', + revision: 1, + }; + const service = new AuctionService( + { getModel: async () => ({ findOne: async () => auction }) } as never, + {} as never, + {} as never, + {} as never, + redis, + commands, + reader, + ); + const bid = ( + requestId: string, + amount: number, + player = playerA, + round = initialRound, + ) => ({ + matchId: id, + bidderTeamIndex: 1, + bidderUserId: captain, + bidPointAmount: amount, + bidRequestId: requestId, + clientObservedAuctionLifecycleId: lifecycle, + clientObservedCurrentBidRoundPlayerUserId: player, + clientObservedBidRoundId: round, + maximumAuctionTeamMemberCount: 5, + minimumBidPointReservePerRemainingAuctionTeamMemberSlot: 10, + maximumStoredAuctionBidHistoryCount: 1000, + maximumStoredBidRequestResultCount: 500, + minimumBidRequestResultReplayWindowMS: 300000, + }); + const undo = async (snapshot: Record) => { + const internal = service as unknown as { + undoRedisAuction(params: unknown): Promise; + }; + return internal.undoRedisAuction({ + auction, + snapshot, + targetHistorySequence: 1, + undoingCompletedAuction: false, + }); + }; + return { + id, + matchId, + lifecycle, + playerA, + playerB, + initialRound, + auction, + service, + bid, + undo, + keys: buildAuctionRedisKeys(id), + }; +} + +it('waits briefly for a busy recovery lock and returns the recovered state', async () => { + const f = await fixture(); + f.auction.revision = 10; + const key = `auction:${f.id}:lifecycleMutationLock`; + await redis.acquireTokenLock(key, 'other-reader', 30000); + const released = new Promise((resolve, reject) => { + setTimeout(() => { + void redis.releaseTokenLock(key, 'other-reader').then(resolve, reject); + }, 100); + }); + const [view] = await Promise.all([ + f.service.getStateByMatchId(f.matchId), + released, + ]); + expect(view).toMatchObject({ auctionLifecycleId: f.lifecycle, revision: 11 }); +}); + +it('bounds recovery waits and reports a retriable state error without releasing another owner lock', async () => { + const f = await fixture(); + f.auction.revision = 10; + const key = `auction:${f.id}:lifecycleMutationLock`; + await redis.acquireTokenLock(key, 'slow-owner', 30000); + await expect(f.service.getStateByMatchId(f.matchId)).rejects.toMatchObject({ + response: { code: 'AUCTION_STATE_RECOVERING', retryAfterMS: 500 }, + }); + expect(await client.get(key)).toBe('slow-owner'); +}); + +it('keeps accepting bids after 1000 history entries and preserves accepted request replay across players', async () => { + const f = await fixture(); + const first = await commands.placeBid( + f.id, + f.bid('accepted-before-next', 10), + ); + expect(first).toMatchObject({ ok: true }); + await client.rPush( + f.keys.bidHistoryKey, + Array.from({ length: 999 }, () => JSON.stringify({ bidPointAmount: 10 })), + ); + await client.hSet(f.keys.liveStateKey, 'phase', 'sold'); + await f.service.next(f.matchId); + await f.service.openBidding(f.matchId); + const state = (await reader.readAuctionStateView(f.id))!; + const result = await commands.placeBid( + f.id, + f.bid('next-player', 10, f.playerB, String(state.currentBidRoundId)), + ); + expect(result).toMatchObject({ ok: true }); + expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000); + expect( + await commands.placeBid(f.id, f.bid('accepted-before-next', 10)), + ).toMatchObject({ ...first, isReplay: true }); +}); + +it('500 rejected requests do not block a valid bid, and rejection retention stays bounded', async () => { + const f = await fixture(); + await client.hSet(f.keys.liveStateKey, 'currentHighestBidPointAmount', '10'); + for (let i = 0; i < 501; i++) { + expect( + await commands.placeBid(f.id, f.bid(`too-low:${i}`, 5)), + ).toMatchObject({ rejected: true, code: 'TOO_LOW' }); + } + expect(await client.lLen(f.keys.rejectedBidRequestResultOrderKey)).toBe(500); + expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(0); + expect(await commands.placeBid(f.id, f.bid('valid', 20))).toMatchObject({ + ok: true, + }); + expect(await commands.placeBid(f.id, f.bid('valid', 30))).toMatchObject({ + rejected: true, + code: 'DUPLICATE_REQUEST_CONFLICT', + }); +}); + +it('still protects 500 accepted results within their replay window', async () => { + const f = await fixture(); + for (let i = 1; i <= 500; i++) + expect( + await commands.placeBid(f.id, f.bid(`accepted:${i}`, i)), + ).toMatchObject({ ok: true }); + expect(await commands.placeBid(f.id, f.bid('overflow', 501))).toMatchObject({ + rejected: true, + code: 'BID_REQUEST_RESULT_LIMIT', + }); + expect(await commands.placeBid(f.id, f.bid('accepted:1', 1))).toMatchObject({ + ok: true, + isReplay: true, + }); +}); + +it('migrates rejected results from the former shared quota without losing accepted replays', async () => { + const f = await fixture(); + await commands.placeBid(f.id, f.bid('accepted', 10)); + await client.hDel(f.keys.bidRequestResultsKey, '_acceptedOnlyOrderV1'); + for (let i = 0; i < 499; i++) { + const id = `legacy-rejected:${i}`; + await client.hSet(f.keys.bidRequestResultsKey, { + [`${id}:status`]: 'rejected', + [`${id}:createdAtTimestampMS`]: String(Date.now()), + }); + await client.rPush(f.keys.bidRequestResultOrderKey, id); + } + expect( + await commands.placeBid(f.id, f.bid('next-accepted', 20)), + ).toMatchObject({ ok: true }); + expect(await client.lLen(f.keys.bidRequestResultOrderKey)).toBe(2); + expect(await commands.placeBid(f.id, f.bid('accepted', 10))).toMatchObject({ + ok: true, + isReplay: true, + }); +}); + +it('encodes opaque round identities safely in bid history', async () => { + const f = await fixture(); + const round = 'round:with"quotes\\and-slashes'; + await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', round); + expect( + await commands.placeBid(f.id, f.bid('quoted-round', 10, f.playerA, round)), + ).toMatchObject({ ok: true, auctionBidHistoryEntry: { bidRoundId: round } }); + expect((await reader.readAuctionStateView(f.id))?.auctionBidHistory).toEqual([ + expect.objectContaining({ bidRoundId: round }), + ]); +}); + +it('rejects delayed pre-undo bids and omitted round identities while accepting the restored round', async () => { + const f = await fixture(); + const snapshot = (await reader.readAuctionStateView(f.id))!; + await f.undo(snapshot); + const restored = (await reader.readAuctionStateView(f.id))!; + expect(restored.currentBidRoundId).not.toBe(f.initialRound); + expect(await commands.placeBid(f.id, f.bid('old-round', 100))).toMatchObject({ + code: 'STALE_BID_ROUND', + }); + expect( + await commands.placeBid(f.id, { + ...f.bid('missing-round', 100), + clientObservedBidRoundId: undefined, + }), + ).toMatchObject({ code: 'BID_CLIENT_UPDATE_REQUIRED' }); + expect( + await commands.placeBid( + f.id, + f.bid('new-round', 100, f.playerA, String(restored.currentBidRoundId)), + ), + ).toMatchObject({ + ok: true, + auctionBidHistoryEntry: { + playerUserId: f.playerA, + bidRoundId: restored.currentBidRoundId, + }, + }); +}); + +it('removes completion TTLs when resuming an auction', async () => { + const f = await fixture(); + const snapshot = (await reader.readAuctionStateView(f.id))!; + await client.expire(f.keys.liveStateKey, 604800); + await client.expire(f.keys.structureStateKey, 604800); + await f.undo(snapshot); + expect(await client.ttl(f.keys.liveStateKey)).toBe(-1); + expect(await client.ttl(f.keys.structureStateKey)).toBe(-1); +}); + +it('fences direct foreign-lifecycle hydration and repairs an orphan using current Mongo authority', async () => { + const f = await fixture(); + await client.hSet(f.keys.liveStateKey, 'revision', '150'); + expect( + await commands.hydrateState(f.id, { + matchId: f.id, + auctionLifecycleId: 'replacement', + revision: 0, + auctionHistorySequence: 1, + phase: 'presenting', + maximumBidRoundDurationSeconds: 30, + structureStateJson: '{}', + }), + ).toMatchObject({ rejected: true, code: 'LIFECYCLE_CONFLICT' }); + const current = { + ...f.auction, + auctionLifecycleId: 'replacement', + phase: 'presenting', + revision: 1, + auctionTeams: [], + auctionPlayerOrder: [], + }; + jest.spyOn(f.service, 'getByMatchId').mockResolvedValue(current as never); + expect(await f.service.getStateByMatchId(f.matchId)).toMatchObject({ + auctionLifecycleId: 'replacement', + phase: 'presenting', + }); + expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe( + 'replacement', + ); +}); + +it('does not let an old Mongo read overwrite the newly started lifecycle', async () => { + const f = await fixture(); + const stale = { ...f.auction, auctionLifecycleId: 'obsolete', revision: 500 }; + jest + .spyOn(f.service, 'getByMatchId') + .mockResolvedValueOnce(stale as never) + .mockResolvedValue(f.auction as never); + await expect(f.service.getStateByMatchId(f.matchId)).rejects.toThrow( + 'lifecycle has changed', + ); + expect(await client.hGet(f.keys.liveStateKey, 'auctionLifecycleId')).toBe( + f.lifecycle, + ); +}); + +it('serializes lifecycle mutations without deleting another owner lock', async () => { + const lock = new AuctionLifecycleLock(redis); + await lock.run('match', async () => { + await expect(lock.run('match', async () => {})).rejects.toThrow( + AuctionLifecycleBusyError, + ); + }); + await expect(lock.run('match', async () => 'done')).resolves.toBe('done'); +}); + +it('persists the exact Redis deadline when opening bidding and restoring a bidding snapshot', async () => { + const f = await fixture(); + await client.hSet(f.keys.liveStateKey, 'phase', 'presenting'); + await client.hDel(f.keys.liveStateKey, 'currentBidRoundDeadlineTimestampMS'); + await f.service.openBidding(f.matchId); + const scheduler = new AuctionDurabilityScheduler({} as never, redis); + const verify = async () => { + const live = (await reader.readAuctionStateView(f.id))!; + const queued = await client.zRange( + AUCTION_DURABILITY_RETRY_QUEUE_KEY, + 0, + -1, + ); + const payloads = await Promise.all( + queued.map((key) => scheduler.readDurabilityPayload(key)), + ); + const payload = payloads.find( + (entry) => entry?.revision === live.revision, + )!; + expect( + payload.afterAuctionMongoProjection.currentBidRoundDeadlineTimestampMS, + ).toBe(live.currentBidRoundDeadlineTimestampMS); + expect( + new Date( + String(payload.afterAuctionMongoProjection.currentDeadline), + ).getTime(), + ).toBe(live.currentBidRoundDeadlineTimestampMS); + expect( + payload.afterAuctionMongoProjection.passedAuctionPlayerQueue, + ).toEqual([]); + return live; + }; + const snapshot = await verify(); + await f.undo({ ...snapshot, currentBidRoundDeadlineTimestampMS: 1 }); + await verify(); +}); + +it('archives every accepted bid beyond the recent 1000, with no duplicate archive event for a replay', async () => { + const f = await fixture(); + for (let amount = 1; amount <= 1005; amount++) { + expect( + await commands.placeBid(f.id, { + ...f.bid(`archive:${amount}`, amount), + maximumStoredBidRequestResultCount: 2000, + }), + ).toMatchObject({ ok: true }); + } + expect(await client.lLen(f.keys.bidHistoryKey)).toBe(1000); + expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005); + expect( + await commands.placeBid(f.id, { + ...f.bid('archive:1', 1), + maximumStoredBidRequestResultCount: 2000, + }), + ).toMatchObject({ ok: true, isReplay: true }); + expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(1005); + const [first] = await client.zRange(AUCTION_BID_ARCHIVE_QUEUE_KEY, 0, 0); + expect( + JSON.parse((await client.hGet(AUCTION_BID_ARCHIVE_PAYLOADS_KEY, first))!), + ).toMatchObject({ + bidPointAmount: 1, + bidRequestId: 'archive:1', + playerUserId: f.playerA, + auctionLifecycleId: f.lifecycle, + }); +}); + +it.each(['missing', 'empty'])( + 'archives an accepted pre-upgrade bid with a %s Redis round ID', + async (kind) => { + const f = await fixture(); + if (kind === 'missing') + await client.hDel(f.keys.liveStateKey, 'currentBidRoundId'); + else await client.hSet(f.keys.liveStateKey, 'currentBidRoundId', ''); + expect( + await commands.placeBid(f.id, { + ...f.bid('legacy-round', 10), + clientObservedBidRoundId: undefined, + }), + ).toMatchObject({ + ok: true, + auctionBidHistoryEntry: { bidRoundId: '', playerUserId: f.playerA }, + }); + const bulkWrite = jest.fn(async (operations: unknown[]) => ({ + matchedCount: 0, + upsertedCount: operations.length, + })); + const archive = new AuctionBidArchiveService( + { getModel: async () => ({ bulkWrite }) } as never, + redis, + ); + await expect(archive.persistPendingBids()).resolves.toBe(1); + const write = bulkWrite.mock.calls[0][0][0] as { + updateOne: { update: { $setOnInsert: Record } }; + }; + expect(write.updateOne.update.$setOnInsert).not.toHaveProperty( + 'bidRoundId', + ); + expect(String(write.updateOne.update.$setOnInsert.playerUserId)).toBe( + f.playerA, + ); + expect(await client.zCard(AUCTION_BID_ARCHIVE_QUEUE_KEY)).toBe(0); + }, +); + +it('does not mirror an old generation failure over a newer Redis side effect', async () => { + const f = await fixture(); + await client.hSet(f.keys.liveStateKey, { + matchSideEffectTransitionId: 'new', + matchSideEffectRevision: '2', + matchSideEffectStatus: 'pending', + }); + const sideEffects = new AuctionSideEffectService(redis); + expect( + await sideEffects.mirrorMatchSideEffectStatus({ + matchId: f.id, + auctionLifecycleId: f.lifecycle, + auctionStateTransitionId: 'old', + matchSideEffectRevision: 100, + matchSideEffectStatus: 'failed', + }), + ).toBe(false); + expect(await client.hGet(f.keys.liveStateKey, 'matchSideEffectStatus')).toBe( + 'pending', + ); +}); + +it('queues a terminal notification for ordinary failures and backfills old dead letters', async () => { + const key = 'review:ordinary-failure'; + await client.hSet(key, { payloadJson: '{}', attempts: '9' }); + const scheduler = new AuctionDurabilityScheduler({} as never, redis); + const internal = scheduler as unknown as { + recordDurabilityPayloadFailure( + key: string, + now: number, + error: unknown, + ): Promise; + }; + expect( + await internal.recordDurabilityPayloadFailure( + key, + 1000, + new Error('failed before worker'), + ), + ).toBe('dead-lettered'); + expect(await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key)).toBe( + 1000, + ); + // Simulate an older deployment which did not create notification jobs. + await client.zRem(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key); + await scheduler.reconcileDeadLetterStatuses(); + expect( + await client.hGet(key, 'deadLetterStatusReconciledAtTimestampMS'), + ).not.toBeNull(); + expect( + await client.zScore(AUCTION_DEAD_LETTER_STATUS_QUEUE_KEY, key), + ).toBeNull(); +}); diff --git a/apps/api/test/jest-mongo.json b/apps/api/test/jest-mongo.json new file mode 100644 index 00000000..8760ff59 --- /dev/null +++ b/apps/api/test/jest-mongo.json @@ -0,0 +1,7 @@ +{ + "rootDir": "..", + "testEnvironment": "node", + "testRegex": "test/.*\\.mongo-spec\\.ts$", + "transform": { "^.+\\.ts$": "ts-jest" }, + "testTimeout": 15000 +} diff --git a/apps/api/test/run-mongo-tests.mjs b/apps/api/test/run-mongo-tests.mjs new file mode 100644 index 00000000..0d4865fc --- /dev/null +++ b/apps/api/test/run-mongo-tests.mjs @@ -0,0 +1,134 @@ +import { spawn } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { mkdtemp, mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, relative, resolve } from 'node:path'; +import { createServer } from 'node:net'; +import { setTimeout as delay } from 'node:timers/promises'; +import mongoose from 'mongoose'; + +const require = createRequire(import.meta.url); +const listener = createServer(); +await new Promise((accept) => listener.listen(0, '127.0.0.1', accept)); +const port = listener.address().port; +await new Promise((accept) => listener.close(accept)); +const directory = await mkdtemp(join(tmpdir(), 'lolympus-mongo-test-')); +await mkdir(join(directory, 'data')); +const mongo = spawn( + process.env.MONGODB_TEST_SERVER || 'mongod', + [ + '--bind_ip', + '127.0.0.1', + '--port', + String(port), + '--replSet', + 'auction_review', + '--dbpath', + join(directory, 'data'), + '--logpath', + join(directory, 'mongod.log'), + '--wiredTigerCacheSizeGB', + '0.25', + ], + { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }, +); +let startupError; +mongo.on('error', (error) => { + startupError = error; +}); +const closed = new Promise((accept) => mongo.once('close', accept)); +let output = ''; +mongo.stdout.on('data', (chunk) => { + output += chunk; +}); +mongo.stderr.on('data', (chunk) => { + output += chunk; +}); +const uri = `mongodb://127.0.0.1:${port}/auction_review`; +const client = new mongoose.mongo.MongoClient(`${uri}?directConnection=true`, { + serverSelectionTimeoutMS: 500, +}); +try { + let connected = false; + for (let attempt = 0; attempt < 60; attempt++) { + if (startupError) throw startupError; + if (mongo.exitCode !== null) throw new Error(`MongoDB exited: ${output}`); + try { + await client.connect(); + connected = true; + break; + } catch { + await delay(100); + } + } + if (!connected) throw new Error(`MongoDB did not start: ${output}`); + await client + .db('admin') + .command({ + replSetInitiate: { + _id: 'auction_review', + members: [{ _id: 0, host: `127.0.0.1:${port}` }], + }, + }); + let primary = false; + for (let attempt = 0; attempt < 100; attempt++) { + if ((await client.db('admin').command({ hello: 1 })).isWritablePrimary) { + primary = true; + break; + } + await delay(100); + } + if (!primary) + throw new Error('MongoDB replica set did not elect its primary.'); + const build = await client.db('admin').command({ buildInfo: 1 }); + console.log( + `Integration tests using isolated MongoDB ${build.version}, Mongoose ${mongoose.version}, loopback port ${port}`, + ); + const jest = spawn( + process.execPath, + [ + require.resolve('jest/bin/jest'), + '--config', + 'test/jest-mongo.json', + '--runInBand', + ...process.argv.slice(2), + ], + { + cwd: resolve(import.meta.dirname, '..'), + windowsHide: true, + stdio: 'inherit', + env: { + ...process.env, + NODE_ENV: 'test', + MONGO_TEST_ISOLATED: '1', + MONGO_TEST_URL: `${uri}?replicaSet=auction_review`, + }, + }, + ); + process.exitCode = await new Promise((accept, reject) => { + jest.once('error', reject); + jest.once('exit', (code) => accept(code ?? 1)); + }); +} finally { + await client + .db('admin') + .command({ shutdown: 1, force: true }) + .catch(() => {}); + await client.close().catch(() => {}); + if (mongo.pid && mongo.exitCode === null) mongo.kill(); + await closed; + // Only remove the unique temporary directory created by this runner. + const target = relative(resolve(tmpdir()), resolve(directory)); + if ( + !target.startsWith('lolympus-mongo-test-') || + target.includes('..') || + /[/\\]/.test(target) + ) + throw new Error('Unsafe MongoDB test directory.'); + await rm(directory, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 100, + }); +} diff --git a/apps/web/src/api/auction-resync.ts b/apps/web/src/api/auction-resync.ts new file mode 100644 index 00000000..6c4f4926 --- /dev/null +++ b/apps/web/src/api/auction-resync.ts @@ -0,0 +1,83 @@ +import type { ApiAuction } from './types'; + +export interface AuctionResyncReply { + ok?: boolean; + state?: ApiAuction; + retryAfterMS?: number; +} + +function waitForRetry( + milliseconds: number, + signal: AbortSignal, +): Promise { + return new Promise((resolve) => { + const finish = () => { + clearTimeout(timer); + signal.removeEventListener('abort', finish); + resolve(); + }; + const timer = setTimeout(finish, milliseconds); + signal.addEventListener('abort', finish, { once: true }); + if (signal.aborted) finish(); + }); +} + +/** Coalesces simultaneous gaps and discards replies from an old connection. */ +export function createAuctionResyncRequester( + fetchState: () => Promise, + applyState: (state: ApiAuction) => void, + wait = waitForRetry, +) { + let pending: Promise | undefined; + let controller = new AbortController(); + let minimumRevision = -1; + let lifecycleId: string | undefined; + return { + request(revision = -1, lifecycle?: string): Promise { + if (revision >= 0) { + minimumRevision = + lifecycle === lifecycleId + ? Math.max(minimumRevision, revision) + : revision; + lifecycleId = lifecycle; + } + if (pending) return pending; + const signal = controller.signal; + pending = (async () => { + for (let attempt = 0; attempt < 3 && !signal.aborted; attempt++) { + let retryAfterMS = 500 * (attempt + 1); + try { + const reply = await fetchState(); + if (signal.aborted) return; + if (reply.ok && reply.state) { + applyState(reply.state); + // Another gap may have arrived while this snapshot was in flight. + if ( + reply.state.auctionLifecycleId !== lifecycleId || + (reply.state.revision ?? -1) >= minimumRevision + ) + return; + } + retryAfterMS = Math.min( + 3000, + Math.max(retryAfterMS, reply.retryAfterMS ?? 0), + ); + } catch { + /* A timeout gets a bounded retry, never another room join. */ + } + if (attempt < 2 && !signal.aborted) await wait(retryAfterMS, signal); + } + })().finally(() => { + if (!signal.aborted) pending = undefined; + }); + return pending; + }, + cancel() { + controller.abort(); + controller = new AbortController(); + pending = undefined; + minimumRevision = -1; + lifecycleId = undefined; + }, + }; +} diff --git a/apps/web/src/api/auction-state.ts b/apps/web/src/api/auction-state.ts new file mode 100644 index 00000000..fbdd4ba4 --- /dev/null +++ b/apps/web/src/api/auction-state.ts @@ -0,0 +1,239 @@ +import type { ApiAuction } from "./types"; + +export interface BidDeltaPayload { + currentBidRoundId?: string; + auctionBidHistoryEntry?: { + bidderTeamIndex?: number; + bidPointAmount?: number; + bidderUserId?: string; + bidPlacedAtTimestampMS?: number; + bidRoundId?: string; + playerUserId?: string; + }; + auctionLifecycleId?: string; + bidRequestId?: string; + currentBidRoundDeadlineTimestampMS?: number; + currentHighestBidderTeamIndex?: number; + currentHighestBidPointAmount?: number; + currentServerTimestampMS?: number; + revision?: number; +} + +export function shouldApplyIncomingAuctionState( + current: ApiAuction | null, + incoming: ApiAuction, +): boolean { + if (!current) return true; + if (incoming.shouldForceAuctionStateResync) return true; + if ( + current.auctionLifecycleId && + incoming.auctionLifecycleId && + current.auctionLifecycleId !== incoming.auctionLifecycleId + ) { + return true; + } + if ( + typeof current.revision === "number" && + typeof incoming.revision === "number" && + incoming.revision < current.revision + ) { + return false; + } + return true; +} + +/** Full state can repair a delta at the same revision; side effects have a + * separate revision and must not regress when a delayed full state arrives. */ +export function mergeAuctionState( + current: ApiAuction | null, + incoming: ApiAuction, +): ApiAuction { + if ( + current?.auctionLifecycleId === incoming.auctionLifecycleId && + current?.matchSideEffectTransitionId === + incoming.matchSideEffectTransitionId && + (current?.matchSideEffectRevision ?? -1) > + (incoming.matchSideEffectRevision ?? -1) + ) { + return { + ...incoming, + matchSideEffectRevision: current!.matchSideEffectRevision, + matchSideEffectStatus: current!.matchSideEffectStatus, + }; + } + return incoming; +} + +export function shouldResyncForBidDelta( + current: ApiAuction | null, + payload: BidDeltaPayload, +): boolean { + if (!current) return true; + if ( + payload.auctionLifecycleId && + current.auctionLifecycleId && + payload.auctionLifecycleId !== current.auctionLifecycleId + ) + return true; + if ( + typeof payload.revision !== "number" || + typeof current.revision !== "number" + ) + return true; + if (payload.revision <= current.revision) return false; + return ( + payload.revision !== current.revision + 1 || + current.phase !== "bidding" || + Boolean( + current.currentBidRoundId && + payload.currentBidRoundId !== current.currentBidRoundId, + ) + ); +} + +export function applyBidDelta( + current: ApiAuction | null, + payload: BidDeltaPayload, +): ApiAuction | null { + if (!current) return current; + if (shouldResyncForBidDelta(current, payload)) return current; + if ( + payload.auctionLifecycleId && + current.auctionLifecycleId && + payload.auctionLifecycleId !== current.auctionLifecycleId + ) { + return current; + } + if ( + typeof current.revision === "number" && + typeof payload.revision === "number" && + payload.revision <= current.revision + ) { + return current; + } + + const bidPointAmount = Number(payload.currentHighestBidPointAmount); + const bidderTeamIndex = Number(payload.currentHighestBidderTeamIndex); + if (!Number.isFinite(bidPointAmount) || !Number.isFinite(bidderTeamIndex)) { + return current; + } + + const bidHistoryEntry = normalizeBidHistoryEntry( + payload.auctionBidHistoryEntry, + ); + const currentBidRoundDeadlineTimestampMS = + typeof payload.currentBidRoundDeadlineTimestampMS === "number" + ? payload.currentBidRoundDeadlineTimestampMS + : current.currentBidRoundDeadlineTimestampMS; + + return { + ...current, + auctionBidHistory: appendUniqueAuctionBidHistoryEntry( + current.auctionBidHistory ?? [], + bidHistoryEntry, + ), + bids: bidHistoryEntry + ? appendUniqueLegacyBidEntry(current.bids ?? [], bidHistoryEntry) + : current.bids, + currentBid: bidPointAmount, + currentBidderTeamIndex: bidderTeamIndex, + currentDeadline: currentBidRoundDeadlineTimestampMS + ? new Date(currentBidRoundDeadlineTimestampMS).toISOString() + : current.currentDeadline, + currentHighestBidPointAmount: bidPointAmount, + currentHighestBidderTeamIndex: bidderTeamIndex, + currentBidRoundDeadlineTimestampMS, + currentServerTimestampMS: + payload.currentServerTimestampMS ?? current.currentServerTimestampMS, + revision: payload.revision ?? current.revision, + currentBidRoundId: payload.currentBidRoundId ?? current.currentBidRoundId, + }; +} + +function normalizeBidHistoryEntry( + entry: BidDeltaPayload["auctionBidHistoryEntry"], +): NonNullable[number] | null { + if (!entry) return null; + const bidderTeamIndex = Number(entry.bidderTeamIndex); + const bidPointAmount = Number(entry.bidPointAmount); + const bidPlacedAtTimestampMS = Number(entry.bidPlacedAtTimestampMS); + if (!Number.isFinite(bidderTeamIndex) || !Number.isFinite(bidPointAmount)) { + return null; + } + return { + bidRoundId: entry.bidRoundId, + playerUserId: entry.playerUserId, + bidderTeamIndex, + bidPointAmount, + bidderUserId: entry.bidderUserId ?? "", + bidPlacedAtTimestampMS: Number.isFinite(bidPlacedAtTimestampMS) + ? bidPlacedAtTimestampMS + : Date.now(), + }; +} + +function appendUniqueAuctionBidHistoryEntry( + current: NonNullable, + entry: NonNullable[number] | null, +) { + if (!entry) return current; + if ( + current.some( + (existing) => + existing.bidderTeamIndex === entry.bidderTeamIndex && + existing.bidPointAmount === entry.bidPointAmount && + existing.bidPlacedAtTimestampMS === entry.bidPlacedAtTimestampMS, + ) + ) { + return current; + } + return [...current, entry].slice(-1000); +} + +function appendUniqueLegacyBidEntry( + current: ApiAuction["bids"], + entry: NonNullable[number], +) { + const currentBids = current ?? []; + const legacyBid = { + amount: entry.bidPointAmount, + bidderUserId: entry.bidderUserId, + teamIndex: entry.bidderTeamIndex, + timestamp: new Date(entry.bidPlacedAtTimestampMS).toISOString(), + }; + if ( + currentBids.some( + (existing) => + existing.teamIndex === legacyBid.teamIndex && + existing.amount === legacyBid.amount && + existing.timestamp === legacyBid.timestamp, + ) + ) { + return currentBids; + } + return [...currentBids, legacyBid].slice(-1000); +} + +/** HTTP 폴링 결과를 소켓 내부 상태(입찰 기준가·라운드 ID)에도 반영할지 결정한다. + * lifecycle 교체는 소켓 이벤트로만 받는다. 늦게 도착한 이전 경매의 응답이 + * 새 경매를 덮어쓰지 않도록 기존 상태가 있으면 같은 lifecycle만 반영한다. */ +export function shouldApplyPolledAuctionState( + current: ApiAuction | null, + incoming: ApiAuction, + matchId: string | undefined, +): boolean { + if (!matchId || incoming.matchId !== matchId) return false; + if (!incoming.auctionLifecycleId || !Number.isFinite(incoming.revision)) + return false; + if (!current) return true; + if ( + !current.auctionLifecycleId || + current.auctionLifecycleId !== incoming.auctionLifecycleId + ) + return false; + return ( + typeof current.revision === "number" && + typeof incoming.revision === "number" && + incoming.revision >= current.revision + ); +} diff --git a/apps/web/src/api/auctionRejectionMessages.ts b/apps/web/src/api/auctionRejectionMessages.ts index 8d13ab01..3970b078 100644 --- a/apps/web/src/api/auctionRejectionMessages.ts +++ b/apps/web/src/api/auctionRejectionMessages.ts @@ -28,7 +28,10 @@ const REASON_TO_USER_MESSAGE: Record = { AUCTION_TRANSITION_REJECTED_BECAUSE_REQUEST_COORDINATION_RETRYABLE: '잠시 처리 중이라 반영되지 않았어요. 다시 시도해 주세요.', // 입찰 - BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED: '이미 다음 선수로 넘어갔어요.', + BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED: + '경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요.', + BID_REJECTED_BECAUSE_BID_ROUND_ALREADY_ADVANCED: + '이미 다음 선수로 넘어갔어요.', BID_REJECTED_BECAUSE_BID_ROUND_DEADLINE_PASSED: '마감되어 반영되지 않았어요.', BID_REJECTED_BECAUSE_AUCTION_NOT_IN_BIDDING_PHASE: '지금은 입찰할 수 없는 단계예요.', @@ -100,6 +103,8 @@ export function resolveAuctionErrorMessage(message?: string): string { } /** 최신 상태로 rebaseline 후 재시도가 권장되는 거부인지. */ -export function isAuctionResyncRecommended(ack: AuctionRejectionAckLike): boolean { +export function isAuctionResyncRecommended( + ack: AuctionRejectionAckLike, +): boolean { return ack.resyncRecommended === true; } diff --git a/apps/web/src/api/types.ts b/apps/web/src/api/types.ts index 96a337b1..416e9331 100644 --- a/apps/web/src/api/types.ts +++ b/apps/web/src/api/types.ts @@ -197,6 +197,8 @@ export interface ApiAuction { currentAuctionPlayerOrderIndex: number; bids?: ApiAuctionBid[]; auctionBidHistory?: Array<{ + bidRoundId?: string; + playerUserId?: string; bidderTeamIndex: number; bidPointAmount: number; bidderUserId: string; @@ -213,6 +215,7 @@ export interface ApiAuction { currentDeadline?: string; currentBidRoundDeadlineTimestampMS?: number; currentBidRoundPlayerUserId?: string; + currentBidRoundId?: string; currentServerTimestampMS?: number; initialPoints?: number; initialTeamBidPointAmount: number; @@ -224,6 +227,7 @@ export interface ApiAuction { auctionLifecycleId?: string; matchSideEffectRevision?: number; matchSideEffectStatus?: 'none' | 'pending' | 'processing' | 'completed' | 'failed'; + matchSideEffectTransitionId?: string; canPlaceBidInCurrentBidRound?: boolean; isAuctionStateTransitionInProgress?: boolean; shouldForceAuctionStateResync?: boolean; diff --git a/apps/web/src/api/useAuctionSocket.ts b/apps/web/src/api/useAuctionSocket.ts index 2fa09c00..ceb54a3d 100644 --- a/apps/web/src/api/useAuctionSocket.ts +++ b/apps/web/src/api/useAuctionSocket.ts @@ -3,6 +3,18 @@ import { io, type Socket } from 'socket.io-client'; import { API_BASE_URL, useAuthUserId } from '../config'; import { acquireAccessToken } from '../lib/auth/token-provider'; import type { ApiAuction, ApiAuctionChatMessage, Position } from './types'; +import { + createAuctionResyncRequester, + type AuctionResyncReply, +} from './auction-resync'; +import { + applyBidDelta, + shouldApplyIncomingAuctionState, + shouldResyncForBidDelta, + shouldApplyPolledAuctionState, + mergeAuctionState, + type BidDeltaPayload, +} from './auction-state'; import { resolveAuctionActionRejection } from './auction-action-retry'; import { isAuctionResyncRecommended, @@ -21,6 +33,7 @@ type ConnectionStatus = | 'error'; interface BidPayload { + clientObservedBidRoundId?: string; bidderTeamIndex: number; bidPointAmount: number; bidRequestId?: string; @@ -52,33 +65,19 @@ interface ChatHistoryPayload { interface ResetPayload { matchId?: string; + auctionLifecycleId?: string; } type MatchSideEffectStatus = NonNullable; interface MatchSideEffectPayload { auctionLifecycleId?: string; + auctionStateTransitionId?: string; currentServerTimestampMS?: number; matchSideEffectRevision?: number; matchSideEffectStatus?: MatchSideEffectStatus; } -interface BidDeltaPayload { - auctionBidHistoryEntry?: { - bidderTeamIndex?: number; - bidPointAmount?: number; - bidderUserId?: string; - bidPlacedAtTimestampMS?: number; - }; - auctionLifecycleId?: string; - bidRequestId?: string; - currentBidRoundDeadlineTimestampMS?: number; - currentHighestBidderTeamIndex?: number; - currentHighestBidPointAmount?: number; - currentServerTimestampMS?: number; - revision?: number; -} - interface BidServerTimingPayload { fallbackFullStatePath?: boolean; gatewayTotalDurationMS?: number; @@ -145,6 +144,9 @@ export function useAuctionSocket(matchId: string | undefined) { ? JSON.stringify([matchId, authUserId ?? null]) : null; const socketRef = useRef(null); + const resyncRequesterRef = useRef | null>(null); const pendingBidTimeoutsRef = useRef< Record> >({}); @@ -160,6 +162,7 @@ export function useAuctionSocket(matchId: string | undefined) { const timeSyncSequenceRef = useRef(0); const hasSyncedServerTimeRef = useRef(false); const latestAuctionRef = useRef(null); + const cancelledLifecycleIdsRef = useRef(new Set()); const latestConfirmedBidPointAmountRef = useRef(0); const [auction, setAuction] = useState(null); const [chatMessages, setChatMessages] = useState([]); @@ -187,6 +190,11 @@ export function useAuctionSocket(matchId: string | undefined) { const commitAuctionState = useCallback( (nextAuction: ApiAuction, source = 'socket-state'): boolean => { + if ( + nextAuction.auctionLifecycleId && + cancelledLifecycleIdsRef.current.has(nextAuction.auctionLifecycleId) + ) + return false; const currentAuction = latestAuctionRef.current; const applied = shouldApplyIncomingAuctionState( currentAuction, @@ -198,13 +206,12 @@ export function useAuctionSocket(matchId: string | undefined) { : -1; const incomingRevision = typeof nextAuction?.revision === 'number' ? nextAuction.revision : -1; - // 단조성을 깨고 적용되는 경우(=revision이 같거나 뒤로 가는데도 반영됨). - // force-resync/lifecycle 교체로만 발생하며, "2~3 버전 롤백"의 직접 증거가 된다. + // 같은 revision의 전체 상태는 delta 누락 복구이므로 역행으로 기록하지 않는다. const isBackwardApply = applied && currentRevision >= 0 && incomingRevision >= 0 && - incomingRevision <= currentRevision; + incomingRevision < currentRevision; addNewRelicPageAction('AuctionClientStateApply', { matchId: matchId ?? 'none', source, @@ -219,6 +226,7 @@ export function useAuctionSocket(matchId: string | undefined) { return false; } + nextAuction = mergeAuctionState(currentAuction, nextAuction); latestAuctionRef.current = nextAuction; latestConfirmedBidPointAmountRef.current = resolveCurrentBidPointAmount(nextAuction); @@ -230,6 +238,13 @@ export function useAuctionSocket(matchId: string | undefined) { const commitBidDelta = useCallback((payload: BidDeltaPayload): boolean => { const currentAuction = latestAuctionRef.current; + if (shouldResyncForBidDelta(currentAuction, payload)) { + void resyncRequesterRef.current?.request( + payload.revision, + payload.auctionLifecycleId, + ); + return false; + } const nextAuction = applyBidDelta(currentAuction, payload); if (!nextAuction || nextAuction === currentAuction) { return false; @@ -242,6 +257,29 @@ export function useAuctionSocket(matchId: string | undefined) { return true; }, []); + // HTTP 폴링은 화면만 갱신하므로, 소켓이 전이를 놓친 경우 입찰 기준가와 + // 라운드 ID가 낡은 채 남는다. 같은 경매의 더 새 상태면 소켓 상태에도 반영한다. + const syncPolledAuctionState = useCallback( + (polledAuction: ApiAuction | null | undefined) => { + if (!polledAuction) return; + if ( + polledAuction.auctionLifecycleId && + cancelledLifecycleIdsRef.current.has(polledAuction.auctionLifecycleId) + ) + return; + if ( + !shouldApplyPolledAuctionState( + latestAuctionRef.current, + polledAuction, + matchId, + ) + ) + return; + commitAuctionState(polledAuction, 'http-poll'); + }, + [commitAuctionState, matchId], + ); + const setError = useCallback( (message: string | null) => { setErrorState({ key: connectionKey ?? '', message }); @@ -336,7 +374,10 @@ export function useAuctionSocket(matchId: string | undefined) { : undefined); const clientObservedRemainingBidRoundTimeMS = typeof currentDeadlineTimestampMS === 'number' - ? Math.max(0, currentDeadlineTimestampMS - (Date.now() + serverTimeOffsetMs)) + ? Math.max( + 0, + currentDeadlineTimestampMS - (Date.now() + serverTimeOffsetMs), + ) : undefined; const serverDeadlineTimestampMS = ack.bidDiagnostics?.currentBidRoundDeadlineTimestampMS ?? @@ -399,6 +440,7 @@ export function useAuctionSocket(matchId: string | undefined) { } const activeConnectionKey = connectionKey; + const cancelledLifecycleIds = cancelledLifecycleIdsRef.current; let disposed = false; void (async () => { @@ -418,6 +460,33 @@ export function useAuctionSocket(matchId: string | undefined) { }); socketRef.current = socket; + const receiveAuctionState = (nextAuction: ApiAuction, source: string) => { + const previousRevision = latestAuctionRef.current?.revision ?? -1; + const didApplyState = commitAuctionState(nextAuction, source); + if (didApplyState && (nextAuction.revision ?? 0) > previousRevision) { + clearPendingActionRequests(); + } + applyFallbackServerTimestamp(nextAuction.currentServerTimestampMS); + if (nextAuction.chatMessages) { + setChatMessages(mergeChatMessages([], nextAuction.chatMessages)); + } + if (didApplyState) { + setError(null); + } + }; + const resyncRequester = createAuctionResyncRequester( + async () => { + if (!socket.connected) + throw new Error('Auction socket is disconnected.'); + return (await socket + .timeout(3000) + .emitWithAck('resync', {})) as AuctionResyncReply; + }, + (state) => { + receiveAuctionState(state, 'socket-resync'); + }, + ); + resyncRequesterRef.current = resyncRequester; socket.on('connect', () => { setConnectionState({ @@ -437,6 +506,7 @@ export function useAuctionSocket(matchId: string | undefined) { }); socket.on('disconnect', () => { + resyncRequester.cancel(); clearTimeSyncTimers(); setConnectionState({ key: activeConnectionKey, @@ -444,18 +514,12 @@ export function useAuctionSocket(matchId: string | undefined) { }); }); + socket.on('state-pending', () => { + void resyncRequester.request(); + }); + socket.on('state', (nextAuction: ApiAuction) => { - const didApplyState = commitAuctionState(nextAuction, 'socket-state'); - if (didApplyState) { - clearPendingActionRequests(); - } - applyFallbackServerTimestamp(nextAuction.currentServerTimestampMS); - if (nextAuction.chatMessages) { - setChatMessages(mergeChatMessages([], nextAuction.chatMessages)); - } - if (didApplyState) { - setError(null); - } + receiveAuctionState(nextAuction, 'socket-state'); }); socket.on('chat-history', (payload: ChatHistoryPayload) => { @@ -509,6 +573,13 @@ export function useAuctionSocket(matchId: string | undefined) { const currentAuction = latestAuctionRef.current; if (!currentAuction) return; + if ( + payload.auctionStateTransitionId && + currentAuction.matchSideEffectTransitionId && + payload.auctionStateTransitionId !== + currentAuction.matchSideEffectTransitionId + ) + return; if ( payload.auctionLifecycleId && currentAuction.auctionLifecycleId && @@ -540,6 +611,18 @@ export function useAuctionSocket(matchId: string | undefined) { socket.on('reset', (payload: ResetPayload) => { if (payload.matchId && payload.matchId !== matchId) return; + const lifecycle = + payload.auctionLifecycleId ?? + latestAuctionRef.current?.auctionLifecycleId; + if (lifecycle) cancelledLifecycleIdsRef.current.add(lifecycle); + if ( + payload.auctionLifecycleId && + latestAuctionRef.current?.auctionLifecycleId && + payload.auctionLifecycleId !== + latestAuctionRef.current.auctionLifecycleId + ) + return; + resyncRequester.cancel(); latestAuctionRef.current = null; latestConfirmedBidPointAmountRef.current = 0; setAuction(null); @@ -556,6 +639,8 @@ export function useAuctionSocket(matchId: string | undefined) { return () => { disposed = true; + resyncRequesterRef.current?.cancel(); + resyncRequesterRef.current = null; const socket = socketRef.current; socket?.removeAllListeners(); socket?.disconnect(); @@ -570,6 +655,7 @@ export function useAuctionSocket(matchId: string | undefined) { pendingBidTimeoutsRef.current = {}; pendingActionTimeoutsRef.current = {}; latestAuctionRef.current = null; + cancelledLifecycleIds.clear(); latestConfirmedBidPointAmountRef.current = 0; hasSyncedServerTimeRef.current = false; setPendingBidRequests({}); @@ -621,12 +707,11 @@ export function useAuctionSocket(matchId: string | undefined) { } // 거부 등으로 클라 상태가 낡았을 때 서버 최신 state를 다시 받아 rebaseline한다. - // join 핸들러가 getStateByMatchId → broadcastState로 fresh state를 내려주고, - // revision 가드상 더 높은 revision이라 정상 적용된다(force-resync 아님). + // 요청한 소켓만 최신 상태를 받으며 진행 중인 동기화 요청은 합친다. function resyncAuctionState() { const socket = socketRef.current; if (!socket || !socket.connected || !matchId) return; - socket.emit('join', { matchId }); + void resyncRequesterRef.current?.request(); } // retryAttempt: 거부 후 자동 재시도 횟수(루프 방지를 위해 1회로 제한). @@ -655,10 +740,13 @@ export function useAuctionSocket(matchId: string | undefined) { ...current, [auctionActionRequestId]: { action: eventName }, })); - pendingActionTimeoutsRef.current[auctionActionRequestId] = setTimeout(() => { - clearPendingActionRequest(auctionActionRequestId); - setError('경매 진행 응답이 지연되어 최신 상태를 다시 확인해주세요.'); - }, 10_000); + pendingActionTimeoutsRef.current[auctionActionRequestId] = setTimeout( + () => { + clearPendingActionRequest(auctionActionRequestId); + setError('경매 진행 응답이 지연되어 최신 상태를 다시 확인해주세요.'); + }, + 10_000, + ); addNewRelicPageAction('AuctionClientActionSent', { matchId: matchId ?? 'none', action: eventName, @@ -731,7 +819,10 @@ export function useAuctionSocket(matchId: string | undefined) { sentPlayerUserId: payload.clientObservedCurrentBidRoundPlayerUserId ?? 'none', sentPhase: payload.clientObservedPhase ?? 'none', - ...describeAuctionForTelemetry('observed', latestAuctionRef.current), + ...describeAuctionForTelemetry( + 'observed', + latestAuctionRef.current, + ), ...describeAuctionForTelemetry('server', ack?.state ?? null), }); if (canAutoRetry) { @@ -772,7 +863,9 @@ export function useAuctionSocket(matchId: string | undefined) { const currentAuction = auction; const visibleAuction = - currentAuction && currentAuction.matchId === matchId ? currentAuction : null; + currentAuction && currentAuction.matchId === matchId + ? currentAuction + : null; const visibleConnectionStatus: ConnectionStatus = connectionKey === null ? 'idle' @@ -797,6 +890,7 @@ export function useAuctionSocket(matchId: string | undefined) { bidLatencyDiagnostics, serverRoundTripMs: matchId ? serverRoundTripMs : null, serverTimeOffsetMs: matchId ? serverTimeOffsetMs : 0, + syncPolledAuctionState, placeBid(payload: BidPayload) { const socket = socketRef.current; if (!socket || !socket.connected) { @@ -807,6 +901,12 @@ export function useAuctionSocket(matchId: string | undefined) { return; } if (payload.bidPointAmount <= latestConfirmedBidPointAmountRef.current) { + // 조용히 무시하면 화면과 내부 기준가가 어긋났을 때 입찰 버튼이 반응하지 + // 않는 것처럼 보인다. 안내하고 최신 상태를 다시 받는다. + setError( + '현재 최고 입찰가보다 높은 금액만 입찰할 수 있어요. 최신 경매 상태를 다시 불러올게요.', + ); + resyncAuctionState(); return; } @@ -815,6 +915,8 @@ export function useAuctionSocket(matchId: string | undefined) { const bidPayload: BidPayload = { ...payload, bidRequestId, + clientObservedBidRoundId: + payload.clientObservedBidRoundId ?? currentAuction?.currentBidRoundId, clientObservedAuctionLifecycleId: payload.clientObservedAuctionLifecycleId ?? currentAuction?.auctionLifecycleId, @@ -835,36 +937,32 @@ export function useAuctionSocket(matchId: string | undefined) { clearPendingBidRequest(bidRequestId); setError('입찰 응답이 지연되어 최신 상태를 다시 확인해주세요.'); }, 10_000); - socket.emit( - 'bid', - bidPayload, - (ack?: BidAckPayload) => { - captureBidLatencyDiagnostics(ack); - if (ack?.rejected) { - clearPendingBidRequest(bidRequestId); - // 입찰 거부도 최신 state로 rebaseline(예: 0초 입찰 EXPIRED → 만료된 - // 라운드 화면에서 벗어나게). 단 입찰은 자동 재시도하지 않는다(의도치 - // 않은 금액 재입찰 방지). - if (ack.state) { - commitAuctionState(ack.state); - } else if (isAuctionResyncRecommended(ack)) { - resyncAuctionState(); - } - setError(resolveAuctionRejectionMessage(ack)); - return; + socket.emit('bid', bidPayload, (ack?: BidAckPayload) => { + captureBidLatencyDiagnostics(ack); + if (ack?.rejected) { + clearPendingBidRequest(bidRequestId); + // 입찰 거부도 최신 state로 rebaseline(예: 0초 입찰 EXPIRED → 만료된 + // 라운드 화면에서 벗어나게). 단 입찰은 자동 재시도하지 않는다(의도치 + // 않은 금액 재입찰 방지). + if (ack.state) { + commitAuctionState(ack.state); + } else if (isAuctionResyncRecommended(ack)) { + resyncAuctionState(); } - if (ack?.bidDelta) { - clearPendingBidRequest(bidRequestId); - commitBidDelta(ack.bidDelta); - return; - } - const ackState = ack?.state; - if (ackState) { - clearPendingBidRequest(bidRequestId); - commitAuctionState(ackState); - } - }, - ); + setError(resolveAuctionRejectionMessage(ack)); + return; + } + if (ack?.bidDelta) { + clearPendingBidRequest(bidRequestId); + commitBidDelta(ack.bidDelta); + return; + } + const ackState = ack?.state; + if (ackState) { + clearPendingBidRequest(bidRequestId); + commitAuctionState(ackState); + } + }); }, openBidding() { emitAuctionAction('open-bidding', buildAuctionActionPayload()); @@ -905,7 +1003,10 @@ function resolveCurrentBidPointAmount(auction: ApiAuction): number { } // [AUCTION-DEBUG] 아래 텔레메트리 헬퍼들과 호출부는 조사 종료 후 제거 대상. -type AuctionTelemetryAttributes = Record; +type AuctionTelemetryAttributes = Record< + string, + string | number | boolean | null +>; function readAuctionHistorySequence(auction: ApiAuction | null | undefined) { const value = (auction as { currentAuctionHistorySequence?: unknown } | null) @@ -914,7 +1015,9 @@ function readAuctionHistorySequence(auction: ApiAuction | null | undefined) { } function summarizeAuctionTeams(auction: ApiAuction | null | undefined) { - const teams = Array.isArray(auction?.auctionTeams) ? auction.auctionTeams : []; + const teams = Array.isArray(auction?.auctionTeams) + ? auction.auctionTeams + : []; let assignedMemberCount = 0; for (const team of teams) { const members = (team as { members?: unknown[] }).members; @@ -956,143 +1059,3 @@ function describeAuctionForTelemetry( [`${prefix}ForceResync`]: Boolean(auction?.shouldForceAuctionStateResync), }; } - -function shouldApplyIncomingAuctionState( - current: ApiAuction | null, - incoming: ApiAuction, -): boolean { - if (!current) return true; - if (incoming.shouldForceAuctionStateResync) return true; - if ( - current.auctionLifecycleId && - incoming.auctionLifecycleId && - current.auctionLifecycleId !== incoming.auctionLifecycleId - ) { - return true; - } - if ( - typeof current.revision === 'number' && - typeof incoming.revision === 'number' && - incoming.revision <= current.revision - ) { - return false; - } - return true; -} - -function applyBidDelta( - current: ApiAuction | null, - payload: BidDeltaPayload, -): ApiAuction | null { - if (!current) return current; - if ( - payload.auctionLifecycleId && - current.auctionLifecycleId && - payload.auctionLifecycleId !== current.auctionLifecycleId - ) { - return current; - } - if ( - typeof current.revision === 'number' && - typeof payload.revision === 'number' && - payload.revision <= current.revision - ) { - return current; - } - - const bidPointAmount = Number(payload.currentHighestBidPointAmount); - const bidderTeamIndex = Number(payload.currentHighestBidderTeamIndex); - if (!Number.isFinite(bidPointAmount) || !Number.isFinite(bidderTeamIndex)) { - return current; - } - - const bidHistoryEntry = normalizeBidHistoryEntry(payload.auctionBidHistoryEntry); - const currentBidRoundDeadlineTimestampMS = - typeof payload.currentBidRoundDeadlineTimestampMS === 'number' - ? payload.currentBidRoundDeadlineTimestampMS - : current.currentBidRoundDeadlineTimestampMS; - - return { - ...current, - auctionBidHistory: appendUniqueAuctionBidHistoryEntry( - current.auctionBidHistory ?? [], - bidHistoryEntry, - ), - bids: bidHistoryEntry - ? appendUniqueLegacyBidEntry(current.bids ?? [], bidHistoryEntry) - : current.bids, - currentBid: bidPointAmount, - currentBidderTeamIndex: bidderTeamIndex, - currentDeadline: currentBidRoundDeadlineTimestampMS - ? new Date(currentBidRoundDeadlineTimestampMS).toISOString() - : current.currentDeadline, - currentHighestBidPointAmount: bidPointAmount, - currentHighestBidderTeamIndex: bidderTeamIndex, - currentBidRoundDeadlineTimestampMS, - currentServerTimestampMS: - payload.currentServerTimestampMS ?? current.currentServerTimestampMS, - revision: payload.revision ?? current.revision, - }; -} - -function normalizeBidHistoryEntry( - entry: BidDeltaPayload['auctionBidHistoryEntry'], -): NonNullable[number] | null { - if (!entry) return null; - const bidderTeamIndex = Number(entry.bidderTeamIndex); - const bidPointAmount = Number(entry.bidPointAmount); - const bidPlacedAtTimestampMS = Number(entry.bidPlacedAtTimestampMS); - if (!Number.isFinite(bidderTeamIndex) || !Number.isFinite(bidPointAmount)) { - return null; - } - return { - bidderTeamIndex, - bidPointAmount, - bidderUserId: entry.bidderUserId ?? '', - bidPlacedAtTimestampMS: Number.isFinite(bidPlacedAtTimestampMS) - ? bidPlacedAtTimestampMS - : Date.now(), - }; -} - -function appendUniqueAuctionBidHistoryEntry( - current: NonNullable, - entry: NonNullable[number] | null, -) { - if (!entry) return current; - if ( - current.some( - (existing) => - existing.bidderTeamIndex === entry.bidderTeamIndex && - existing.bidPointAmount === entry.bidPointAmount && - existing.bidPlacedAtTimestampMS === entry.bidPlacedAtTimestampMS, - ) - ) { - return current; - } - return [...current, entry]; -} - -function appendUniqueLegacyBidEntry( - current: ApiAuction['bids'], - entry: NonNullable[number], -) { - const currentBids = current ?? []; - const legacyBid = { - amount: entry.bidPointAmount, - bidderUserId: entry.bidderUserId, - teamIndex: entry.bidderTeamIndex, - timestamp: new Date(entry.bidPlacedAtTimestampMS).toISOString(), - }; - if ( - currentBids.some( - (existing) => - existing.teamIndex === legacyBid.teamIndex && - existing.amount === legacyBid.amount && - existing.timestamp === legacyBid.timestamp, - ) - ) { - return currentBids; - } - return [...currentBids, legacyBid]; -} diff --git a/apps/web/src/pages/AuctionPage.tsx b/apps/web/src/pages/AuctionPage.tsx index 6b4c782a..4c1cc079 100644 --- a/apps/web/src/pages/AuctionPage.tsx +++ b/apps/web/src/pages/AuctionPage.tsx @@ -40,6 +40,7 @@ import { useDevCaptainTeamIndex, } from "../config"; import { ApiError } from "../lib/api"; +import { mergeAuctionState } from "../api/auction-state"; import { championDisplayName, championIconUrl, @@ -152,7 +153,10 @@ function pickLatestAuction( const socketRevision = getAuctionRevision(socketAuction); const queryRevision = getAuctionRevision(queryAuction); if (socketRevision !== null && queryRevision !== null) { - return queryRevision > socketRevision ? queryAuction : socketAuction; + // Polling can repair a partial socket state at the same revision. + return queryRevision >= socketRevision + ? mergeAuctionState(socketAuction, queryAuction) + : mergeAuctionState(queryAuction, socketAuction); } } return getAuctionUpdatedAt(queryAuction) > getAuctionUpdatedAt(socketAuction) @@ -1901,6 +1905,10 @@ export default function AuctionPage() { cancelConfirmInputRef.current?.focus(); }, [showCancelModal]); + const { syncPolledAuctionState } = socket; + useEffect(() => { + syncPolledAuctionState(auctionQuery.data); + }, [auctionQuery.data, syncPolledAuctionState]); const liveAuction = pickLatestAuction(socket.auction, auctionQuery.data); const auctionState = useMemo( () => @@ -2517,7 +2525,7 @@ export default function AuctionPage() {

정말 경매를 취소하고 모집 상태로 되돌리시겠습니까? 경매 - 데이터, 입찰 기록, 팀 배정 정보가 삭제됩니다. + 진행 데이터와 팀 배정 정보가 초기화됩니다.

diff --git a/apps/web/test/auction-rejection-messages.test.ts b/apps/web/test/auction-rejection-messages.test.ts index 5c2acc23..aaabe702 100644 --- a/apps/web/test/auction-rejection-messages.test.ts +++ b/apps/web/test/auction-rejection-messages.test.ts @@ -7,6 +7,26 @@ import { const rosterMessage = "팀 배정 상태를 확인해야 해 경매를 완료하지 못했습니다."; +test("asks an outdated client to reload instead of claiming the player advanced", () => { + const message = + "경매 화면이 업데이트되었습니다. 새로고침한 뒤 다시 입찰해 주세요."; + assert.equal( + resolveAuctionRejectionMessage({ + auctionRejectionReason: "BID_REJECTED_BECAUSE_CLIENT_UPDATE_REQUIRED", + message, + }), + message, + ); + // An older bundle without the new reason mapping still displays the server message. + assert.equal( + resolveAuctionRejectionMessage({ + auctionRejectionReason: "UNRECOGNIZED_NEW_CLIENT_REASON", + message, + }), + message, + ); +}); + test("shows the common error message when roster validation blocks completion", () => { assert.equal( resolveAuctionRejectionMessage({ diff --git a/apps/web/test/auction-resync.test.ts b/apps/web/test/auction-resync.test.ts new file mode 100644 index 00000000..1e9dc72c --- /dev/null +++ b/apps/web/test/auction-resync.test.ts @@ -0,0 +1,95 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + createAuctionResyncRequester, + type AuctionResyncReply, +} from '../src/api/auction-resync.ts'; +import type { ApiAuction } from '../src/api/types.ts'; + +const state = (revision: number) => + ({ revision, auctionLifecycleId: 'life', phase: 'bidding' }) as ApiAuction; +const noWait = async () => {}; + +test('coalesces simultaneous gaps into one snapshot request', async () => { + let finish!: (reply: AuctionResyncReply) => void; + let calls = 0; + const received: number[] = []; + const sync = createAuctionResyncRequester( + () => { + calls++; + return new Promise((resolve) => { + finish = resolve; + }); + }, + (value) => received.push(value.revision!), + noWait, + ); + const requests = Array.from({ length: 100 }, () => sync.request(12, 'life')); + assert.equal(calls, 1); + finish({ ok: true, state: state(12) }); + await Promise.all(requests); + assert.deepEqual(received, [12]); +}); + +test('fetches again when a newer gap arrived while the snapshot was in flight', async () => { + let finish!: (reply: AuctionResyncReply) => void; + let calls = 0; + const received: number[] = []; + const sync = createAuctionResyncRequester( + () => + ++calls === 1 + ? new Promise((resolve) => { + finish = resolve; + }) + : Promise.resolve({ ok: true, state: state(14) }), + (value) => received.push(value.revision!), + noWait, + ); + const pending = sync.request(12, 'life'); + void sync.request(14, 'life'); + void sync.request(); // A generic refresh cannot lower the required revision. + finish({ ok: true, state: state(12) }); + await pending; + assert.deepEqual(received, [12, 14]); + assert.equal(calls, 2); +}); + +test('bounds retries and honors retry delay without rejoining the room', async () => { + let calls = 0; + const waits: number[] = []; + const sync = createAuctionResyncRequester( + async () => { + calls++; + return { ok: false, retryAfterMS: 1500 }; + }, + () => assert.fail('must not apply'), + async (ms) => { + waits.push(ms); + }, + ); + await sync.request(); + assert.equal(calls, 3); + assert.deepEqual(waits, [1500, 1500]); +}); + +test('drops an old connection reply after cancellation and permits a new request', async () => { + let finish!: (reply: AuctionResyncReply) => void; + let calls = 0; + const received: number[] = []; + const sync = createAuctionResyncRequester( + () => + ++calls === 1 + ? new Promise((resolve) => { + finish = resolve; + }) + : Promise.resolve({ ok: true, state: state(20) }), + (value) => received.push(value.revision!), + noWait, + ); + const stale = sync.request(); + sync.cancel(); + await sync.request(); + finish({ ok: true, state: state(10) }); + await stale; + assert.deepEqual(received, [20]); +}); diff --git a/apps/web/test/auction-state.test.ts b/apps/web/test/auction-state.test.ts new file mode 100644 index 00000000..039e52dd --- /dev/null +++ b/apps/web/test/auction-state.test.ts @@ -0,0 +1,188 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + applyBidDelta, + mergeAuctionState, + shouldApplyIncomingAuctionState, + shouldApplyPolledAuctionState, + shouldResyncForBidDelta, +} from "../src/api/auction-state.ts"; +import type { ApiAuction } from "../src/api/types.ts"; + +const current = { + auctionLifecycleId: "life", + revision: 10, + phase: "bidding", + currentBidRoundId: "round-A", + currentBidRoundPlayerUserId: "A", + currentHighestBidPointAmount: 10, + auctionBidHistory: [], + bids: [], +} as unknown as ApiAuction; +const bid = { + auctionLifecycleId: "life", + revision: 11, + currentBidRoundId: "round-A", + currentHighestBidPointAmount: 20, + currentHighestBidderTeamIndex: 1, + auctionBidHistoryEntry: { + bidPointAmount: 20, + bidderTeamIndex: 1, + bidderUserId: "captain", + bidPlacedAtTimestampMS: 1000, + playerUserId: "A", + bidRoundId: "round-A", + }, +}; + +test("a contiguous same-round bid updates price and preserves player identity", () => { + const next = applyBidDelta(current, bid)!; + assert.equal(shouldResyncForBidDelta(current, bid), false); + assert.equal(next.currentHighestBidPointAmount, 20); + assert.equal(next.currentBidRoundPlayerUserId, "A"); + assert.equal(next.auctionBidHistory?.[0].bidRoundId, "round-A"); +}); + +test("a revision gap resyncs instead of attaching a new player bid to the old structure", () => { + const missedTransition = { + ...bid, + revision: 13, + currentBidRoundId: "round-B", + }; + assert.equal(shouldResyncForBidDelta(current, missedTransition), true); + assert.equal(applyBidDelta(current, missedTransition), current); +}); + +test("a delta cannot open bidding or change the round by itself", () => { + assert.equal( + shouldResyncForBidDelta({ ...current, phase: "presenting" }, bid), + true, + ); + assert.equal( + shouldResyncForBidDelta(current, { ...bid, currentBidRoundId: "round-B" }), + true, + ); +}); + +test("a full snapshot can repair the same revision but cannot regress auction revision", () => { + assert.equal( + shouldApplyIncomingAuctionState(current, { + ...current, + currentBidRoundPlayerUserId: "B", + }), + true, + ); + assert.equal( + shouldApplyIncomingAuctionState(current, { ...current, revision: 9 }), + false, + ); +}); + +test("full state repair preserves a newer independently versioned side effect", () => { + const live = { + ...current, + matchSideEffectRevision: 3, + matchSideEffectStatus: "completed" as const, + }; + const incoming = { + ...current, + matchSideEffectRevision: 1, + matchSideEffectStatus: "pending" as const, + }; + assert.equal( + mergeAuctionState(live, incoming).matchSideEffectStatus, + "completed", + ); + assert.equal( + mergeAuctionState(live, { ...incoming, auctionLifecycleId: "new-life" }) + .matchSideEffectStatus, + "pending", + ); +}); + +test("replayed old bids do not resync or change the current state", () => { + const replay = { ...bid, revision: 9 }; + assert.equal(shouldResyncForBidDelta(current, replay), false); + assert.equal(applyBidDelta(current, replay), current); +}); + +test("polled state repairs a socket that missed a transition", () => { + const polled = { + ...current, + matchId: "match", + revision: 12, + currentBidRoundId: "round-B", + currentBidRoundPlayerUserId: "B", + currentHighestBidPointAmount: 0, + } as unknown as ApiAuction; + const socketState = { ...current, matchId: "match" } as ApiAuction; + assert.equal( + shouldApplyPolledAuctionState(socketState, polled, "match"), + true, + ); +}); + +test("polled state never regresses or replaces the socket lifecycle", () => { + const socketState = { ...current, matchId: "match" } as ApiAuction; + const same = { ...socketState } as ApiAuction; + const older = { ...socketState, revision: 9 } as ApiAuction; + const otherLifecycle = { + ...socketState, + auctionLifecycleId: "old-life", + revision: 99, + } as ApiAuction; + const otherMatch = { ...socketState, matchId: "other", revision: 11 }; + assert.equal(shouldApplyPolledAuctionState(socketState, same, "match"), true); + assert.equal( + shouldApplyPolledAuctionState(socketState, older, "match"), + false, + ); + assert.equal( + shouldApplyPolledAuctionState(socketState, otherLifecycle, "match"), + false, + ); + assert.equal( + shouldApplyPolledAuctionState(socketState, otherMatch, "match"), + false, + ); + assert.equal(shouldApplyPolledAuctionState(null, same, "match"), true); +}); + +test("polling initializes a missing socket baseline but rejects another match or missing identity", () => { + const incoming = { + ...current, + matchId: "match", + currentBidRoundId: "round-B", + currentHighestBidPointAmount: 0, + } as ApiAuction; + assert.equal(shouldApplyPolledAuctionState(null, incoming, "match"), true); + assert.equal( + shouldApplyPolledAuctionState(null, incoming, "another-match"), + false, + ); + assert.equal( + shouldApplyPolledAuctionState( + null, + { ...incoming, auctionLifecycleId: undefined }, + "match", + ), + false, + ); +}); + +test("a prior side-effect generation cannot replace the status of a new transition", () => { + const old = { + ...current, + matchSideEffectTransitionId: "old", + matchSideEffectRevision: 100, + matchSideEffectStatus: "failed" as const, + }; + const next = { + ...current, + revision: 12, + matchSideEffectTransitionId: "new", + matchSideEffectRevision: 3, + matchSideEffectStatus: "pending" as const, + }; + assert.equal(mergeAuctionState(old, next).matchSideEffectStatus, "pending"); +});