diff --git a/apps/api/test/auction-cluster/.gitattributes b/apps/api/test/auction-cluster/.gitattributes new file mode 100644 index 00000000..fcadb2cf --- /dev/null +++ b/apps/api/test/auction-cluster/.gitattributes @@ -0,0 +1 @@ +* text eol=lf diff --git a/apps/api/test/auction-cluster/.gitignore b/apps/api/test/auction-cluster/.gitignore new file mode 100644 index 00000000..fbca2253 --- /dev/null +++ b/apps/api/test/auction-cluster/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/apps/api/test/auction-cluster/Dockerfile b/apps/api/test/auction-cluster/Dockerfile new file mode 100644 index 00000000..42c8cf21 --- /dev/null +++ b/apps/api/test/auction-cluster/Dockerfile @@ -0,0 +1,13 @@ +FROM node:26-bookworm-slim +RUN npm install --global pnpm@10.4.1 +WORKDIR /workspace +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY packages/contracts ./packages/contracts +COPY apps/api/package.json ./apps/api/package.json +COPY apps/web/package.json ./apps/web/package.json +RUN pnpm install --frozen-lockfile --filter api --filter web --ignore-scripts +COPY apps/api ./apps/api +COPY apps/web/src/api ./apps/web/src/api +WORKDIR /workspace/apps/api +ENV NODE_ENV=test NEW_RELIC_ENABLED=false +CMD ["node", "-r", "ts-node/register/transpile-only", "test/auction-cluster/node.ts"] diff --git a/apps/api/test/auction-cluster/compose.yaml b/apps/api/test/auction-cluster/compose.yaml new file mode 100644 index 00000000..a1697187 --- /dev/null +++ b/apps/api/test/auction-cluster/compose.yaml @@ -0,0 +1,61 @@ +x-app: &app + image: ${COMPOSE_PROJECT_NAME}-app + build: + context: ../../../.. + dockerfile: apps/api/test/auction-cluster/Dockerfile + environment: &environment + MONGO_TEST_URL: mongodb://mongo:27017/auction_cluster?replicaSet=auction_cluster + REDIS_TEST_URL: redis://redis-proxy:6379 + NODE_ENV: test + NEW_RELIC_ENABLED: 'false' + volumes: + - ./results:/results + networks: [cluster] + mem_limit: 1g + +services: + redis: + image: redis:8-alpine + command: [redis-server, --save, '', --appendonly, 'no'] + networks: [cluster] + mem_limit: 256m + mongo: + # MongoDB 8+ refuses the test host's Linux 7.0.0 kernel (SERVER-121912). + image: ${MONGO_TEST_IMAGE:-mongo:7.0} + command: + [ + mongod, + --replSet, + auction_cluster, + --bind_ip_all, + --wiredTigerCacheSizeGB, + '0.25', + ] + networks: [cluster] + mem_limit: 768m + redis-proxy: + <<: *app + command: [node, test/auction-cluster/fault-proxy.mjs] + mem_limit: 128m + api-a: + <<: *app + environment: + <<: *environment + CLUSTER_NODE: api-a + api-b: + <<: *app + environment: + <<: *environment + CLUSTER_NODE: api-b + lb: + image: nginx:stable-alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + networks: [cluster] + mem_limit: 128m + runner: + <<: *app + command: [node, test/auction-cluster/run.mjs, verify] + +networks: + cluster: {} diff --git a/apps/api/test/auction-cluster/fault-proxy.mjs b/apps/api/test/auction-cluster/fault-proxy.mjs new file mode 100644 index 00000000..0f3ab9fa --- /dev/null +++ b/apps/api/test/auction-cluster/fault-proxy.mjs @@ -0,0 +1,60 @@ +import { createServer as createTcpServer, connect } from 'node:net'; +import { createServer as createHttpServer } from 'node:http'; + +// Real TCP connections; only connections issuing SUBSCRIBE/PSUBSCRIBE are cut. +// Redis commands, TIME and PUBLISH continue through the same proxy. +let blocked = false; +let dropped = 0; +const connections = new Set(); +createTcpServer((socket) => { + const upstream = connect(6379, 'redis'); + const connection = { socket, upstream, subscriber: false, prefix: '' }; + connections.add(connection); + const close = () => { + socket.destroy(); + upstream.destroy(); + connections.delete(connection); + }; + socket.on('error', close); + upstream.on('error', close); + socket.on('close', close); + upstream.on('close', close); + socket.on('data', (chunk) => { + connection.prefix = (connection.prefix + chunk.toString('utf8')).slice( + -4096, + ); + if (/\r\n(?:p?subscribe|ssubscribe)\r\n/i.test(connection.prefix)) { + connection.subscriber = true; + } + if (blocked && connection.subscriber) { + dropped += 1; + close(); + } else upstream.write(chunk); + }); + upstream.on('data', (chunk) => socket.write(chunk)); +}).listen(6379, '0.0.0.0'); + +createHttpServer((request, response) => { + if (request.method === 'POST' && request.url === '/block') { + blocked = true; + for (const connection of connections) { + if (!connection.subscriber) continue; + dropped += 1; + connection.socket.destroy(); + connection.upstream.destroy(); + } + } else if (request.method === 'POST' && request.url === '/restore') { + blocked = false; + } else if (request.url !== '/status') { + response.writeHead(404).end(); + return; + } + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + blocked, + dropped, + subscribers: [...connections].filter((c) => c.subscriber).length, + }), + ); +}).listen(8081, '0.0.0.0'); diff --git a/apps/api/test/auction-cluster/nginx.conf b/apps/api/test/auction-cluster/nginx.conf new file mode 100644 index 00000000..78b63824 --- /dev/null +++ b/apps/api/test/auction-cluster/nginx.conf @@ -0,0 +1,18 @@ +events {} +http { + upstream auction { + server api-a:3000; + server api-b:3000; + } + server { + listen 8080; + location / { + proxy_pass http://auction; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 120s; + add_header X-Cluster-Upstream $upstream_addr always; + } + } +} diff --git a/apps/api/test/auction-cluster/node.ts b/apps/api/test/auction-cluster/node.ts new file mode 100644 index 00000000..6aa9addd --- /dev/null +++ b/apps/api/test/auction-cluster/node.ts @@ -0,0 +1,203 @@ +import 'reflect-metadata'; +import { readFile } from 'node:fs/promises'; +import { Body, Controller, Get, Module, Post } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; +import mongoose from 'mongoose'; +import { createClient } from 'redis'; +import type { Namespace } from 'socket.io'; +import { RedisIoAdapter } from '../../src/redis-io.adapter'; +import { AuctionController } from '../../src/modules/auction/auction.controller'; +import { AuctionGateway } from '../../src/modules/auction/auction.gateway'; +import { AuctionEventBusService } from '../../src/modules/auction/auction-event-bus.service'; +import { AuctionService } from '../../src/modules/auction/auction.service'; +import { AuctionBidArchiveService } from '../../src/modules/auction/auction-bid-archive.service'; +import { + AuctionDurabilityScheduler, + type AuctionDurabilityPayload, +} from '../../src/modules/auction/auction-durability.scheduler'; +import { AuctionRedisService } from '../../src/modules/auction/redis/auction-redis.service'; +import { AuctionRedisCommandService } from '../../src/modules/auction/redis/auction-redis-command.service'; +import { AuctionRedisStateReaderService } from '../../src/modules/auction/redis/auction-redis-state-reader.service'; +import { AuthService } from '../../src/modules/auth/auth.service'; +import { JwtService } from '../../src/modules/auth/jwt.service'; +import { IdentityService } from '../../src/modules/auth/identity.service'; +import { ConnectionService } from '../../src/modules/global/database/connection.service'; +import { + AuctionSchema, + AuctionHistoryEntrySchema, + AuctionSideEffectApplicationSchema, + MatchSchema, +} from '../../src/modules/global/database/schema'; + +async function bootstrap() { + if ( + !process.env.MONGO_TEST_URL?.startsWith( + 'mongodb://mongo:27017/auction_cluster?', + ) || + process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379' + ) { + throw new Error( + 'This test node only accepts the isolated Compose services.', + ); + } + const fixture = JSON.parse( + await readFile('/results/fixture.json', 'utf8'), + ) as { communityId: string }; + const connection = await mongoose + .createConnection(process.env.MONGO_TEST_URL, { autoIndex: false }) + .asPromise(); + const models = { + match: connection.model('match', MatchSchema), + auction: connection.model('auction', AuctionSchema), + auctionHistory: connection.model( + 'auctionHistory', + AuctionHistoryEntrySchema, + ), + auctionSideEffectApplication: connection.model( + 'auctionSideEffectApplication', + AuctionSideEffectApplicationSchema, + ), + }; + let releaseCommit: (() => void) | undefined; + let commitGate: Promise | undefined; + let commitsWaiting = 0; + const database = { + getModel: async (name: keyof typeof models) => { + if (!models[name]) throw new Error(`Unexpected test model: ${name}`); + return models[name]; + }, + createSession: async () => { + const session = await connection.startSession(); + const commit = session.commitTransaction.bind(session); + session.commitTransaction = async () => { + if (commitGate) { + commitsWaiting += 1; + await commitGate; + commitsWaiting -= 1; + } + await commit(); + }; + return session; + }, + }; + const client = createClient({ url: process.env.REDIS_TEST_URL }); + client.on('error', (error) => + console.error('command-client:', error.message), + ); + await client.connect(); + const redis = new AuctionRedisService(client); + const commands = new AuctionRedisCommandService(redis); + const reader = new AuctionRedisStateReaderService(redis); + const bus = new AuctionEventBusService(); + const service = new AuctionService( + database as never, + {} as never, + {} as never, + {} as never, + redis, + commands, + reader, + bus, + ); + // Explicit draining lets the test hold real Mongo commits and reorder durable + // payloads; no in-memory database or fake Redis adapter is involved. + const scheduler = new AuctionDurabilityScheduler( + database as never, + redis, + undefined, + undefined, + undefined, + bus, + reader, + service, + ); + + @Controller('_cluster') + class ControlController { + @Get('health') health() { + const namespace = Reflect.get(gateway, 'server') as Namespace; + return { + node: process.env.CLUSTER_NODE, + sockets: [...namespace.sockets.keys()], + pid: process.pid, + commitsWaiting, + }; + } + @Post('hold-commit') hold() { + if (commitGate) throw new Error('Commit already held'); + commitGate = new Promise((resolve) => { + releaseCommit = resolve; + }); + return { held: true }; + } + @Post('release-commit') release() { + releaseCommit?.(); + commitGate = undefined; + releaseCommit = undefined; + return { released: true }; + } + @Post('drain') drain() { + return scheduler.processDueDurabilityPayloads(100); + } + @Post('apply') async apply(@Body() payload: AuctionDurabilityPayload) { + await scheduler.applyDurabilityPayload(payload); + return { applied: true }; + } + } + @Module({ + controllers: [AuctionController, ControlController], + providers: [ + AuctionGateway, + { provide: AuctionService, useValue: service }, + { provide: ConnectionService, useValue: database }, + { provide: AuctionRedisService, useValue: redis }, + { provide: AuctionEventBusService, useValue: bus }, + { provide: AuctionBidArchiveService, useValue: {} }, + { + provide: AuthService, + useValue: { resolveDevUserById: async () => null }, + }, + { provide: JwtService, useValue: {} }, + { provide: IdentityService, useValue: {} }, + ], + }) + class TestModule {} + const app = await NestFactory.create(TestModule, { + logger: ['error', 'warn'], + }); + // Identity-provider authentication is outside this transport/durability test. + // The real controller's RolesGuard still checks this isolated fixture admin. + app.use( + ( + request: { user?: unknown }, + response: { setHeader(name: string, value: string): void }, + next: () => void, + ) => { + request.user = { + role: 'admin', + communityId: new mongoose.Types.ObjectId(fixture.communityId), + }; + response.setHeader( + 'X-Cluster-Node', + process.env.CLUSTER_NODE ?? 'unknown', + ); + next(); + }, + ); + const adapter = new RedisIoAdapter(app); + await adapter.connectToRedis(); + app.useWebSocketAdapter(adapter); + const gateway = app.get(AuctionGateway); + await app.listen(3000, '0.0.0.0'); + console.log(`cluster node ready: ${process.env.CLUSTER_NODE}`); + process.on('SIGTERM', () => { + void app + .close() + .then(() => connection.close()) + .then(() => process.exit(0)); + }); +} +void bootstrap().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/apps/api/test/auction-cluster/run.mjs b/apps/api/test/auction-cluster/run.mjs new file mode 100644 index 00000000..7a5613cc --- /dev/null +++ b/apps/api/test/auction-cluster/run.mjs @@ -0,0 +1,641 @@ +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { readFile, writeFile } from 'node:fs/promises'; +import { request } from 'node:http'; +import { setTimeout as delay } from 'node:timers/promises'; +import { randomUUID } from 'node:crypto'; +import mongoose from 'mongoose'; +import { createClient } from 'redis'; + +const require = createRequire(import.meta.url); +require('ts-node/register/transpile-only'); +const { + AuctionRedisService, +} = require('../../src/modules/auction/redis/auction-redis.service.ts'); +const { + AuctionRedisCommandService, +} = require('../../src/modules/auction/redis/auction-redis-command.service.ts'); +const { + AUCTION_DURABILITY_RETRY_QUEUE_KEY, +} = require('../../src/modules/auction/redis/auction-redis.keys.ts'); +const { + MatchSchema, + AuctionSchema, + AuctionHistoryEntrySchema, + AuctionSideEffectApplicationSchema, +} = require('../../src/modules/global/database/schema.ts'); +const webRequire = createRequire( + new URL('../../../web/package.json', import.meta.url), +); +const { io } = webRequire('socket.io-client'); +const { QueryClient, QueryObserver, environmentManager } = webRequire( + '@tanstack/react-query', +); +const { + createAuctionHeartbeatMonitor, + shouldApplyPolledAuctionState, + mergeAuctionState, +} = await import('../../../web/src/api/auction-state.ts'); +const { createAuctionResyncRequester } = + await import('../../../web/src/api/auction-resync.ts'); +const { AUCTION_SAFETY_POLL_INTERVAL_MS } = + await import('../../../web/src/api/auction-polling.ts'); + +if ( + process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379' || + !process.env.MONGO_TEST_URL?.startsWith( + 'mongodb://mongo:27017/auction_cluster?', + ) +) { + throw new Error('Only the isolated Compose services are allowed.'); +} +const redis = createClient({ url: 'redis://redis:6379' }); +redis.on('error', () => {}); +await redis.connect(); +const cleanup = [() => (redis.isOpen ? redis.quit() : undefined)]; +const report = { + startedAt: new Date().toISOString(), + checks: [], + requests: [], + limitation: + 'Real API processes, Nginx, Redis TCP faults and Mongo transactions; fixture identity replaces the external login provider. Client recovery uses production helpers and QueryObserver, without rendering the React page.', +}; + +async function until(check, description, timeout = 15_000) { + const deadline = Date.now() + timeout; + let lastError; + while (Date.now() < deadline) { + try { + const value = await check(); + if (value) return value; + } catch (error) { + lastError = error; + } + await delay(50); + } + throw new Error( + `Timed out: ${description}${lastError ? `: ${lastError.message}` : ''}`, + ); +} +async function http(url, body) { + const started = Date.now(); + return new Promise((resolve, reject) => { + const req = request( + url, + { + method: body === undefined ? 'GET' : 'POST', + agent: false, + headers: { 'content-type': 'application/json' }, + }, + (res) => { + let data = ''; + res.on('data', (chunk) => { + data += chunk; + }); + res.on('end', () => { + report.requests.push({ + url, + node: res.headers['x-cluster-node'], + status: res.statusCode, + durationMS: Date.now() - started, + }); + if (res.statusCode < 200 || res.statusCode >= 300) + return reject(new Error(`${url}: ${res.statusCode} ${data}`)); + try { + resolve({ + body: JSON.parse(data), + node: res.headers['x-cluster-node'], + }); + } catch (error) { + reject(error); + } + }); + }, + ); + req.setTimeout(10_000, () => + req.destroy(new Error(`HTTP timeout: ${url}`)), + ); + req.on('error', reject); + req.end(body === undefined ? undefined : JSON.stringify(body)); + }); +} +const pass = (name, details = {}) => { + report.checks.push({ name, passed: true, ...details }); + console.log(`PASS ${name} ${JSON.stringify(details)}`); +}; + +async function initialize() { + const admin = new mongoose.mongo.MongoClient( + 'mongodb://mongo:27017/?directConnection=true', + { serverSelectionTimeoutMS: 1000 }, + ); + await until( + async () => { + await admin.connect(); + return true; + }, + 'Mongo startup', + 60_000, + ); + await admin.db('admin').command({ + replSetInitiate: { + _id: 'auction_cluster', + members: [{ _id: 0, host: 'mongo:27017' }], + }, + }); + await until( + async () => + (await admin.db('admin').command({ hello: 1 })).isWritablePrimary, + 'Mongo primary', + 60_000, + ); + const build = await admin.db('admin').command({ buildInfo: 1 }); + await admin.close(); + const connection = await mongoose + .createConnection(process.env.MONGO_TEST_URL, { autoIndex: false }) + .asPromise(); + try { + const models = { + match: connection.model('match', MatchSchema), + auction: connection.model('auction', AuctionSchema), + history: connection.model('auctionHistory', AuctionHistoryEntrySchema), + sideEffects: connection.model( + 'auctionSideEffectApplication', + AuctionSideEffectApplicationSchema, + ), + }; + await Promise.all( + Object.values(models).map((model) => model.createCollection()), + ); + await models.history.createIndexes(); + const matchId = new mongoose.Types.ObjectId(), + communityId = new mongoose.Types.ObjectId(), + auctionId = new mongoose.Types.ObjectId(); + const lifecycle = `cluster:${matchId}`; + const playerA = new mongoose.Types.ObjectId(), + playerB = new mongoose.Types.ObjectId(), + captain = new mongoose.Types.ObjectId(); + const structure = { + auctionPlayerOrder: [ + { userId: playerA.toString() }, + { userId: playerB.toString() }, + ], + auctionTeams: [ + { + teamIndex: 1, + captainUserId: captain.toString(), + remainingBidPointAmount: 1000, + members: [], + }, + ], + currentAuctionPlayerOrderIndex: 0, + currentAuctionRound: 1, + passedAuctionPlayerQueue: [], + maximumBidRoundDurationSeconds: 600, + bidRoundExtensionSeconds: 5, + initialTeamBidPointAmount: 1000, + topTierAuctionPriorityThreshold: 'NONE', + manualTopTierAuctionPriorityUserIds: [], + }; + await models.match.collection.insertOne({ + _id: matchId, + communityId, + title: 'Isolated cluster test', + participants: [], + teams: [], + status: 'team-building', + scheduledAt: new Date(), + }); + await models.auction.collection.insertOne({ + _id: auctionId, + matchId, + auctionEngineVersion: 'redis-v1', + auctionLifecycleId: lifecycle, + phase: 'presenting', + revision: 0, + currentAuctionHistorySequence: 0, + chatMessages: [], + ...structure, + }); + const commands = new AuctionRedisCommandService( + new AuctionRedisService(redis), + ); + await commands.hydrateState(matchId.toString(), { + matchId: matchId.toString(), + auctionLifecycleId: lifecycle, + phase: 'presenting', + revision: 0, + auctionHistorySequence: 0, + maximumBidRoundDurationSeconds: 600, + structureStateJson: JSON.stringify(structure), + liveStatePatch: { + currentAuctionPlayerOrderIndex: 0, + currentBidRoundPlayerUserId: playerA.toString(), + currentBidRoundId: 'initial', + currentHighestBidPointAmount: 0, + currentHighestBidderTeamIndex: -1, + maximumBidRoundDurationSeconds: 600, + bidRoundExtensionSeconds: 5, + 'team:1:captainUserId': captain.toString(), + 'team:1:remainingBidPointAmount': 1000, + 'team:1:assignedAuctionTeamMemberCount': 1, + }, + }); + const fixture = { + matchId: matchId.toString(), + communityId: communityId.toString(), + auctionId: auctionId.toString(), + lifecycle, + mongoVersion: build.version, + redisVersion: (await redis.info('server')).match( + /redis_version:([^\r\n]+)/, + )?.[1], + }; + await writeFile('/results/fixture.json', JSON.stringify(fixture, null, 2)); + console.log( + `Initialized isolated MongoDB ${build.version} and Redis ${fixture.redisVersion}`, + ); + } finally { + await connection.close(); + } +} + +async function verify() { + const fixture = JSON.parse(await readFile('/results/fixture.json', 'utf8')); + report.versions = { + node: process.version, + mongo: fixture.mongoVersion, + redis: fixture.redisVersion, + }; + const route = `/communities/${fixture.communityId}/matches/${fixture.matchId}/auction`; + const url = `http://lb:8080${route}`; + await until( + async () => (await http('http://api-a:3000/_cluster/health')).body.node, + 'API A startup', + 60_000, + ); + await until( + async () => (await http('http://api-b:3000/_cluster/health')).body.node, + 'API B startup', + 60_000, + ); + const routedNodes = new Set(); + for (let i = 0; i < 4; i += 1) routedNodes.add((await http(url)).node); + assert.deepEqual([...routedNodes].sort(), ['api-a', 'api-b']); + pass('Nginx routes real HTTP requests to both API processes'); + + async function connectViewer() { + const socket = io('http://lb:8080/auction', { + transports: ['websocket'], + autoConnect: false, + reconnection: false, + }); + const viewer = { + socket, + state: null, + states: [], + histories: [], + errors: [], + }; + socket.on('state', (state) => { + viewer.state = state; + viewer.states.push(state); + }); + socket.on('error', (error) => viewer.errors.push(error)); + socket.on('connect_error', (error) => viewer.errors.push(error.message)); + socket.on('history-updated', (event) => { + void http(`${url}/history`) + .then(({ body, node }) => viewer.histories.push({ event, body, node })) + .catch((error) => viewer.errors.push(error.message)); + }); + cleanup.push(() => socket.disconnect()); + socket.connect(); + await until( + () => socket.connected, + `socket connect: ${viewer.errors.join(',')}`, + ); + const ack = await socket + .timeout(5000) + .emitWithAck('join', { matchId: fixture.matchId }); + assert.equal(ack.ok, true); + assert.equal(viewer.state.matchId, fixture.matchId); + assert.equal(viewer.state.auctionEngineVersion, 'redis-v1'); + const health = await Promise.all( + ['api-a', 'api-b'].map((node) => + http(`http://${node}:3000/_cluster/health`), + ), + ); + viewer.node = health.find((entry) => + entry.body.sockets.includes(socket.id), + ).body.node; + return viewer; + } + const one = await connectViewer(), + two = await connectViewer(); + assert.notEqual(one.node, two.node); + const a = one.node === 'api-a' ? one : two; + const b = one.node === 'api-b' ? one : two; + const initialRevision = a.state.revision; + const revisionFor = (sequence) => initialRevision + sequence; + assert.equal(b.state.revision, initialRevision); + pass('WebSocket viewers join different nodes through Nginx', { + nodes: [one.node, two.node], + }); + + const sequences = async () => + (await http(`${url}/history`)).body.map((entry) => entry.sequence); + const action = async (viewer, name) => { + const ack = await viewer.socket + .timeout(5000) + .emitWithAck(name, { auctionActionRequestId: randomUUID() }); + assert.equal(ack.ok, true, JSON.stringify(ack)); + }; + for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []); + await action(a, 'open-bidding'); + await until( + () => + a.state.revision === revisionFor(1) && + b.state.revision === revisionFor(1), + 'cross-node full state', + ); + assert.equal(a.histories.length + b.histories.length, 0); + await http('http://api-a:3000/_cluster/hold-commit', {}); + const pendingDrain = http('http://api-a:3000/_cluster/drain', {}); + await until( + async () => + (await http('http://api-a:3000/_cluster/health')).body.commitsWaiting === + 1, + 'real Mongo commit barrier', + ); + for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []); + assert.equal(a.histories.length + b.histories.length, 0); + await http('http://api-a:3000/_cluster/release-commit', {}); + assert.equal((await pendingDrain).body.applied, 1); + await until( + () => a.histories.length === 1 && b.histories.length === 1, + 'post-commit history event on both nodes', + ); + for (const viewer of [a, b]) + assert.deepEqual( + viewer.histories[0].body.map((entry) => entry.sequence), + [1], + ); + pass( + 'History notification crosses nodes only after a real Mongo transaction commits', + ); + + await action(a, 'passed'); + await action(b, 'next'); + await until( + () => + a.state.revision === revisionFor(3) && + b.state.revision === revisionFor(3), + 'state fanout in both directions', + ); + const keys = await redis.zRange(AUCTION_DURABILITY_RETRY_QUEUE_KEY, 0, -1); + const queued = await Promise.all( + keys.map(async (key) => ({ + key, + payload: JSON.parse(await redis.hGet(key, 'payloadJson')), + })), + ); + const third = queued.find( + ({ payload }) => payload.auctionHistorySequence === 3, + ); + const second = queued.find( + ({ payload }) => payload.auctionHistorySequence === 2, + ); + assert.ok(third && second); + await http('http://api-b:3000/_cluster/apply', third.payload); + assert.deepEqual(await sequences(), [3, 1]); + const previousEvents = b.histories.length; + await http('http://api-a:3000/_cluster/apply', second.payload); + await until( + () => + b.histories.length > previousEvents && + b.histories.at(-1).body.some((entry) => entry.sequence === 2), + 'late append history refresh', + ); + assert.deepEqual(await sequences(), [3, 2, 1]); + for (const item of queued) { + await redis.zRem(AUCTION_DURABILITY_RETRY_QUEUE_KEY, item.key); + await redis.del(item.key); + } + pass( + 'A delayed history append refreshes another node with the same maximum sequence', + { maximumSequence: 3 }, + ); + + const undo = await http(`${url}/undo`, { + sequence: 1, + auctionActionRequestId: randomUUID(), + }); + assert.equal(undo.body.auctionEngineVersion, 'redis-v1'); + assert.equal(undo.body.revision, revisionFor(4)); + assert.deepEqual(await sequences(), [3, 2, 1]); + await http('http://api-b:3000/_cluster/drain', {}); + await until( + async () => JSON.stringify(await sequences()) === '[4,1]', + 'durable Undo history', + ); + await until( + () => b.histories.at(-1)?.body[0]?.sequence === 4, + 'Undo history event', + ); + pass('Undo increases sequence and replaces history on every node', { + sequence: 4, + history: [4, 1], + }); + + await until( + async () => + (await http('http://redis-proxy:8081/status')).body.subscribers >= 2, + 'Redis subscriptions', + ); + await http('http://redis-proxy:8081/block', {}); + console.log( + 'FAULT Pub/Sub TCP connections blocked; Redis commands and HTTP stay available', + ); + await delay(500); + assert.equal( + (await http('http://api-a:3000/_cluster/health')).body.node, + 'api-a', + ); + assert.equal( + (await http('http://api-b:3000/_cluster/health')).body.node, + 'api-b', + ); + assert.equal(a.socket.connected && b.socket.connected, true); + const timestamp = await b.socket + .timeout(3000) + .emitWithAck('time-sync', { sequence: 1 }); + assert.equal(timestamp.ok, true); + pass( + 'Both API processes and socket clock sync survive Pub/Sub-only TCP disconnection', + ); + + environmentManager.setIsServer(() => false); + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: Infinity } }, + }); + cleanup.push(() => queryClient.clear()); + let stateByHttp = b.state, + historyByHttp, + statePolls = 0, + historyPolls = 0; + const pollingStarted = Date.now(); + const stateObserver = new QueryObserver(queryClient, { + queryKey: ['auction', fixture.matchId], + refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS, + queryFn: async () => { + statePolls += 1; + return (await http(url)).body; + }, + }); + const historyObserver = new QueryObserver(queryClient, { + queryKey: ['auction-history', fixture.matchId], + refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS, + queryFn: async () => { + historyPolls += 1; + return (await http(`${url}/history`)).body; + }, + }); + cleanup.push( + stateObserver.subscribe((result) => { + if ( + result.data && + shouldApplyPolledAuctionState(stateByHttp, result.data, fixture.matchId) + ) + stateByHttp = mergeAuctionState(stateByHttp, result.data); + }), + ); + cleanup.push( + historyObserver.subscribe((result) => { + if (result.data) historyByHttp = result.data; + }), + ); + await until( + () => + stateByHttp.revision === revisionFor(4) && + historyByHttp?.[0]?.sequence === 4, + 'baseline HTTP reads', + ); + const beforeHistoryEvents = b.histories.length; + await action(a, 'open-bidding'); + await http('http://api-a:3000/_cluster/drain', {}); + await delay(250); + assert.equal(b.state.revision, revisionFor(4)); + assert.equal(b.histories.length, beforeHistoryEvents); + assert.equal(a.state.revision, revisionFor(5)); + // Do not run heartbeat recovery yet: verify the independent 30-second safety + // route even if the client identity checker fails to notice a missed event. + console.log( + 'WAIT Real 30-second HTTP safety polling with Pub/Sub still blocked', + ); + await until( + () => + stateByHttp.revision === revisionFor(5) && + historyByHttp?.[0]?.sequence === 5, + '30-second HTTP safety fallback', + 35_000, + ); + assert.ok(Date.now() - pollingStarted >= 29_000); + pass( + 'Real 30-second HTTP polls repair missed state and history through Nginx', + { elapsedMS: Date.now() - pollingStarted, statePolls, historyPolls }, + ); + + const monitor = createAuctionHeartbeatMonitor(); + const heartbeat = async (sequence) => { + const ack = await b.socket + .timeout(3000) + .emitWithAck('time-sync', { sequence }); + assert.equal(ack.ok, true); + return monitor.inspect(b.state, ack.auctionState); + }; + assert.equal((await heartbeat(2)).needsResync, false); + await delay(10_000); + assert.equal((await heartbeat(3)).needsResync, true); + let resyncs = 0; + const requester = createAuctionResyncRequester( + async () => { + resyncs += 1; + return b.socket.timeout(3000).emitWithAck('resync', {}); + }, + (state) => { + b.state = state; + }, + ); + await requester.request(revisionFor(5), fixture.lifecycle); + assert.equal(b.state.revision, revisionFor(5)); + assert.equal(resyncs, 1); + pass( + 'Two real heartbeats detect a persistent gap and private socket resync works during Pub/Sub outage', + { resyncs }, + ); + + await http('http://redis-proxy:8081/restore', {}); + await until( + async () => + (await http('http://redis-proxy:8081/status')).body.subscribers >= 2, + 'Pub/Sub resubscription', + 20_000, + ); + await action(a, 'passed'); + await http('http://api-a:3000/_cluster/drain', {}); + await until( + () => + b.state.revision === revisionFor(6) && + b.histories.at(-1)?.body[0]?.sequence === 6, + 'events after Pub/Sub restoration', + ); + pass( + 'Cross-node state and history events recover after restoring the TCP path', + ); + const fault = (await http('http://redis-proxy:8081/status')).body; + report.fault = fault; + const previousId = b.socket.id; + b.socket.disconnect(); + b.socket.connect(); + await until( + () => b.socket.connected && b.socket.id !== previousId, + 'socket reconnect through Nginx', + ); + assert.equal( + ( + await b.socket + .timeout(5000) + .emitWithAck('join', { matchId: fixture.matchId }) + ).ok, + true, + ); + await until( + () => b.state.revision === revisionFor(6), + 'reconnect full state', + ); + pass('A reconnected viewer receives the latest snapshot through Nginx'); +} + +try { + if (process.argv[2] === 'init') await initialize(); + else { + await verify(); + report.passed = true; + } +} catch (error) { + report.passed = false; + report.failure = { message: error.message, stack: error.stack }; + console.error(error); + process.exitCode = 1; +} finally { + if (process.argv[2] !== 'init') { + await http('http://redis-proxy:8081/restore', {}).catch(() => {}); + report.finishedAt = new Date().toISOString(); + await writeFile('/results/report.json', JSON.stringify(report, null, 2)); + } + for (const close of cleanup.reverse()) { + try { + await close(); + } catch {} + } +} diff --git a/apps/api/test/auction-cluster/run.sh b/apps/api/test/auction-cluster/run.sh new file mode 100644 index 00000000..8b4a18cb --- /dev/null +++ b/apps/api/test/auction-cluster/run.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# From the repository root: sh apps/api/test/auction-cluster/run.sh +# Requires Docker Compose. No host ports or production credentials are used. +# Results and container logs remain in this directory's ignored results/ folder. +# Authentication uses a fixture admin; client recovery uses the production +# helpers and QueryObserver without rendering React. MongoDB 7 is the default +# for the test host's kernel; set MONGO_TEST_IMAGE on a compatible host to match +# the production database version. +set -eu +cd "$(dirname "$0")" +export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-lolympus-auction-cluster-$(date +%s)}" +case "$COMPOSE_PROJECT_NAME" in + lolympus-auction-cluster-*) ;; + *) echo 'Refusing to use a Compose project outside the auction-cluster test prefix.' >&2; exit 1 ;; +esac +mkdir -p results +cleanup() { + docker compose logs --no-color api-a api-b redis-proxy > results/containers.log 2>&1 || true + docker compose down --volumes --remove-orphans +} +trap cleanup EXIT INT TERM +docker compose build runner +docker compose up -d redis mongo redis-proxy +docker compose run --rm runner node test/auction-cluster/run.mjs init +docker compose up -d api-a api-b lb +docker compose run --rm runner node test/auction-cluster/run.mjs verify