test(auction): add two-node Docker failure integration suite

Exercise Nginx routing, Mongo transaction ordering, history backfills, Undo, Pub/Sub TCP loss, safety polling, resync, and reconnection. Keep fixtures and results isolated from production and git.
This commit is contained in:
dahyeon committed 2026-11-12 01:49:35 +09:00
1 parent 23a0a3f8dd
commit 8d741ef9a7
9 files changed
+1024

No files matched your search

@@ -0,0 +1 @@
* text eol=lf
+1
View File
@@ -0,0 +1 @@
results/
+13
View File
@@ -0,0 +1,13 @@
FROM node:26-bookworm-slim
RUN npm install --global pnpm@10.4.1
WORKDIR /workspace
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
COPY packages/contracts ./packages/contracts
COPY apps/api/package.json ./apps/api/package.json
COPY apps/web/package.json ./apps/web/package.json
RUN pnpm install --frozen-lockfile --filter api --filter web --ignore-scripts
COPY apps/api ./apps/api
COPY apps/web/src/api ./apps/web/src/api
WORKDIR /workspace/apps/api
ENV NODE_ENV=test NEW_RELIC_ENABLED=false
CMD ["node", "-r", "ts-node/register/transpile-only", "test/auction-cluster/node.ts"]
@@ -0,0 +1,61 @@
x-app: &app
image: ${COMPOSE_PROJECT_NAME}-app
build:
context: ../../../..
dockerfile: apps/api/test/auction-cluster/Dockerfile
environment: &environment
MONGO_TEST_URL: mongodb://mongo:27017/auction_cluster?replicaSet=auction_cluster
REDIS_TEST_URL: redis://redis-proxy:6379
NODE_ENV: test
NEW_RELIC_ENABLED: 'false'
volumes:
- ./results:/results
networks: [cluster]
mem_limit: 1g
services:
redis:
image: redis:8-alpine
command: [redis-server, --save, '', --appendonly, 'no']
networks: [cluster]
mem_limit: 256m
mongo:
# MongoDB 8+ refuses the test host's Linux 7.0.0 kernel (SERVER-121912).
image: ${MONGO_TEST_IMAGE:-mongo:7.0}
command:
[
mongod,
--replSet,
auction_cluster,
--bind_ip_all,
--wiredTigerCacheSizeGB,
'0.25',
]
networks: [cluster]
mem_limit: 768m
redis-proxy:
<<: *app
command: [node, test/auction-cluster/fault-proxy.mjs]
mem_limit: 128m
api-a:
<<: *app
environment:
<<: *environment
CLUSTER_NODE: api-a
api-b:
<<: *app
environment:
<<: *environment
CLUSTER_NODE: api-b
lb:
image: nginx:stable-alpine
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
networks: [cluster]
mem_limit: 128m
runner:
<<: *app
command: [node, test/auction-cluster/run.mjs, verify]
networks:
cluster: {}
@@ -0,0 +1,60 @@
import { createServer as createTcpServer, connect } from 'node:net';
import { createServer as createHttpServer } from 'node:http';
// Real TCP connections; only connections issuing SUBSCRIBE/PSUBSCRIBE are cut.
// Redis commands, TIME and PUBLISH continue through the same proxy.
let blocked = false;
let dropped = 0;
const connections = new Set();
createTcpServer((socket) => {
const upstream = connect(6379, 'redis');
const connection = { socket, upstream, subscriber: false, prefix: '' };
connections.add(connection);
const close = () => {
socket.destroy();
upstream.destroy();
connections.delete(connection);
};
socket.on('error', close);
upstream.on('error', close);
socket.on('close', close);
upstream.on('close', close);
socket.on('data', (chunk) => {
connection.prefix = (connection.prefix + chunk.toString('utf8')).slice(
-4096,
);
if (/\r\n(?:p?subscribe|ssubscribe)\r\n/i.test(connection.prefix)) {
connection.subscriber = true;
}
if (blocked && connection.subscriber) {
dropped += 1;
close();
} else upstream.write(chunk);
});
upstream.on('data', (chunk) => socket.write(chunk));
}).listen(6379, '0.0.0.0');
createHttpServer((request, response) => {
if (request.method === 'POST' && request.url === '/block') {
blocked = true;
for (const connection of connections) {
if (!connection.subscriber) continue;
dropped += 1;
connection.socket.destroy();
connection.upstream.destroy();
}
} else if (request.method === 'POST' && request.url === '/restore') {
blocked = false;
} else if (request.url !== '/status') {
response.writeHead(404).end();
return;
}
response.setHeader('content-type', 'application/json');
response.end(
JSON.stringify({
blocked,
dropped,
subscribers: [...connections].filter((c) => c.subscriber).length,
}),
);
}).listen(8081, '0.0.0.0');
+18
View File
@@ -0,0 +1,18 @@
events {}
http {
upstream auction {
server api-a:3000;
server api-b:3000;
}
server {
listen 8080;
location / {
proxy_pass http://auction;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 120s;
add_header X-Cluster-Upstream $upstream_addr always;
}
}
}
+203
View File
@@ -0,0 +1,203 @@
import 'reflect-metadata';
import { readFile } from 'node:fs/promises';
import { Body, Controller, Get, Module, Post } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import mongoose from 'mongoose';
import { createClient } from 'redis';
import type { Namespace } from 'socket.io';
import { RedisIoAdapter } from '../../src/redis-io.adapter';
import { AuctionController } from '../../src/modules/auction/auction.controller';
import { AuctionGateway } from '../../src/modules/auction/auction.gateway';
import { AuctionEventBusService } from '../../src/modules/auction/auction-event-bus.service';
import { AuctionService } from '../../src/modules/auction/auction.service';
import { AuctionBidArchiveService } from '../../src/modules/auction/auction-bid-archive.service';
import {
AuctionDurabilityScheduler,
type AuctionDurabilityPayload,
} from '../../src/modules/auction/auction-durability.scheduler';
import { AuctionRedisService } from '../../src/modules/auction/redis/auction-redis.service';
import { AuctionRedisCommandService } from '../../src/modules/auction/redis/auction-redis-command.service';
import { AuctionRedisStateReaderService } from '../../src/modules/auction/redis/auction-redis-state-reader.service';
import { AuthService } from '../../src/modules/auth/auth.service';
import { JwtService } from '../../src/modules/auth/jwt.service';
import { IdentityService } from '../../src/modules/auth/identity.service';
import { ConnectionService } from '../../src/modules/global/database/connection.service';
import {
AuctionSchema,
AuctionHistoryEntrySchema,
AuctionSideEffectApplicationSchema,
MatchSchema,
} from '../../src/modules/global/database/schema';
async function bootstrap() {
if (
!process.env.MONGO_TEST_URL?.startsWith(
'mongodb://mongo:27017/auction_cluster?',
) ||
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379'
) {
throw new Error(
'This test node only accepts the isolated Compose services.',
);
}
const fixture = JSON.parse(
await readFile('/results/fixture.json', 'utf8'),
) as { communityId: string };
const connection = await mongoose
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
.asPromise();
const models = {
match: connection.model('match', MatchSchema),
auction: connection.model('auction', AuctionSchema),
auctionHistory: connection.model(
'auctionHistory',
AuctionHistoryEntrySchema,
),
auctionSideEffectApplication: connection.model(
'auctionSideEffectApplication',
AuctionSideEffectApplicationSchema,
),
};
let releaseCommit: (() => void) | undefined;
let commitGate: Promise<void> | undefined;
let commitsWaiting = 0;
const database = {
getModel: async (name: keyof typeof models) => {
if (!models[name]) throw new Error(`Unexpected test model: ${name}`);
return models[name];
},
createSession: async () => {
const session = await connection.startSession();
const commit = session.commitTransaction.bind(session);
session.commitTransaction = async () => {
if (commitGate) {
commitsWaiting += 1;
await commitGate;
commitsWaiting -= 1;
}
await commit();
};
return session;
},
};
const client = createClient({ url: process.env.REDIS_TEST_URL });
client.on('error', (error) =>
console.error('command-client:', error.message),
);
await client.connect();
const redis = new AuctionRedisService(client);
const commands = new AuctionRedisCommandService(redis);
const reader = new AuctionRedisStateReaderService(redis);
const bus = new AuctionEventBusService();
const service = new AuctionService(
database as never,
{} as never,
{} as never,
{} as never,
redis,
commands,
reader,
bus,
);
// Explicit draining lets the test hold real Mongo commits and reorder durable
// payloads; no in-memory database or fake Redis adapter is involved.
const scheduler = new AuctionDurabilityScheduler(
database as never,
redis,
undefined,
undefined,
undefined,
bus,
reader,
service,
);
@Controller('_cluster')
class ControlController {
@Get('health') health() {
const namespace = Reflect.get(gateway, 'server') as Namespace;
return {
node: process.env.CLUSTER_NODE,
sockets: [...namespace.sockets.keys()],
pid: process.pid,
commitsWaiting,
};
}
@Post('hold-commit') hold() {
if (commitGate) throw new Error('Commit already held');
commitGate = new Promise<void>((resolve) => {
releaseCommit = resolve;
});
return { held: true };
}
@Post('release-commit') release() {
releaseCommit?.();
commitGate = undefined;
releaseCommit = undefined;
return { released: true };
}
@Post('drain') drain() {
return scheduler.processDueDurabilityPayloads(100);
}
@Post('apply') async apply(@Body() payload: AuctionDurabilityPayload) {
await scheduler.applyDurabilityPayload(payload);
return { applied: true };
}
}
@Module({
controllers: [AuctionController, ControlController],
providers: [
AuctionGateway,
{ provide: AuctionService, useValue: service },
{ provide: ConnectionService, useValue: database },
{ provide: AuctionRedisService, useValue: redis },
{ provide: AuctionEventBusService, useValue: bus },
{ provide: AuctionBidArchiveService, useValue: {} },
{
provide: AuthService,
useValue: { resolveDevUserById: async () => null },
},
{ provide: JwtService, useValue: {} },
{ provide: IdentityService, useValue: {} },
],
})
class TestModule {}
const app = await NestFactory.create(TestModule, {
logger: ['error', 'warn'],
});
// Identity-provider authentication is outside this transport/durability test.
// The real controller's RolesGuard still checks this isolated fixture admin.
app.use(
(
request: { user?: unknown },
response: { setHeader(name: string, value: string): void },
next: () => void,
) => {
request.user = {
role: 'admin',
communityId: new mongoose.Types.ObjectId(fixture.communityId),
};
response.setHeader(
'X-Cluster-Node',
process.env.CLUSTER_NODE ?? 'unknown',
);
next();
},
);
const adapter = new RedisIoAdapter(app);
await adapter.connectToRedis();
app.useWebSocketAdapter(adapter);
const gateway = app.get(AuctionGateway);
await app.listen(3000, '0.0.0.0');
console.log(`cluster node ready: ${process.env.CLUSTER_NODE}`);
process.on('SIGTERM', () => {
void app
.close()
.then(() => connection.close())
.then(() => process.exit(0));
});
}
void bootstrap().catch((error: unknown) => {
console.error(error);
process.exitCode = 1;
});
+641
View File
@@ -0,0 +1,641 @@
import assert from 'node:assert/strict';
import { createRequire } from 'node:module';
import { readFile, writeFile } from 'node:fs/promises';
import { request } from 'node:http';
import { setTimeout as delay } from 'node:timers/promises';
import { randomUUID } from 'node:crypto';
import mongoose from 'mongoose';
import { createClient } from 'redis';
const require = createRequire(import.meta.url);
require('ts-node/register/transpile-only');
const {
AuctionRedisService,
} = require('../../src/modules/auction/redis/auction-redis.service.ts');
const {
AuctionRedisCommandService,
} = require('../../src/modules/auction/redis/auction-redis-command.service.ts');
const {
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
} = require('../../src/modules/auction/redis/auction-redis.keys.ts');
const {
MatchSchema,
AuctionSchema,
AuctionHistoryEntrySchema,
AuctionSideEffectApplicationSchema,
} = require('../../src/modules/global/database/schema.ts');
const webRequire = createRequire(
new URL('../../../web/package.json', import.meta.url),
);
const { io } = webRequire('socket.io-client');
const { QueryClient, QueryObserver, environmentManager } = webRequire(
'@tanstack/react-query',
);
const {
createAuctionHeartbeatMonitor,
shouldApplyPolledAuctionState,
mergeAuctionState,
} = await import('../../../web/src/api/auction-state.ts');
const { createAuctionResyncRequester } =
await import('../../../web/src/api/auction-resync.ts');
const { AUCTION_SAFETY_POLL_INTERVAL_MS } =
await import('../../../web/src/api/auction-polling.ts');
if (
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379' ||
!process.env.MONGO_TEST_URL?.startsWith(
'mongodb://mongo:27017/auction_cluster?',
)
) {
throw new Error('Only the isolated Compose services are allowed.');
}
const redis = createClient({ url: 'redis://redis:6379' });
redis.on('error', () => {});
await redis.connect();
const cleanup = [() => (redis.isOpen ? redis.quit() : undefined)];
const report = {
startedAt: new Date().toISOString(),
checks: [],
requests: [],
limitation:
'Real API processes, Nginx, Redis TCP faults and Mongo transactions; fixture identity replaces the external login provider. Client recovery uses production helpers and QueryObserver, without rendering the React page.',
};
async function until(check, description, timeout = 15_000) {
const deadline = Date.now() + timeout;
let lastError;
while (Date.now() < deadline) {
try {
const value = await check();
if (value) return value;
} catch (error) {
lastError = error;
}
await delay(50);
}
throw new Error(
`Timed out: ${description}${lastError ? `: ${lastError.message}` : ''}`,
);
}
async function http(url, body) {
const started = Date.now();
return new Promise((resolve, reject) => {
const req = request(
url,
{
method: body === undefined ? 'GET' : 'POST',
agent: false,
headers: { 'content-type': 'application/json' },
},
(res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
report.requests.push({
url,
node: res.headers['x-cluster-node'],
status: res.statusCode,
durationMS: Date.now() - started,
});
if (res.statusCode < 200 || res.statusCode >= 300)
return reject(new Error(`${url}: ${res.statusCode} ${data}`));
try {
resolve({
body: JSON.parse(data),
node: res.headers['x-cluster-node'],
});
} catch (error) {
reject(error);
}
});
},
);
req.setTimeout(10_000, () =>
req.destroy(new Error(`HTTP timeout: ${url}`)),
);
req.on('error', reject);
req.end(body === undefined ? undefined : JSON.stringify(body));
});
}
const pass = (name, details = {}) => {
report.checks.push({ name, passed: true, ...details });
console.log(`PASS ${name} ${JSON.stringify(details)}`);
};
async function initialize() {
const admin = new mongoose.mongo.MongoClient(
'mongodb://mongo:27017/?directConnection=true',
{ serverSelectionTimeoutMS: 1000 },
);
await until(
async () => {
await admin.connect();
return true;
},
'Mongo startup',
60_000,
);
await admin.db('admin').command({
replSetInitiate: {
_id: 'auction_cluster',
members: [{ _id: 0, host: 'mongo:27017' }],
},
});
await until(
async () =>
(await admin.db('admin').command({ hello: 1 })).isWritablePrimary,
'Mongo primary',
60_000,
);
const build = await admin.db('admin').command({ buildInfo: 1 });
await admin.close();
const connection = await mongoose
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
.asPromise();
try {
const models = {
match: connection.model('match', MatchSchema),
auction: connection.model('auction', AuctionSchema),
history: connection.model('auctionHistory', AuctionHistoryEntrySchema),
sideEffects: connection.model(
'auctionSideEffectApplication',
AuctionSideEffectApplicationSchema,
),
};
await Promise.all(
Object.values(models).map((model) => model.createCollection()),
);
await models.history.createIndexes();
const matchId = new mongoose.Types.ObjectId(),
communityId = new mongoose.Types.ObjectId(),
auctionId = new mongoose.Types.ObjectId();
const lifecycle = `cluster:${matchId}`;
const playerA = new mongoose.Types.ObjectId(),
playerB = new mongoose.Types.ObjectId(),
captain = new mongoose.Types.ObjectId();
const structure = {
auctionPlayerOrder: [
{ userId: playerA.toString() },
{ userId: playerB.toString() },
],
auctionTeams: [
{
teamIndex: 1,
captainUserId: captain.toString(),
remainingBidPointAmount: 1000,
members: [],
},
],
currentAuctionPlayerOrderIndex: 0,
currentAuctionRound: 1,
passedAuctionPlayerQueue: [],
maximumBidRoundDurationSeconds: 600,
bidRoundExtensionSeconds: 5,
initialTeamBidPointAmount: 1000,
topTierAuctionPriorityThreshold: 'NONE',
manualTopTierAuctionPriorityUserIds: [],
};
await models.match.collection.insertOne({
_id: matchId,
communityId,
title: 'Isolated cluster test',
participants: [],
teams: [],
status: 'team-building',
scheduledAt: new Date(),
});
await models.auction.collection.insertOne({
_id: auctionId,
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: lifecycle,
phase: 'presenting',
revision: 0,
currentAuctionHistorySequence: 0,
chatMessages: [],
...structure,
});
const commands = new AuctionRedisCommandService(
new AuctionRedisService(redis),
);
await commands.hydrateState(matchId.toString(), {
matchId: matchId.toString(),
auctionLifecycleId: lifecycle,
phase: 'presenting',
revision: 0,
auctionHistorySequence: 0,
maximumBidRoundDurationSeconds: 600,
structureStateJson: JSON.stringify(structure),
liveStatePatch: {
currentAuctionPlayerOrderIndex: 0,
currentBidRoundPlayerUserId: playerA.toString(),
currentBidRoundId: 'initial',
currentHighestBidPointAmount: 0,
currentHighestBidderTeamIndex: -1,
maximumBidRoundDurationSeconds: 600,
bidRoundExtensionSeconds: 5,
'team:1:captainUserId': captain.toString(),
'team:1:remainingBidPointAmount': 1000,
'team:1:assignedAuctionTeamMemberCount': 1,
},
});
const fixture = {
matchId: matchId.toString(),
communityId: communityId.toString(),
auctionId: auctionId.toString(),
lifecycle,
mongoVersion: build.version,
redisVersion: (await redis.info('server')).match(
/redis_version:([^\r\n]+)/,
)?.[1],
};
await writeFile('/results/fixture.json', JSON.stringify(fixture, null, 2));
console.log(
`Initialized isolated MongoDB ${build.version} and Redis ${fixture.redisVersion}`,
);
} finally {
await connection.close();
}
}
async function verify() {
const fixture = JSON.parse(await readFile('/results/fixture.json', 'utf8'));
report.versions = {
node: process.version,
mongo: fixture.mongoVersion,
redis: fixture.redisVersion,
};
const route = `/communities/${fixture.communityId}/matches/${fixture.matchId}/auction`;
const url = `http://lb:8080${route}`;
await until(
async () => (await http('http://api-a:3000/_cluster/health')).body.node,
'API A startup',
60_000,
);
await until(
async () => (await http('http://api-b:3000/_cluster/health')).body.node,
'API B startup',
60_000,
);
const routedNodes = new Set();
for (let i = 0; i < 4; i += 1) routedNodes.add((await http(url)).node);
assert.deepEqual([...routedNodes].sort(), ['api-a', 'api-b']);
pass('Nginx routes real HTTP requests to both API processes');
async function connectViewer() {
const socket = io('http://lb:8080/auction', {
transports: ['websocket'],
autoConnect: false,
reconnection: false,
});
const viewer = {
socket,
state: null,
states: [],
histories: [],
errors: [],
};
socket.on('state', (state) => {
viewer.state = state;
viewer.states.push(state);
});
socket.on('error', (error) => viewer.errors.push(error));
socket.on('connect_error', (error) => viewer.errors.push(error.message));
socket.on('history-updated', (event) => {
void http(`${url}/history`)
.then(({ body, node }) => viewer.histories.push({ event, body, node }))
.catch((error) => viewer.errors.push(error.message));
});
cleanup.push(() => socket.disconnect());
socket.connect();
await until(
() => socket.connected,
`socket connect: ${viewer.errors.join(',')}`,
);
const ack = await socket
.timeout(5000)
.emitWithAck('join', { matchId: fixture.matchId });
assert.equal(ack.ok, true);
assert.equal(viewer.state.matchId, fixture.matchId);
assert.equal(viewer.state.auctionEngineVersion, 'redis-v1');
const health = await Promise.all(
['api-a', 'api-b'].map((node) =>
http(`http://${node}:3000/_cluster/health`),
),
);
viewer.node = health.find((entry) =>
entry.body.sockets.includes(socket.id),
).body.node;
return viewer;
}
const one = await connectViewer(),
two = await connectViewer();
assert.notEqual(one.node, two.node);
const a = one.node === 'api-a' ? one : two;
const b = one.node === 'api-b' ? one : two;
const initialRevision = a.state.revision;
const revisionFor = (sequence) => initialRevision + sequence;
assert.equal(b.state.revision, initialRevision);
pass('WebSocket viewers join different nodes through Nginx', {
nodes: [one.node, two.node],
});
const sequences = async () =>
(await http(`${url}/history`)).body.map((entry) => entry.sequence);
const action = async (viewer, name) => {
const ack = await viewer.socket
.timeout(5000)
.emitWithAck(name, { auctionActionRequestId: randomUUID() });
assert.equal(ack.ok, true, JSON.stringify(ack));
};
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
await action(a, 'open-bidding');
await until(
() =>
a.state.revision === revisionFor(1) &&
b.state.revision === revisionFor(1),
'cross-node full state',
);
assert.equal(a.histories.length + b.histories.length, 0);
await http('http://api-a:3000/_cluster/hold-commit', {});
const pendingDrain = http('http://api-a:3000/_cluster/drain', {});
await until(
async () =>
(await http('http://api-a:3000/_cluster/health')).body.commitsWaiting ===
1,
'real Mongo commit barrier',
);
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
assert.equal(a.histories.length + b.histories.length, 0);
await http('http://api-a:3000/_cluster/release-commit', {});
assert.equal((await pendingDrain).body.applied, 1);
await until(
() => a.histories.length === 1 && b.histories.length === 1,
'post-commit history event on both nodes',
);
for (const viewer of [a, b])
assert.deepEqual(
viewer.histories[0].body.map((entry) => entry.sequence),
[1],
);
pass(
'History notification crosses nodes only after a real Mongo transaction commits',
);
await action(a, 'passed');
await action(b, 'next');
await until(
() =>
a.state.revision === revisionFor(3) &&
b.state.revision === revisionFor(3),
'state fanout in both directions',
);
const keys = await redis.zRange(AUCTION_DURABILITY_RETRY_QUEUE_KEY, 0, -1);
const queued = await Promise.all(
keys.map(async (key) => ({
key,
payload: JSON.parse(await redis.hGet(key, 'payloadJson')),
})),
);
const third = queued.find(
({ payload }) => payload.auctionHistorySequence === 3,
);
const second = queued.find(
({ payload }) => payload.auctionHistorySequence === 2,
);
assert.ok(third && second);
await http('http://api-b:3000/_cluster/apply', third.payload);
assert.deepEqual(await sequences(), [3, 1]);
const previousEvents = b.histories.length;
await http('http://api-a:3000/_cluster/apply', second.payload);
await until(
() =>
b.histories.length > previousEvents &&
b.histories.at(-1).body.some((entry) => entry.sequence === 2),
'late append history refresh',
);
assert.deepEqual(await sequences(), [3, 2, 1]);
for (const item of queued) {
await redis.zRem(AUCTION_DURABILITY_RETRY_QUEUE_KEY, item.key);
await redis.del(item.key);
}
pass(
'A delayed history append refreshes another node with the same maximum sequence',
{ maximumSequence: 3 },
);
const undo = await http(`${url}/undo`, {
sequence: 1,
auctionActionRequestId: randomUUID(),
});
assert.equal(undo.body.auctionEngineVersion, 'redis-v1');
assert.equal(undo.body.revision, revisionFor(4));
assert.deepEqual(await sequences(), [3, 2, 1]);
await http('http://api-b:3000/_cluster/drain', {});
await until(
async () => JSON.stringify(await sequences()) === '[4,1]',
'durable Undo history',
);
await until(
() => b.histories.at(-1)?.body[0]?.sequence === 4,
'Undo history event',
);
pass('Undo increases sequence and replaces history on every node', {
sequence: 4,
history: [4, 1],
});
await until(
async () =>
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
'Redis subscriptions',
);
await http('http://redis-proxy:8081/block', {});
console.log(
'FAULT Pub/Sub TCP connections blocked; Redis commands and HTTP stay available',
);
await delay(500);
assert.equal(
(await http('http://api-a:3000/_cluster/health')).body.node,
'api-a',
);
assert.equal(
(await http('http://api-b:3000/_cluster/health')).body.node,
'api-b',
);
assert.equal(a.socket.connected && b.socket.connected, true);
const timestamp = await b.socket
.timeout(3000)
.emitWithAck('time-sync', { sequence: 1 });
assert.equal(timestamp.ok, true);
pass(
'Both API processes and socket clock sync survive Pub/Sub-only TCP disconnection',
);
environmentManager.setIsServer(() => false);
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false, gcTime: Infinity } },
});
cleanup.push(() => queryClient.clear());
let stateByHttp = b.state,
historyByHttp,
statePolls = 0,
historyPolls = 0;
const pollingStarted = Date.now();
const stateObserver = new QueryObserver(queryClient, {
queryKey: ['auction', fixture.matchId],
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
queryFn: async () => {
statePolls += 1;
return (await http(url)).body;
},
});
const historyObserver = new QueryObserver(queryClient, {
queryKey: ['auction-history', fixture.matchId],
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
queryFn: async () => {
historyPolls += 1;
return (await http(`${url}/history`)).body;
},
});
cleanup.push(
stateObserver.subscribe((result) => {
if (
result.data &&
shouldApplyPolledAuctionState(stateByHttp, result.data, fixture.matchId)
)
stateByHttp = mergeAuctionState(stateByHttp, result.data);
}),
);
cleanup.push(
historyObserver.subscribe((result) => {
if (result.data) historyByHttp = result.data;
}),
);
await until(
() =>
stateByHttp.revision === revisionFor(4) &&
historyByHttp?.[0]?.sequence === 4,
'baseline HTTP reads',
);
const beforeHistoryEvents = b.histories.length;
await action(a, 'open-bidding');
await http('http://api-a:3000/_cluster/drain', {});
await delay(250);
assert.equal(b.state.revision, revisionFor(4));
assert.equal(b.histories.length, beforeHistoryEvents);
assert.equal(a.state.revision, revisionFor(5));
// Do not run heartbeat recovery yet: verify the independent 30-second safety
// route even if the client identity checker fails to notice a missed event.
console.log(
'WAIT Real 30-second HTTP safety polling with Pub/Sub still blocked',
);
await until(
() =>
stateByHttp.revision === revisionFor(5) &&
historyByHttp?.[0]?.sequence === 5,
'30-second HTTP safety fallback',
35_000,
);
assert.ok(Date.now() - pollingStarted >= 29_000);
pass(
'Real 30-second HTTP polls repair missed state and history through Nginx',
{ elapsedMS: Date.now() - pollingStarted, statePolls, historyPolls },
);
const monitor = createAuctionHeartbeatMonitor();
const heartbeat = async (sequence) => {
const ack = await b.socket
.timeout(3000)
.emitWithAck('time-sync', { sequence });
assert.equal(ack.ok, true);
return monitor.inspect(b.state, ack.auctionState);
};
assert.equal((await heartbeat(2)).needsResync, false);
await delay(10_000);
assert.equal((await heartbeat(3)).needsResync, true);
let resyncs = 0;
const requester = createAuctionResyncRequester(
async () => {
resyncs += 1;
return b.socket.timeout(3000).emitWithAck('resync', {});
},
(state) => {
b.state = state;
},
);
await requester.request(revisionFor(5), fixture.lifecycle);
assert.equal(b.state.revision, revisionFor(5));
assert.equal(resyncs, 1);
pass(
'Two real heartbeats detect a persistent gap and private socket resync works during Pub/Sub outage',
{ resyncs },
);
await http('http://redis-proxy:8081/restore', {});
await until(
async () =>
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
'Pub/Sub resubscription',
20_000,
);
await action(a, 'passed');
await http('http://api-a:3000/_cluster/drain', {});
await until(
() =>
b.state.revision === revisionFor(6) &&
b.histories.at(-1)?.body[0]?.sequence === 6,
'events after Pub/Sub restoration',
);
pass(
'Cross-node state and history events recover after restoring the TCP path',
);
const fault = (await http('http://redis-proxy:8081/status')).body;
report.fault = fault;
const previousId = b.socket.id;
b.socket.disconnect();
b.socket.connect();
await until(
() => b.socket.connected && b.socket.id !== previousId,
'socket reconnect through Nginx',
);
assert.equal(
(
await b.socket
.timeout(5000)
.emitWithAck('join', { matchId: fixture.matchId })
).ok,
true,
);
await until(
() => b.state.revision === revisionFor(6),
'reconnect full state',
);
pass('A reconnected viewer receives the latest snapshot through Nginx');
}
try {
if (process.argv[2] === 'init') await initialize();
else {
await verify();
report.passed = true;
}
} catch (error) {
report.passed = false;
report.failure = { message: error.message, stack: error.stack };
console.error(error);
process.exitCode = 1;
} finally {
if (process.argv[2] !== 'init') {
await http('http://redis-proxy:8081/restore', {}).catch(() => {});
report.finishedAt = new Date().toISOString();
await writeFile('/results/report.json', JSON.stringify(report, null, 2));
}
for (const close of cleanup.reverse()) {
try {
await close();
} catch {}
}
}
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
# From the repository root: sh apps/api/test/auction-cluster/run.sh
# Requires Docker Compose. No host ports or production credentials are used.
# Results and container logs remain in this directory's ignored results/ folder.
# Authentication uses a fixture admin; client recovery uses the production
# helpers and QueryObserver without rendering React. MongoDB 7 is the default
# for the test host's kernel; set MONGO_TEST_IMAGE on a compatible host to match
# the production database version.
set -eu
cd "$(dirname "$0")"
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-lolympus-auction-cluster-$(date +%s)}"
case "$COMPOSE_PROJECT_NAME" in
lolympus-auction-cluster-*) ;;
*) echo 'Refusing to use a Compose project outside the auction-cluster test prefix.' >&2; exit 1 ;;
esac
mkdir -p results
cleanup() {
docker compose logs --no-color api-a api-b redis-proxy > results/containers.log 2>&1 || true
docker compose down --volumes --remove-orphans
}
trap cleanup EXIT INT TERM
docker compose build runner
docker compose up -d redis mongo redis-proxy
docker compose run --rm runner node test/auction-cluster/run.mjs init
docker compose up -d api-a api-b lb
docker compose run --rm runner node test/auction-cluster/run.mjs verify