test(auction): add two-node Docker failure integration suite
Exercise Nginx routing, Mongo transaction ordering, history backfills, Undo, Pub/Sub TCP loss, safety polling, resync, and reconnection. Keep fixtures and results isolated from production and git.
This commit is contained in:
1 parent
23a0a3f8dd
commit
8d741ef9a7
9 files changed
+1024
No files matched your search
@@ -0,0 +1 @@
|
||||
* text eol=lf
|
||||
@@ -0,0 +1 @@
|
||||
results/
|
||||
@@ -0,0 +1,13 @@
|
||||
FROM node:26-bookworm-slim
|
||||
RUN npm install --global pnpm@10.4.1
|
||||
WORKDIR /workspace
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY packages/contracts ./packages/contracts
|
||||
COPY apps/api/package.json ./apps/api/package.json
|
||||
COPY apps/web/package.json ./apps/web/package.json
|
||||
RUN pnpm install --frozen-lockfile --filter api --filter web --ignore-scripts
|
||||
COPY apps/api ./apps/api
|
||||
COPY apps/web/src/api ./apps/web/src/api
|
||||
WORKDIR /workspace/apps/api
|
||||
ENV NODE_ENV=test NEW_RELIC_ENABLED=false
|
||||
CMD ["node", "-r", "ts-node/register/transpile-only", "test/auction-cluster/node.ts"]
|
||||
@@ -0,0 +1,61 @@
|
||||
x-app: &app
|
||||
image: ${COMPOSE_PROJECT_NAME}-app
|
||||
build:
|
||||
context: ../../../..
|
||||
dockerfile: apps/api/test/auction-cluster/Dockerfile
|
||||
environment: &environment
|
||||
MONGO_TEST_URL: mongodb://mongo:27017/auction_cluster?replicaSet=auction_cluster
|
||||
REDIS_TEST_URL: redis://redis-proxy:6379
|
||||
NODE_ENV: test
|
||||
NEW_RELIC_ENABLED: 'false'
|
||||
volumes:
|
||||
- ./results:/results
|
||||
networks: [cluster]
|
||||
mem_limit: 1g
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:8-alpine
|
||||
command: [redis-server, --save, '', --appendonly, 'no']
|
||||
networks: [cluster]
|
||||
mem_limit: 256m
|
||||
mongo:
|
||||
# MongoDB 8+ refuses the test host's Linux 7.0.0 kernel (SERVER-121912).
|
||||
image: ${MONGO_TEST_IMAGE:-mongo:7.0}
|
||||
command:
|
||||
[
|
||||
mongod,
|
||||
--replSet,
|
||||
auction_cluster,
|
||||
--bind_ip_all,
|
||||
--wiredTigerCacheSizeGB,
|
||||
'0.25',
|
||||
]
|
||||
networks: [cluster]
|
||||
mem_limit: 768m
|
||||
redis-proxy:
|
||||
<<: *app
|
||||
command: [node, test/auction-cluster/fault-proxy.mjs]
|
||||
mem_limit: 128m
|
||||
api-a:
|
||||
<<: *app
|
||||
environment:
|
||||
<<: *environment
|
||||
CLUSTER_NODE: api-a
|
||||
api-b:
|
||||
<<: *app
|
||||
environment:
|
||||
<<: *environment
|
||||
CLUSTER_NODE: api-b
|
||||
lb:
|
||||
image: nginx:stable-alpine
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
networks: [cluster]
|
||||
mem_limit: 128m
|
||||
runner:
|
||||
<<: *app
|
||||
command: [node, test/auction-cluster/run.mjs, verify]
|
||||
|
||||
networks:
|
||||
cluster: {}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { createServer as createTcpServer, connect } from 'node:net';
|
||||
import { createServer as createHttpServer } from 'node:http';
|
||||
|
||||
// Real TCP connections; only connections issuing SUBSCRIBE/PSUBSCRIBE are cut.
|
||||
// Redis commands, TIME and PUBLISH continue through the same proxy.
|
||||
let blocked = false;
|
||||
let dropped = 0;
|
||||
const connections = new Set();
|
||||
createTcpServer((socket) => {
|
||||
const upstream = connect(6379, 'redis');
|
||||
const connection = { socket, upstream, subscriber: false, prefix: '' };
|
||||
connections.add(connection);
|
||||
const close = () => {
|
||||
socket.destroy();
|
||||
upstream.destroy();
|
||||
connections.delete(connection);
|
||||
};
|
||||
socket.on('error', close);
|
||||
upstream.on('error', close);
|
||||
socket.on('close', close);
|
||||
upstream.on('close', close);
|
||||
socket.on('data', (chunk) => {
|
||||
connection.prefix = (connection.prefix + chunk.toString('utf8')).slice(
|
||||
-4096,
|
||||
);
|
||||
if (/\r\n(?:p?subscribe|ssubscribe)\r\n/i.test(connection.prefix)) {
|
||||
connection.subscriber = true;
|
||||
}
|
||||
if (blocked && connection.subscriber) {
|
||||
dropped += 1;
|
||||
close();
|
||||
} else upstream.write(chunk);
|
||||
});
|
||||
upstream.on('data', (chunk) => socket.write(chunk));
|
||||
}).listen(6379, '0.0.0.0');
|
||||
|
||||
createHttpServer((request, response) => {
|
||||
if (request.method === 'POST' && request.url === '/block') {
|
||||
blocked = true;
|
||||
for (const connection of connections) {
|
||||
if (!connection.subscriber) continue;
|
||||
dropped += 1;
|
||||
connection.socket.destroy();
|
||||
connection.upstream.destroy();
|
||||
}
|
||||
} else if (request.method === 'POST' && request.url === '/restore') {
|
||||
blocked = false;
|
||||
} else if (request.url !== '/status') {
|
||||
response.writeHead(404).end();
|
||||
return;
|
||||
}
|
||||
response.setHeader('content-type', 'application/json');
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
blocked,
|
||||
dropped,
|
||||
subscribers: [...connections].filter((c) => c.subscriber).length,
|
||||
}),
|
||||
);
|
||||
}).listen(8081, '0.0.0.0');
|
||||
@@ -0,0 +1,18 @@
|
||||
events {}
|
||||
http {
|
||||
upstream auction {
|
||||
server api-a:3000;
|
||||
server api-b:3000;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
location / {
|
||||
proxy_pass http://auction;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_read_timeout 120s;
|
||||
add_header X-Cluster-Upstream $upstream_addr always;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
import 'reflect-metadata';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { Body, Controller, Get, Module, Post } from '@nestjs/common';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import mongoose from 'mongoose';
|
||||
import { createClient } from 'redis';
|
||||
import type { Namespace } from 'socket.io';
|
||||
import { RedisIoAdapter } from '../../src/redis-io.adapter';
|
||||
import { AuctionController } from '../../src/modules/auction/auction.controller';
|
||||
import { AuctionGateway } from '../../src/modules/auction/auction.gateway';
|
||||
import { AuctionEventBusService } from '../../src/modules/auction/auction-event-bus.service';
|
||||
import { AuctionService } from '../../src/modules/auction/auction.service';
|
||||
import { AuctionBidArchiveService } from '../../src/modules/auction/auction-bid-archive.service';
|
||||
import {
|
||||
AuctionDurabilityScheduler,
|
||||
type AuctionDurabilityPayload,
|
||||
} from '../../src/modules/auction/auction-durability.scheduler';
|
||||
import { AuctionRedisService } from '../../src/modules/auction/redis/auction-redis.service';
|
||||
import { AuctionRedisCommandService } from '../../src/modules/auction/redis/auction-redis-command.service';
|
||||
import { AuctionRedisStateReaderService } from '../../src/modules/auction/redis/auction-redis-state-reader.service';
|
||||
import { AuthService } from '../../src/modules/auth/auth.service';
|
||||
import { JwtService } from '../../src/modules/auth/jwt.service';
|
||||
import { IdentityService } from '../../src/modules/auth/identity.service';
|
||||
import { ConnectionService } from '../../src/modules/global/database/connection.service';
|
||||
import {
|
||||
AuctionSchema,
|
||||
AuctionHistoryEntrySchema,
|
||||
AuctionSideEffectApplicationSchema,
|
||||
MatchSchema,
|
||||
} from '../../src/modules/global/database/schema';
|
||||
|
||||
async function bootstrap() {
|
||||
if (
|
||||
!process.env.MONGO_TEST_URL?.startsWith(
|
||||
'mongodb://mongo:27017/auction_cluster?',
|
||||
) ||
|
||||
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379'
|
||||
) {
|
||||
throw new Error(
|
||||
'This test node only accepts the isolated Compose services.',
|
||||
);
|
||||
}
|
||||
const fixture = JSON.parse(
|
||||
await readFile('/results/fixture.json', 'utf8'),
|
||||
) as { communityId: string };
|
||||
const connection = await mongoose
|
||||
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
|
||||
.asPromise();
|
||||
const models = {
|
||||
match: connection.model('match', MatchSchema),
|
||||
auction: connection.model('auction', AuctionSchema),
|
||||
auctionHistory: connection.model(
|
||||
'auctionHistory',
|
||||
AuctionHistoryEntrySchema,
|
||||
),
|
||||
auctionSideEffectApplication: connection.model(
|
||||
'auctionSideEffectApplication',
|
||||
AuctionSideEffectApplicationSchema,
|
||||
),
|
||||
};
|
||||
let releaseCommit: (() => void) | undefined;
|
||||
let commitGate: Promise<void> | undefined;
|
||||
let commitsWaiting = 0;
|
||||
const database = {
|
||||
getModel: async (name: keyof typeof models) => {
|
||||
if (!models[name]) throw new Error(`Unexpected test model: ${name}`);
|
||||
return models[name];
|
||||
},
|
||||
createSession: async () => {
|
||||
const session = await connection.startSession();
|
||||
const commit = session.commitTransaction.bind(session);
|
||||
session.commitTransaction = async () => {
|
||||
if (commitGate) {
|
||||
commitsWaiting += 1;
|
||||
await commitGate;
|
||||
commitsWaiting -= 1;
|
||||
}
|
||||
await commit();
|
||||
};
|
||||
return session;
|
||||
},
|
||||
};
|
||||
const client = createClient({ url: process.env.REDIS_TEST_URL });
|
||||
client.on('error', (error) =>
|
||||
console.error('command-client:', error.message),
|
||||
);
|
||||
await client.connect();
|
||||
const redis = new AuctionRedisService(client);
|
||||
const commands = new AuctionRedisCommandService(redis);
|
||||
const reader = new AuctionRedisStateReaderService(redis);
|
||||
const bus = new AuctionEventBusService();
|
||||
const service = new AuctionService(
|
||||
database as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
redis,
|
||||
commands,
|
||||
reader,
|
||||
bus,
|
||||
);
|
||||
// Explicit draining lets the test hold real Mongo commits and reorder durable
|
||||
// payloads; no in-memory database or fake Redis adapter is involved.
|
||||
const scheduler = new AuctionDurabilityScheduler(
|
||||
database as never,
|
||||
redis,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
bus,
|
||||
reader,
|
||||
service,
|
||||
);
|
||||
|
||||
@Controller('_cluster')
|
||||
class ControlController {
|
||||
@Get('health') health() {
|
||||
const namespace = Reflect.get(gateway, 'server') as Namespace;
|
||||
return {
|
||||
node: process.env.CLUSTER_NODE,
|
||||
sockets: [...namespace.sockets.keys()],
|
||||
pid: process.pid,
|
||||
commitsWaiting,
|
||||
};
|
||||
}
|
||||
@Post('hold-commit') hold() {
|
||||
if (commitGate) throw new Error('Commit already held');
|
||||
commitGate = new Promise<void>((resolve) => {
|
||||
releaseCommit = resolve;
|
||||
});
|
||||
return { held: true };
|
||||
}
|
||||
@Post('release-commit') release() {
|
||||
releaseCommit?.();
|
||||
commitGate = undefined;
|
||||
releaseCommit = undefined;
|
||||
return { released: true };
|
||||
}
|
||||
@Post('drain') drain() {
|
||||
return scheduler.processDueDurabilityPayloads(100);
|
||||
}
|
||||
@Post('apply') async apply(@Body() payload: AuctionDurabilityPayload) {
|
||||
await scheduler.applyDurabilityPayload(payload);
|
||||
return { applied: true };
|
||||
}
|
||||
}
|
||||
@Module({
|
||||
controllers: [AuctionController, ControlController],
|
||||
providers: [
|
||||
AuctionGateway,
|
||||
{ provide: AuctionService, useValue: service },
|
||||
{ provide: ConnectionService, useValue: database },
|
||||
{ provide: AuctionRedisService, useValue: redis },
|
||||
{ provide: AuctionEventBusService, useValue: bus },
|
||||
{ provide: AuctionBidArchiveService, useValue: {} },
|
||||
{
|
||||
provide: AuthService,
|
||||
useValue: { resolveDevUserById: async () => null },
|
||||
},
|
||||
{ provide: JwtService, useValue: {} },
|
||||
{ provide: IdentityService, useValue: {} },
|
||||
],
|
||||
})
|
||||
class TestModule {}
|
||||
const app = await NestFactory.create(TestModule, {
|
||||
logger: ['error', 'warn'],
|
||||
});
|
||||
// Identity-provider authentication is outside this transport/durability test.
|
||||
// The real controller's RolesGuard still checks this isolated fixture admin.
|
||||
app.use(
|
||||
(
|
||||
request: { user?: unknown },
|
||||
response: { setHeader(name: string, value: string): void },
|
||||
next: () => void,
|
||||
) => {
|
||||
request.user = {
|
||||
role: 'admin',
|
||||
communityId: new mongoose.Types.ObjectId(fixture.communityId),
|
||||
};
|
||||
response.setHeader(
|
||||
'X-Cluster-Node',
|
||||
process.env.CLUSTER_NODE ?? 'unknown',
|
||||
);
|
||||
next();
|
||||
},
|
||||
);
|
||||
const adapter = new RedisIoAdapter(app);
|
||||
await adapter.connectToRedis();
|
||||
app.useWebSocketAdapter(adapter);
|
||||
const gateway = app.get(AuctionGateway);
|
||||
await app.listen(3000, '0.0.0.0');
|
||||
console.log(`cluster node ready: ${process.env.CLUSTER_NODE}`);
|
||||
process.on('SIGTERM', () => {
|
||||
void app
|
||||
.close()
|
||||
.then(() => connection.close())
|
||||
.then(() => process.exit(0));
|
||||
});
|
||||
}
|
||||
void bootstrap().catch((error: unknown) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,641 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createRequire } from 'node:module';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { request } from 'node:http';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import mongoose from 'mongoose';
|
||||
import { createClient } from 'redis';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
require('ts-node/register/transpile-only');
|
||||
const {
|
||||
AuctionRedisService,
|
||||
} = require('../../src/modules/auction/redis/auction-redis.service.ts');
|
||||
const {
|
||||
AuctionRedisCommandService,
|
||||
} = require('../../src/modules/auction/redis/auction-redis-command.service.ts');
|
||||
const {
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
} = require('../../src/modules/auction/redis/auction-redis.keys.ts');
|
||||
const {
|
||||
MatchSchema,
|
||||
AuctionSchema,
|
||||
AuctionHistoryEntrySchema,
|
||||
AuctionSideEffectApplicationSchema,
|
||||
} = require('../../src/modules/global/database/schema.ts');
|
||||
const webRequire = createRequire(
|
||||
new URL('../../../web/package.json', import.meta.url),
|
||||
);
|
||||
const { io } = webRequire('socket.io-client');
|
||||
const { QueryClient, QueryObserver, environmentManager } = webRequire(
|
||||
'@tanstack/react-query',
|
||||
);
|
||||
const {
|
||||
createAuctionHeartbeatMonitor,
|
||||
shouldApplyPolledAuctionState,
|
||||
mergeAuctionState,
|
||||
} = await import('../../../web/src/api/auction-state.ts');
|
||||
const { createAuctionResyncRequester } =
|
||||
await import('../../../web/src/api/auction-resync.ts');
|
||||
const { AUCTION_SAFETY_POLL_INTERVAL_MS } =
|
||||
await import('../../../web/src/api/auction-polling.ts');
|
||||
|
||||
if (
|
||||
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379' ||
|
||||
!process.env.MONGO_TEST_URL?.startsWith(
|
||||
'mongodb://mongo:27017/auction_cluster?',
|
||||
)
|
||||
) {
|
||||
throw new Error('Only the isolated Compose services are allowed.');
|
||||
}
|
||||
const redis = createClient({ url: 'redis://redis:6379' });
|
||||
redis.on('error', () => {});
|
||||
await redis.connect();
|
||||
const cleanup = [() => (redis.isOpen ? redis.quit() : undefined)];
|
||||
const report = {
|
||||
startedAt: new Date().toISOString(),
|
||||
checks: [],
|
||||
requests: [],
|
||||
limitation:
|
||||
'Real API processes, Nginx, Redis TCP faults and Mongo transactions; fixture identity replaces the external login provider. Client recovery uses production helpers and QueryObserver, without rendering the React page.',
|
||||
};
|
||||
|
||||
async function until(check, description, timeout = 15_000) {
|
||||
const deadline = Date.now() + timeout;
|
||||
let lastError;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const value = await check();
|
||||
if (value) return value;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
await delay(50);
|
||||
}
|
||||
throw new Error(
|
||||
`Timed out: ${description}${lastError ? `: ${lastError.message}` : ''}`,
|
||||
);
|
||||
}
|
||||
async function http(url, body) {
|
||||
const started = Date.now();
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = request(
|
||||
url,
|
||||
{
|
||||
method: body === undefined ? 'GET' : 'POST',
|
||||
agent: false,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
},
|
||||
(res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
res.on('end', () => {
|
||||
report.requests.push({
|
||||
url,
|
||||
node: res.headers['x-cluster-node'],
|
||||
status: res.statusCode,
|
||||
durationMS: Date.now() - started,
|
||||
});
|
||||
if (res.statusCode < 200 || res.statusCode >= 300)
|
||||
return reject(new Error(`${url}: ${res.statusCode} ${data}`));
|
||||
try {
|
||||
resolve({
|
||||
body: JSON.parse(data),
|
||||
node: res.headers['x-cluster-node'],
|
||||
});
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
req.setTimeout(10_000, () =>
|
||||
req.destroy(new Error(`HTTP timeout: ${url}`)),
|
||||
);
|
||||
req.on('error', reject);
|
||||
req.end(body === undefined ? undefined : JSON.stringify(body));
|
||||
});
|
||||
}
|
||||
const pass = (name, details = {}) => {
|
||||
report.checks.push({ name, passed: true, ...details });
|
||||
console.log(`PASS ${name} ${JSON.stringify(details)}`);
|
||||
};
|
||||
|
||||
async function initialize() {
|
||||
const admin = new mongoose.mongo.MongoClient(
|
||||
'mongodb://mongo:27017/?directConnection=true',
|
||||
{ serverSelectionTimeoutMS: 1000 },
|
||||
);
|
||||
await until(
|
||||
async () => {
|
||||
await admin.connect();
|
||||
return true;
|
||||
},
|
||||
'Mongo startup',
|
||||
60_000,
|
||||
);
|
||||
await admin.db('admin').command({
|
||||
replSetInitiate: {
|
||||
_id: 'auction_cluster',
|
||||
members: [{ _id: 0, host: 'mongo:27017' }],
|
||||
},
|
||||
});
|
||||
await until(
|
||||
async () =>
|
||||
(await admin.db('admin').command({ hello: 1 })).isWritablePrimary,
|
||||
'Mongo primary',
|
||||
60_000,
|
||||
);
|
||||
const build = await admin.db('admin').command({ buildInfo: 1 });
|
||||
await admin.close();
|
||||
const connection = await mongoose
|
||||
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
|
||||
.asPromise();
|
||||
try {
|
||||
const models = {
|
||||
match: connection.model('match', MatchSchema),
|
||||
auction: connection.model('auction', AuctionSchema),
|
||||
history: connection.model('auctionHistory', AuctionHistoryEntrySchema),
|
||||
sideEffects: connection.model(
|
||||
'auctionSideEffectApplication',
|
||||
AuctionSideEffectApplicationSchema,
|
||||
),
|
||||
};
|
||||
await Promise.all(
|
||||
Object.values(models).map((model) => model.createCollection()),
|
||||
);
|
||||
await models.history.createIndexes();
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId(),
|
||||
auctionId = new mongoose.Types.ObjectId();
|
||||
const lifecycle = `cluster:${matchId}`;
|
||||
const playerA = new mongoose.Types.ObjectId(),
|
||||
playerB = new mongoose.Types.ObjectId(),
|
||||
captain = new mongoose.Types.ObjectId();
|
||||
const structure = {
|
||||
auctionPlayerOrder: [
|
||||
{ userId: playerA.toString() },
|
||||
{ userId: playerB.toString() },
|
||||
],
|
||||
auctionTeams: [
|
||||
{
|
||||
teamIndex: 1,
|
||||
captainUserId: captain.toString(),
|
||||
remainingBidPointAmount: 1000,
|
||||
members: [],
|
||||
},
|
||||
],
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentAuctionRound: 1,
|
||||
passedAuctionPlayerQueue: [],
|
||||
maximumBidRoundDurationSeconds: 600,
|
||||
bidRoundExtensionSeconds: 5,
|
||||
initialTeamBidPointAmount: 1000,
|
||||
topTierAuctionPriorityThreshold: 'NONE',
|
||||
manualTopTierAuctionPriorityUserIds: [],
|
||||
};
|
||||
await models.match.collection.insertOne({
|
||||
_id: matchId,
|
||||
communityId,
|
||||
title: 'Isolated cluster test',
|
||||
participants: [],
|
||||
teams: [],
|
||||
status: 'team-building',
|
||||
scheduledAt: new Date(),
|
||||
});
|
||||
await models.auction.collection.insertOne({
|
||||
_id: auctionId,
|
||||
matchId,
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: lifecycle,
|
||||
phase: 'presenting',
|
||||
revision: 0,
|
||||
currentAuctionHistorySequence: 0,
|
||||
chatMessages: [],
|
||||
...structure,
|
||||
});
|
||||
const commands = new AuctionRedisCommandService(
|
||||
new AuctionRedisService(redis),
|
||||
);
|
||||
await commands.hydrateState(matchId.toString(), {
|
||||
matchId: matchId.toString(),
|
||||
auctionLifecycleId: lifecycle,
|
||||
phase: 'presenting',
|
||||
revision: 0,
|
||||
auctionHistorySequence: 0,
|
||||
maximumBidRoundDurationSeconds: 600,
|
||||
structureStateJson: JSON.stringify(structure),
|
||||
liveStatePatch: {
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentBidRoundPlayerUserId: playerA.toString(),
|
||||
currentBidRoundId: 'initial',
|
||||
currentHighestBidPointAmount: 0,
|
||||
currentHighestBidderTeamIndex: -1,
|
||||
maximumBidRoundDurationSeconds: 600,
|
||||
bidRoundExtensionSeconds: 5,
|
||||
'team:1:captainUserId': captain.toString(),
|
||||
'team:1:remainingBidPointAmount': 1000,
|
||||
'team:1:assignedAuctionTeamMemberCount': 1,
|
||||
},
|
||||
});
|
||||
const fixture = {
|
||||
matchId: matchId.toString(),
|
||||
communityId: communityId.toString(),
|
||||
auctionId: auctionId.toString(),
|
||||
lifecycle,
|
||||
mongoVersion: build.version,
|
||||
redisVersion: (await redis.info('server')).match(
|
||||
/redis_version:([^\r\n]+)/,
|
||||
)?.[1],
|
||||
};
|
||||
await writeFile('/results/fixture.json', JSON.stringify(fixture, null, 2));
|
||||
console.log(
|
||||
`Initialized isolated MongoDB ${build.version} and Redis ${fixture.redisVersion}`,
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function verify() {
|
||||
const fixture = JSON.parse(await readFile('/results/fixture.json', 'utf8'));
|
||||
report.versions = {
|
||||
node: process.version,
|
||||
mongo: fixture.mongoVersion,
|
||||
redis: fixture.redisVersion,
|
||||
};
|
||||
const route = `/communities/${fixture.communityId}/matches/${fixture.matchId}/auction`;
|
||||
const url = `http://lb:8080${route}`;
|
||||
await until(
|
||||
async () => (await http('http://api-a:3000/_cluster/health')).body.node,
|
||||
'API A startup',
|
||||
60_000,
|
||||
);
|
||||
await until(
|
||||
async () => (await http('http://api-b:3000/_cluster/health')).body.node,
|
||||
'API B startup',
|
||||
60_000,
|
||||
);
|
||||
const routedNodes = new Set();
|
||||
for (let i = 0; i < 4; i += 1) routedNodes.add((await http(url)).node);
|
||||
assert.deepEqual([...routedNodes].sort(), ['api-a', 'api-b']);
|
||||
pass('Nginx routes real HTTP requests to both API processes');
|
||||
|
||||
async function connectViewer() {
|
||||
const socket = io('http://lb:8080/auction', {
|
||||
transports: ['websocket'],
|
||||
autoConnect: false,
|
||||
reconnection: false,
|
||||
});
|
||||
const viewer = {
|
||||
socket,
|
||||
state: null,
|
||||
states: [],
|
||||
histories: [],
|
||||
errors: [],
|
||||
};
|
||||
socket.on('state', (state) => {
|
||||
viewer.state = state;
|
||||
viewer.states.push(state);
|
||||
});
|
||||
socket.on('error', (error) => viewer.errors.push(error));
|
||||
socket.on('connect_error', (error) => viewer.errors.push(error.message));
|
||||
socket.on('history-updated', (event) => {
|
||||
void http(`${url}/history`)
|
||||
.then(({ body, node }) => viewer.histories.push({ event, body, node }))
|
||||
.catch((error) => viewer.errors.push(error.message));
|
||||
});
|
||||
cleanup.push(() => socket.disconnect());
|
||||
socket.connect();
|
||||
await until(
|
||||
() => socket.connected,
|
||||
`socket connect: ${viewer.errors.join(',')}`,
|
||||
);
|
||||
const ack = await socket
|
||||
.timeout(5000)
|
||||
.emitWithAck('join', { matchId: fixture.matchId });
|
||||
assert.equal(ack.ok, true);
|
||||
assert.equal(viewer.state.matchId, fixture.matchId);
|
||||
assert.equal(viewer.state.auctionEngineVersion, 'redis-v1');
|
||||
const health = await Promise.all(
|
||||
['api-a', 'api-b'].map((node) =>
|
||||
http(`http://${node}:3000/_cluster/health`),
|
||||
),
|
||||
);
|
||||
viewer.node = health.find((entry) =>
|
||||
entry.body.sockets.includes(socket.id),
|
||||
).body.node;
|
||||
return viewer;
|
||||
}
|
||||
const one = await connectViewer(),
|
||||
two = await connectViewer();
|
||||
assert.notEqual(one.node, two.node);
|
||||
const a = one.node === 'api-a' ? one : two;
|
||||
const b = one.node === 'api-b' ? one : two;
|
||||
const initialRevision = a.state.revision;
|
||||
const revisionFor = (sequence) => initialRevision + sequence;
|
||||
assert.equal(b.state.revision, initialRevision);
|
||||
pass('WebSocket viewers join different nodes through Nginx', {
|
||||
nodes: [one.node, two.node],
|
||||
});
|
||||
|
||||
const sequences = async () =>
|
||||
(await http(`${url}/history`)).body.map((entry) => entry.sequence);
|
||||
const action = async (viewer, name) => {
|
||||
const ack = await viewer.socket
|
||||
.timeout(5000)
|
||||
.emitWithAck(name, { auctionActionRequestId: randomUUID() });
|
||||
assert.equal(ack.ok, true, JSON.stringify(ack));
|
||||
};
|
||||
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
|
||||
await action(a, 'open-bidding');
|
||||
await until(
|
||||
() =>
|
||||
a.state.revision === revisionFor(1) &&
|
||||
b.state.revision === revisionFor(1),
|
||||
'cross-node full state',
|
||||
);
|
||||
assert.equal(a.histories.length + b.histories.length, 0);
|
||||
await http('http://api-a:3000/_cluster/hold-commit', {});
|
||||
const pendingDrain = http('http://api-a:3000/_cluster/drain', {});
|
||||
await until(
|
||||
async () =>
|
||||
(await http('http://api-a:3000/_cluster/health')).body.commitsWaiting ===
|
||||
1,
|
||||
'real Mongo commit barrier',
|
||||
);
|
||||
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
|
||||
assert.equal(a.histories.length + b.histories.length, 0);
|
||||
await http('http://api-a:3000/_cluster/release-commit', {});
|
||||
assert.equal((await pendingDrain).body.applied, 1);
|
||||
await until(
|
||||
() => a.histories.length === 1 && b.histories.length === 1,
|
||||
'post-commit history event on both nodes',
|
||||
);
|
||||
for (const viewer of [a, b])
|
||||
assert.deepEqual(
|
||||
viewer.histories[0].body.map((entry) => entry.sequence),
|
||||
[1],
|
||||
);
|
||||
pass(
|
||||
'History notification crosses nodes only after a real Mongo transaction commits',
|
||||
);
|
||||
|
||||
await action(a, 'passed');
|
||||
await action(b, 'next');
|
||||
await until(
|
||||
() =>
|
||||
a.state.revision === revisionFor(3) &&
|
||||
b.state.revision === revisionFor(3),
|
||||
'state fanout in both directions',
|
||||
);
|
||||
const keys = await redis.zRange(AUCTION_DURABILITY_RETRY_QUEUE_KEY, 0, -1);
|
||||
const queued = await Promise.all(
|
||||
keys.map(async (key) => ({
|
||||
key,
|
||||
payload: JSON.parse(await redis.hGet(key, 'payloadJson')),
|
||||
})),
|
||||
);
|
||||
const third = queued.find(
|
||||
({ payload }) => payload.auctionHistorySequence === 3,
|
||||
);
|
||||
const second = queued.find(
|
||||
({ payload }) => payload.auctionHistorySequence === 2,
|
||||
);
|
||||
assert.ok(third && second);
|
||||
await http('http://api-b:3000/_cluster/apply', third.payload);
|
||||
assert.deepEqual(await sequences(), [3, 1]);
|
||||
const previousEvents = b.histories.length;
|
||||
await http('http://api-a:3000/_cluster/apply', second.payload);
|
||||
await until(
|
||||
() =>
|
||||
b.histories.length > previousEvents &&
|
||||
b.histories.at(-1).body.some((entry) => entry.sequence === 2),
|
||||
'late append history refresh',
|
||||
);
|
||||
assert.deepEqual(await sequences(), [3, 2, 1]);
|
||||
for (const item of queued) {
|
||||
await redis.zRem(AUCTION_DURABILITY_RETRY_QUEUE_KEY, item.key);
|
||||
await redis.del(item.key);
|
||||
}
|
||||
pass(
|
||||
'A delayed history append refreshes another node with the same maximum sequence',
|
||||
{ maximumSequence: 3 },
|
||||
);
|
||||
|
||||
const undo = await http(`${url}/undo`, {
|
||||
sequence: 1,
|
||||
auctionActionRequestId: randomUUID(),
|
||||
});
|
||||
assert.equal(undo.body.auctionEngineVersion, 'redis-v1');
|
||||
assert.equal(undo.body.revision, revisionFor(4));
|
||||
assert.deepEqual(await sequences(), [3, 2, 1]);
|
||||
await http('http://api-b:3000/_cluster/drain', {});
|
||||
await until(
|
||||
async () => JSON.stringify(await sequences()) === '[4,1]',
|
||||
'durable Undo history',
|
||||
);
|
||||
await until(
|
||||
() => b.histories.at(-1)?.body[0]?.sequence === 4,
|
||||
'Undo history event',
|
||||
);
|
||||
pass('Undo increases sequence and replaces history on every node', {
|
||||
sequence: 4,
|
||||
history: [4, 1],
|
||||
});
|
||||
|
||||
await until(
|
||||
async () =>
|
||||
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
|
||||
'Redis subscriptions',
|
||||
);
|
||||
await http('http://redis-proxy:8081/block', {});
|
||||
console.log(
|
||||
'FAULT Pub/Sub TCP connections blocked; Redis commands and HTTP stay available',
|
||||
);
|
||||
await delay(500);
|
||||
assert.equal(
|
||||
(await http('http://api-a:3000/_cluster/health')).body.node,
|
||||
'api-a',
|
||||
);
|
||||
assert.equal(
|
||||
(await http('http://api-b:3000/_cluster/health')).body.node,
|
||||
'api-b',
|
||||
);
|
||||
assert.equal(a.socket.connected && b.socket.connected, true);
|
||||
const timestamp = await b.socket
|
||||
.timeout(3000)
|
||||
.emitWithAck('time-sync', { sequence: 1 });
|
||||
assert.equal(timestamp.ok, true);
|
||||
pass(
|
||||
'Both API processes and socket clock sync survive Pub/Sub-only TCP disconnection',
|
||||
);
|
||||
|
||||
environmentManager.setIsServer(() => false);
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: { queries: { retry: false, gcTime: Infinity } },
|
||||
});
|
||||
cleanup.push(() => queryClient.clear());
|
||||
let stateByHttp = b.state,
|
||||
historyByHttp,
|
||||
statePolls = 0,
|
||||
historyPolls = 0;
|
||||
const pollingStarted = Date.now();
|
||||
const stateObserver = new QueryObserver(queryClient, {
|
||||
queryKey: ['auction', fixture.matchId],
|
||||
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
|
||||
queryFn: async () => {
|
||||
statePolls += 1;
|
||||
return (await http(url)).body;
|
||||
},
|
||||
});
|
||||
const historyObserver = new QueryObserver(queryClient, {
|
||||
queryKey: ['auction-history', fixture.matchId],
|
||||
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
|
||||
queryFn: async () => {
|
||||
historyPolls += 1;
|
||||
return (await http(`${url}/history`)).body;
|
||||
},
|
||||
});
|
||||
cleanup.push(
|
||||
stateObserver.subscribe((result) => {
|
||||
if (
|
||||
result.data &&
|
||||
shouldApplyPolledAuctionState(stateByHttp, result.data, fixture.matchId)
|
||||
)
|
||||
stateByHttp = mergeAuctionState(stateByHttp, result.data);
|
||||
}),
|
||||
);
|
||||
cleanup.push(
|
||||
historyObserver.subscribe((result) => {
|
||||
if (result.data) historyByHttp = result.data;
|
||||
}),
|
||||
);
|
||||
await until(
|
||||
() =>
|
||||
stateByHttp.revision === revisionFor(4) &&
|
||||
historyByHttp?.[0]?.sequence === 4,
|
||||
'baseline HTTP reads',
|
||||
);
|
||||
const beforeHistoryEvents = b.histories.length;
|
||||
await action(a, 'open-bidding');
|
||||
await http('http://api-a:3000/_cluster/drain', {});
|
||||
await delay(250);
|
||||
assert.equal(b.state.revision, revisionFor(4));
|
||||
assert.equal(b.histories.length, beforeHistoryEvents);
|
||||
assert.equal(a.state.revision, revisionFor(5));
|
||||
// Do not run heartbeat recovery yet: verify the independent 30-second safety
|
||||
// route even if the client identity checker fails to notice a missed event.
|
||||
console.log(
|
||||
'WAIT Real 30-second HTTP safety polling with Pub/Sub still blocked',
|
||||
);
|
||||
await until(
|
||||
() =>
|
||||
stateByHttp.revision === revisionFor(5) &&
|
||||
historyByHttp?.[0]?.sequence === 5,
|
||||
'30-second HTTP safety fallback',
|
||||
35_000,
|
||||
);
|
||||
assert.ok(Date.now() - pollingStarted >= 29_000);
|
||||
pass(
|
||||
'Real 30-second HTTP polls repair missed state and history through Nginx',
|
||||
{ elapsedMS: Date.now() - pollingStarted, statePolls, historyPolls },
|
||||
);
|
||||
|
||||
const monitor = createAuctionHeartbeatMonitor();
|
||||
const heartbeat = async (sequence) => {
|
||||
const ack = await b.socket
|
||||
.timeout(3000)
|
||||
.emitWithAck('time-sync', { sequence });
|
||||
assert.equal(ack.ok, true);
|
||||
return monitor.inspect(b.state, ack.auctionState);
|
||||
};
|
||||
assert.equal((await heartbeat(2)).needsResync, false);
|
||||
await delay(10_000);
|
||||
assert.equal((await heartbeat(3)).needsResync, true);
|
||||
let resyncs = 0;
|
||||
const requester = createAuctionResyncRequester(
|
||||
async () => {
|
||||
resyncs += 1;
|
||||
return b.socket.timeout(3000).emitWithAck('resync', {});
|
||||
},
|
||||
(state) => {
|
||||
b.state = state;
|
||||
},
|
||||
);
|
||||
await requester.request(revisionFor(5), fixture.lifecycle);
|
||||
assert.equal(b.state.revision, revisionFor(5));
|
||||
assert.equal(resyncs, 1);
|
||||
pass(
|
||||
'Two real heartbeats detect a persistent gap and private socket resync works during Pub/Sub outage',
|
||||
{ resyncs },
|
||||
);
|
||||
|
||||
await http('http://redis-proxy:8081/restore', {});
|
||||
await until(
|
||||
async () =>
|
||||
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
|
||||
'Pub/Sub resubscription',
|
||||
20_000,
|
||||
);
|
||||
await action(a, 'passed');
|
||||
await http('http://api-a:3000/_cluster/drain', {});
|
||||
await until(
|
||||
() =>
|
||||
b.state.revision === revisionFor(6) &&
|
||||
b.histories.at(-1)?.body[0]?.sequence === 6,
|
||||
'events after Pub/Sub restoration',
|
||||
);
|
||||
pass(
|
||||
'Cross-node state and history events recover after restoring the TCP path',
|
||||
);
|
||||
const fault = (await http('http://redis-proxy:8081/status')).body;
|
||||
report.fault = fault;
|
||||
const previousId = b.socket.id;
|
||||
b.socket.disconnect();
|
||||
b.socket.connect();
|
||||
await until(
|
||||
() => b.socket.connected && b.socket.id !== previousId,
|
||||
'socket reconnect through Nginx',
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await b.socket
|
||||
.timeout(5000)
|
||||
.emitWithAck('join', { matchId: fixture.matchId })
|
||||
).ok,
|
||||
true,
|
||||
);
|
||||
await until(
|
||||
() => b.state.revision === revisionFor(6),
|
||||
'reconnect full state',
|
||||
);
|
||||
pass('A reconnected viewer receives the latest snapshot through Nginx');
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.argv[2] === 'init') await initialize();
|
||||
else {
|
||||
await verify();
|
||||
report.passed = true;
|
||||
}
|
||||
} catch (error) {
|
||||
report.passed = false;
|
||||
report.failure = { message: error.message, stack: error.stack };
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
if (process.argv[2] !== 'init') {
|
||||
await http('http://redis-proxy:8081/restore', {}).catch(() => {});
|
||||
report.finishedAt = new Date().toISOString();
|
||||
await writeFile('/results/report.json', JSON.stringify(report, null, 2));
|
||||
}
|
||||
for (const close of cleanup.reverse()) {
|
||||
try {
|
||||
await close();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
# From the repository root: sh apps/api/test/auction-cluster/run.sh
|
||||
# Requires Docker Compose. No host ports or production credentials are used.
|
||||
# Results and container logs remain in this directory's ignored results/ folder.
|
||||
# Authentication uses a fixture admin; client recovery uses the production
|
||||
# helpers and QueryObserver without rendering React. MongoDB 7 is the default
|
||||
# for the test host's kernel; set MONGO_TEST_IMAGE on a compatible host to match
|
||||
# the production database version.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-lolympus-auction-cluster-$(date +%s)}"
|
||||
case "$COMPOSE_PROJECT_NAME" in
|
||||
lolympus-auction-cluster-*) ;;
|
||||
*) echo 'Refusing to use a Compose project outside the auction-cluster test prefix.' >&2; exit 1 ;;
|
||||
esac
|
||||
mkdir -p results
|
||||
cleanup() {
|
||||
docker compose logs --no-color api-a api-b redis-proxy > results/containers.log 2>&1 || true
|
||||
docker compose down --volumes --remove-orphans
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
docker compose build runner
|
||||
docker compose up -d redis mongo redis-proxy
|
||||
docker compose run --rm runner node test/auction-cluster/run.mjs init
|
||||
docker compose up -d api-a api-b lb
|
||||
docker compose run --rm runner node test/auction-cluster/run.mjs verify
|
||||
Reference in new issue
Block a user