test(auction): add two-node Docker failure integration suite
Exercise Nginx routing, Mongo transaction ordering, history backfills, Undo, Pub/Sub TCP loss, safety polling, resync, and reconnection. Keep fixtures and results isolated from production and git.
This commit is contained in:
1 parent
23a0a3f8dd
commit
8d741ef9a7
9 files changed
+1024
No files matched your search
@@ -0,0 +1 @@
|
|||||||
|
* text eol=lf
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
results/
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
FROM node:26-bookworm-slim
|
||||||
|
RUN npm install --global pnpm@10.4.1
|
||||||
|
WORKDIR /workspace
|
||||||
|
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||||
|
COPY packages/contracts ./packages/contracts
|
||||||
|
COPY apps/api/package.json ./apps/api/package.json
|
||||||
|
COPY apps/web/package.json ./apps/web/package.json
|
||||||
|
RUN pnpm install --frozen-lockfile --filter api --filter web --ignore-scripts
|
||||||
|
COPY apps/api ./apps/api
|
||||||
|
COPY apps/web/src/api ./apps/web/src/api
|
||||||
|
WORKDIR /workspace/apps/api
|
||||||
|
ENV NODE_ENV=test NEW_RELIC_ENABLED=false
|
||||||
|
CMD ["node", "-r", "ts-node/register/transpile-only", "test/auction-cluster/node.ts"]
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
x-app: &app
|
||||||
|
image: ${COMPOSE_PROJECT_NAME}-app
|
||||||
|
build:
|
||||||
|
context: ../../../..
|
||||||
|
dockerfile: apps/api/test/auction-cluster/Dockerfile
|
||||||
|
environment: &environment
|
||||||
|
MONGO_TEST_URL: mongodb://mongo:27017/auction_cluster?replicaSet=auction_cluster
|
||||||
|
REDIS_TEST_URL: redis://redis-proxy:6379
|
||||||
|
NODE_ENV: test
|
||||||
|
NEW_RELIC_ENABLED: 'false'
|
||||||
|
volumes:
|
||||||
|
- ./results:/results
|
||||||
|
networks: [cluster]
|
||||||
|
mem_limit: 1g
|
||||||
|
|
||||||
|
services:
|
||||||
|
redis:
|
||||||
|
image: redis:8-alpine
|
||||||
|
command: [redis-server, --save, '', --appendonly, 'no']
|
||||||
|
networks: [cluster]
|
||||||
|
mem_limit: 256m
|
||||||
|
mongo:
|
||||||
|
# MongoDB 8+ refuses the test host's Linux 7.0.0 kernel (SERVER-121912).
|
||||||
|
image: ${MONGO_TEST_IMAGE:-mongo:7.0}
|
||||||
|
command:
|
||||||
|
[
|
||||||
|
mongod,
|
||||||
|
--replSet,
|
||||||
|
auction_cluster,
|
||||||
|
--bind_ip_all,
|
||||||
|
--wiredTigerCacheSizeGB,
|
||||||
|
'0.25',
|
||||||
|
]
|
||||||
|
networks: [cluster]
|
||||||
|
mem_limit: 768m
|
||||||
|
redis-proxy:
|
||||||
|
<<: *app
|
||||||
|
command: [node, test/auction-cluster/fault-proxy.mjs]
|
||||||
|
mem_limit: 128m
|
||||||
|
api-a:
|
||||||
|
<<: *app
|
||||||
|
environment:
|
||||||
|
<<: *environment
|
||||||
|
CLUSTER_NODE: api-a
|
||||||
|
api-b:
|
||||||
|
<<: *app
|
||||||
|
environment:
|
||||||
|
<<: *environment
|
||||||
|
CLUSTER_NODE: api-b
|
||||||
|
lb:
|
||||||
|
image: nginx:stable-alpine
|
||||||
|
volumes:
|
||||||
|
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
networks: [cluster]
|
||||||
|
mem_limit: 128m
|
||||||
|
runner:
|
||||||
|
<<: *app
|
||||||
|
command: [node, test/auction-cluster/run.mjs, verify]
|
||||||
|
|
||||||
|
networks:
|
||||||
|
cluster: {}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
import { createServer as createTcpServer, connect } from 'node:net';
|
||||||
|
import { createServer as createHttpServer } from 'node:http';
|
||||||
|
|
||||||
|
// Real TCP connections; only connections issuing SUBSCRIBE/PSUBSCRIBE are cut.
|
||||||
|
// Redis commands, TIME and PUBLISH continue through the same proxy.
|
||||||
|
let blocked = false;
|
||||||
|
let dropped = 0;
|
||||||
|
const connections = new Set();
|
||||||
|
createTcpServer((socket) => {
|
||||||
|
const upstream = connect(6379, 'redis');
|
||||||
|
const connection = { socket, upstream, subscriber: false, prefix: '' };
|
||||||
|
connections.add(connection);
|
||||||
|
const close = () => {
|
||||||
|
socket.destroy();
|
||||||
|
upstream.destroy();
|
||||||
|
connections.delete(connection);
|
||||||
|
};
|
||||||
|
socket.on('error', close);
|
||||||
|
upstream.on('error', close);
|
||||||
|
socket.on('close', close);
|
||||||
|
upstream.on('close', close);
|
||||||
|
socket.on('data', (chunk) => {
|
||||||
|
connection.prefix = (connection.prefix + chunk.toString('utf8')).slice(
|
||||||
|
-4096,
|
||||||
|
);
|
||||||
|
if (/\r\n(?:p?subscribe|ssubscribe)\r\n/i.test(connection.prefix)) {
|
||||||
|
connection.subscriber = true;
|
||||||
|
}
|
||||||
|
if (blocked && connection.subscriber) {
|
||||||
|
dropped += 1;
|
||||||
|
close();
|
||||||
|
} else upstream.write(chunk);
|
||||||
|
});
|
||||||
|
upstream.on('data', (chunk) => socket.write(chunk));
|
||||||
|
}).listen(6379, '0.0.0.0');
|
||||||
|
|
||||||
|
createHttpServer((request, response) => {
|
||||||
|
if (request.method === 'POST' && request.url === '/block') {
|
||||||
|
blocked = true;
|
||||||
|
for (const connection of connections) {
|
||||||
|
if (!connection.subscriber) continue;
|
||||||
|
dropped += 1;
|
||||||
|
connection.socket.destroy();
|
||||||
|
connection.upstream.destroy();
|
||||||
|
}
|
||||||
|
} else if (request.method === 'POST' && request.url === '/restore') {
|
||||||
|
blocked = false;
|
||||||
|
} else if (request.url !== '/status') {
|
||||||
|
response.writeHead(404).end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.setHeader('content-type', 'application/json');
|
||||||
|
response.end(
|
||||||
|
JSON.stringify({
|
||||||
|
blocked,
|
||||||
|
dropped,
|
||||||
|
subscribers: [...connections].filter((c) => c.subscriber).length,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}).listen(8081, '0.0.0.0');
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
events {}
|
||||||
|
http {
|
||||||
|
upstream auction {
|
||||||
|
server api-a:3000;
|
||||||
|
server api-b:3000;
|
||||||
|
}
|
||||||
|
server {
|
||||||
|
listen 8080;
|
||||||
|
location / {
|
||||||
|
proxy_pass http://auction;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
proxy_read_timeout 120s;
|
||||||
|
add_header X-Cluster-Upstream $upstream_addr always;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
import 'reflect-metadata';
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { Body, Controller, Get, Module, Post } from '@nestjs/common';
|
||||||
|
import { NestFactory } from '@nestjs/core';
|
||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { createClient } from 'redis';
|
||||||
|
import type { Namespace } from 'socket.io';
|
||||||
|
import { RedisIoAdapter } from '../../src/redis-io.adapter';
|
||||||
|
import { AuctionController } from '../../src/modules/auction/auction.controller';
|
||||||
|
import { AuctionGateway } from '../../src/modules/auction/auction.gateway';
|
||||||
|
import { AuctionEventBusService } from '../../src/modules/auction/auction-event-bus.service';
|
||||||
|
import { AuctionService } from '../../src/modules/auction/auction.service';
|
||||||
|
import { AuctionBidArchiveService } from '../../src/modules/auction/auction-bid-archive.service';
|
||||||
|
import {
|
||||||
|
AuctionDurabilityScheduler,
|
||||||
|
type AuctionDurabilityPayload,
|
||||||
|
} from '../../src/modules/auction/auction-durability.scheduler';
|
||||||
|
import { AuctionRedisService } from '../../src/modules/auction/redis/auction-redis.service';
|
||||||
|
import { AuctionRedisCommandService } from '../../src/modules/auction/redis/auction-redis-command.service';
|
||||||
|
import { AuctionRedisStateReaderService } from '../../src/modules/auction/redis/auction-redis-state-reader.service';
|
||||||
|
import { AuthService } from '../../src/modules/auth/auth.service';
|
||||||
|
import { JwtService } from '../../src/modules/auth/jwt.service';
|
||||||
|
import { IdentityService } from '../../src/modules/auth/identity.service';
|
||||||
|
import { ConnectionService } from '../../src/modules/global/database/connection.service';
|
||||||
|
import {
|
||||||
|
AuctionSchema,
|
||||||
|
AuctionHistoryEntrySchema,
|
||||||
|
AuctionSideEffectApplicationSchema,
|
||||||
|
MatchSchema,
|
||||||
|
} from '../../src/modules/global/database/schema';
|
||||||
|
|
||||||
|
async function bootstrap() {
|
||||||
|
if (
|
||||||
|
!process.env.MONGO_TEST_URL?.startsWith(
|
||||||
|
'mongodb://mongo:27017/auction_cluster?',
|
||||||
|
) ||
|
||||||
|
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379'
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
'This test node only accepts the isolated Compose services.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const fixture = JSON.parse(
|
||||||
|
await readFile('/results/fixture.json', 'utf8'),
|
||||||
|
) as { communityId: string };
|
||||||
|
const connection = await mongoose
|
||||||
|
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
|
||||||
|
.asPromise();
|
||||||
|
const models = {
|
||||||
|
match: connection.model('match', MatchSchema),
|
||||||
|
auction: connection.model('auction', AuctionSchema),
|
||||||
|
auctionHistory: connection.model(
|
||||||
|
'auctionHistory',
|
||||||
|
AuctionHistoryEntrySchema,
|
||||||
|
),
|
||||||
|
auctionSideEffectApplication: connection.model(
|
||||||
|
'auctionSideEffectApplication',
|
||||||
|
AuctionSideEffectApplicationSchema,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
let releaseCommit: (() => void) | undefined;
|
||||||
|
let commitGate: Promise<void> | undefined;
|
||||||
|
let commitsWaiting = 0;
|
||||||
|
const database = {
|
||||||
|
getModel: async (name: keyof typeof models) => {
|
||||||
|
if (!models[name]) throw new Error(`Unexpected test model: ${name}`);
|
||||||
|
return models[name];
|
||||||
|
},
|
||||||
|
createSession: async () => {
|
||||||
|
const session = await connection.startSession();
|
||||||
|
const commit = session.commitTransaction.bind(session);
|
||||||
|
session.commitTransaction = async () => {
|
||||||
|
if (commitGate) {
|
||||||
|
commitsWaiting += 1;
|
||||||
|
await commitGate;
|
||||||
|
commitsWaiting -= 1;
|
||||||
|
}
|
||||||
|
await commit();
|
||||||
|
};
|
||||||
|
return session;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const client = createClient({ url: process.env.REDIS_TEST_URL });
|
||||||
|
client.on('error', (error) =>
|
||||||
|
console.error('command-client:', error.message),
|
||||||
|
);
|
||||||
|
await client.connect();
|
||||||
|
const redis = new AuctionRedisService(client);
|
||||||
|
const commands = new AuctionRedisCommandService(redis);
|
||||||
|
const reader = new AuctionRedisStateReaderService(redis);
|
||||||
|
const bus = new AuctionEventBusService();
|
||||||
|
const service = new AuctionService(
|
||||||
|
database as never,
|
||||||
|
{} as never,
|
||||||
|
{} as never,
|
||||||
|
{} as never,
|
||||||
|
redis,
|
||||||
|
commands,
|
||||||
|
reader,
|
||||||
|
bus,
|
||||||
|
);
|
||||||
|
// Explicit draining lets the test hold real Mongo commits and reorder durable
|
||||||
|
// payloads; no in-memory database or fake Redis adapter is involved.
|
||||||
|
const scheduler = new AuctionDurabilityScheduler(
|
||||||
|
database as never,
|
||||||
|
redis,
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
bus,
|
||||||
|
reader,
|
||||||
|
service,
|
||||||
|
);
|
||||||
|
|
||||||
|
@Controller('_cluster')
|
||||||
|
class ControlController {
|
||||||
|
@Get('health') health() {
|
||||||
|
const namespace = Reflect.get(gateway, 'server') as Namespace;
|
||||||
|
return {
|
||||||
|
node: process.env.CLUSTER_NODE,
|
||||||
|
sockets: [...namespace.sockets.keys()],
|
||||||
|
pid: process.pid,
|
||||||
|
commitsWaiting,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
@Post('hold-commit') hold() {
|
||||||
|
if (commitGate) throw new Error('Commit already held');
|
||||||
|
commitGate = new Promise<void>((resolve) => {
|
||||||
|
releaseCommit = resolve;
|
||||||
|
});
|
||||||
|
return { held: true };
|
||||||
|
}
|
||||||
|
@Post('release-commit') release() {
|
||||||
|
releaseCommit?.();
|
||||||
|
commitGate = undefined;
|
||||||
|
releaseCommit = undefined;
|
||||||
|
return { released: true };
|
||||||
|
}
|
||||||
|
@Post('drain') drain() {
|
||||||
|
return scheduler.processDueDurabilityPayloads(100);
|
||||||
|
}
|
||||||
|
@Post('apply') async apply(@Body() payload: AuctionDurabilityPayload) {
|
||||||
|
await scheduler.applyDurabilityPayload(payload);
|
||||||
|
return { applied: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@Module({
|
||||||
|
controllers: [AuctionController, ControlController],
|
||||||
|
providers: [
|
||||||
|
AuctionGateway,
|
||||||
|
{ provide: AuctionService, useValue: service },
|
||||||
|
{ provide: ConnectionService, useValue: database },
|
||||||
|
{ provide: AuctionRedisService, useValue: redis },
|
||||||
|
{ provide: AuctionEventBusService, useValue: bus },
|
||||||
|
{ provide: AuctionBidArchiveService, useValue: {} },
|
||||||
|
{
|
||||||
|
provide: AuthService,
|
||||||
|
useValue: { resolveDevUserById: async () => null },
|
||||||
|
},
|
||||||
|
{ provide: JwtService, useValue: {} },
|
||||||
|
{ provide: IdentityService, useValue: {} },
|
||||||
|
],
|
||||||
|
})
|
||||||
|
class TestModule {}
|
||||||
|
const app = await NestFactory.create(TestModule, {
|
||||||
|
logger: ['error', 'warn'],
|
||||||
|
});
|
||||||
|
// Identity-provider authentication is outside this transport/durability test.
|
||||||
|
// The real controller's RolesGuard still checks this isolated fixture admin.
|
||||||
|
app.use(
|
||||||
|
(
|
||||||
|
request: { user?: unknown },
|
||||||
|
response: { setHeader(name: string, value: string): void },
|
||||||
|
next: () => void,
|
||||||
|
) => {
|
||||||
|
request.user = {
|
||||||
|
role: 'admin',
|
||||||
|
communityId: new mongoose.Types.ObjectId(fixture.communityId),
|
||||||
|
};
|
||||||
|
response.setHeader(
|
||||||
|
'X-Cluster-Node',
|
||||||
|
process.env.CLUSTER_NODE ?? 'unknown',
|
||||||
|
);
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const adapter = new RedisIoAdapter(app);
|
||||||
|
await adapter.connectToRedis();
|
||||||
|
app.useWebSocketAdapter(adapter);
|
||||||
|
const gateway = app.get(AuctionGateway);
|
||||||
|
await app.listen(3000, '0.0.0.0');
|
||||||
|
console.log(`cluster node ready: ${process.env.CLUSTER_NODE}`);
|
||||||
|
process.on('SIGTERM', () => {
|
||||||
|
void app
|
||||||
|
.close()
|
||||||
|
.then(() => connection.close())
|
||||||
|
.then(() => process.exit(0));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
void bootstrap().catch((error: unknown) => {
|
||||||
|
console.error(error);
|
||||||
|
process.exitCode = 1;
|
||||||
|
});
|
||||||
@@ -0,0 +1,641 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
|
import { readFile, writeFile } from 'node:fs/promises';
|
||||||
|
import { request } from 'node:http';
|
||||||
|
import { setTimeout as delay } from 'node:timers/promises';
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { createClient } from 'redis';
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
require('ts-node/register/transpile-only');
|
||||||
|
const {
|
||||||
|
AuctionRedisService,
|
||||||
|
} = require('../../src/modules/auction/redis/auction-redis.service.ts');
|
||||||
|
const {
|
||||||
|
AuctionRedisCommandService,
|
||||||
|
} = require('../../src/modules/auction/redis/auction-redis-command.service.ts');
|
||||||
|
const {
|
||||||
|
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||||
|
} = require('../../src/modules/auction/redis/auction-redis.keys.ts');
|
||||||
|
const {
|
||||||
|
MatchSchema,
|
||||||
|
AuctionSchema,
|
||||||
|
AuctionHistoryEntrySchema,
|
||||||
|
AuctionSideEffectApplicationSchema,
|
||||||
|
} = require('../../src/modules/global/database/schema.ts');
|
||||||
|
const webRequire = createRequire(
|
||||||
|
new URL('../../../web/package.json', import.meta.url),
|
||||||
|
);
|
||||||
|
const { io } = webRequire('socket.io-client');
|
||||||
|
const { QueryClient, QueryObserver, environmentManager } = webRequire(
|
||||||
|
'@tanstack/react-query',
|
||||||
|
);
|
||||||
|
const {
|
||||||
|
createAuctionHeartbeatMonitor,
|
||||||
|
shouldApplyPolledAuctionState,
|
||||||
|
mergeAuctionState,
|
||||||
|
} = await import('../../../web/src/api/auction-state.ts');
|
||||||
|
const { createAuctionResyncRequester } =
|
||||||
|
await import('../../../web/src/api/auction-resync.ts');
|
||||||
|
const { AUCTION_SAFETY_POLL_INTERVAL_MS } =
|
||||||
|
await import('../../../web/src/api/auction-polling.ts');
|
||||||
|
|
||||||
|
if (
|
||||||
|
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379' ||
|
||||||
|
!process.env.MONGO_TEST_URL?.startsWith(
|
||||||
|
'mongodb://mongo:27017/auction_cluster?',
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new Error('Only the isolated Compose services are allowed.');
|
||||||
|
}
|
||||||
|
const redis = createClient({ url: 'redis://redis:6379' });
|
||||||
|
redis.on('error', () => {});
|
||||||
|
await redis.connect();
|
||||||
|
const cleanup = [() => (redis.isOpen ? redis.quit() : undefined)];
|
||||||
|
const report = {
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
checks: [],
|
||||||
|
requests: [],
|
||||||
|
limitation:
|
||||||
|
'Real API processes, Nginx, Redis TCP faults and Mongo transactions; fixture identity replaces the external login provider. Client recovery uses production helpers and QueryObserver, without rendering the React page.',
|
||||||
|
};
|
||||||
|
|
||||||
|
async function until(check, description, timeout = 15_000) {
|
||||||
|
const deadline = Date.now() + timeout;
|
||||||
|
let lastError;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
try {
|
||||||
|
const value = await check();
|
||||||
|
if (value) return value;
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error;
|
||||||
|
}
|
||||||
|
await delay(50);
|
||||||
|
}
|
||||||
|
throw new Error(
|
||||||
|
`Timed out: ${description}${lastError ? `: ${lastError.message}` : ''}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
async function http(url, body) {
|
||||||
|
const started = Date.now();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const req = request(
|
||||||
|
url,
|
||||||
|
{
|
||||||
|
method: body === undefined ? 'GET' : 'POST',
|
||||||
|
agent: false,
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
},
|
||||||
|
(res) => {
|
||||||
|
let data = '';
|
||||||
|
res.on('data', (chunk) => {
|
||||||
|
data += chunk;
|
||||||
|
});
|
||||||
|
res.on('end', () => {
|
||||||
|
report.requests.push({
|
||||||
|
url,
|
||||||
|
node: res.headers['x-cluster-node'],
|
||||||
|
status: res.statusCode,
|
||||||
|
durationMS: Date.now() - started,
|
||||||
|
});
|
||||||
|
if (res.statusCode < 200 || res.statusCode >= 300)
|
||||||
|
return reject(new Error(`${url}: ${res.statusCode} ${data}`));
|
||||||
|
try {
|
||||||
|
resolve({
|
||||||
|
body: JSON.parse(data),
|
||||||
|
node: res.headers['x-cluster-node'],
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
req.setTimeout(10_000, () =>
|
||||||
|
req.destroy(new Error(`HTTP timeout: ${url}`)),
|
||||||
|
);
|
||||||
|
req.on('error', reject);
|
||||||
|
req.end(body === undefined ? undefined : JSON.stringify(body));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const pass = (name, details = {}) => {
|
||||||
|
report.checks.push({ name, passed: true, ...details });
|
||||||
|
console.log(`PASS ${name} ${JSON.stringify(details)}`);
|
||||||
|
};
|
||||||
|
|
||||||
|
async function initialize() {
|
||||||
|
const admin = new mongoose.mongo.MongoClient(
|
||||||
|
'mongodb://mongo:27017/?directConnection=true',
|
||||||
|
{ serverSelectionTimeoutMS: 1000 },
|
||||||
|
);
|
||||||
|
await until(
|
||||||
|
async () => {
|
||||||
|
await admin.connect();
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
'Mongo startup',
|
||||||
|
60_000,
|
||||||
|
);
|
||||||
|
await admin.db('admin').command({
|
||||||
|
replSetInitiate: {
|
||||||
|
_id: 'auction_cluster',
|
||||||
|
members: [{ _id: 0, host: 'mongo:27017' }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await until(
|
||||||
|
async () =>
|
||||||
|
(await admin.db('admin').command({ hello: 1 })).isWritablePrimary,
|
||||||
|
'Mongo primary',
|
||||||
|
60_000,
|
||||||
|
);
|
||||||
|
const build = await admin.db('admin').command({ buildInfo: 1 });
|
||||||
|
await admin.close();
|
||||||
|
const connection = await mongoose
|
||||||
|
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
|
||||||
|
.asPromise();
|
||||||
|
try {
|
||||||
|
const models = {
|
||||||
|
match: connection.model('match', MatchSchema),
|
||||||
|
auction: connection.model('auction', AuctionSchema),
|
||||||
|
history: connection.model('auctionHistory', AuctionHistoryEntrySchema),
|
||||||
|
sideEffects: connection.model(
|
||||||
|
'auctionSideEffectApplication',
|
||||||
|
AuctionSideEffectApplicationSchema,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
await Promise.all(
|
||||||
|
Object.values(models).map((model) => model.createCollection()),
|
||||||
|
);
|
||||||
|
await models.history.createIndexes();
|
||||||
|
const matchId = new mongoose.Types.ObjectId(),
|
||||||
|
communityId = new mongoose.Types.ObjectId(),
|
||||||
|
auctionId = new mongoose.Types.ObjectId();
|
||||||
|
const lifecycle = `cluster:${matchId}`;
|
||||||
|
const playerA = new mongoose.Types.ObjectId(),
|
||||||
|
playerB = new mongoose.Types.ObjectId(),
|
||||||
|
captain = new mongoose.Types.ObjectId();
|
||||||
|
const structure = {
|
||||||
|
auctionPlayerOrder: [
|
||||||
|
{ userId: playerA.toString() },
|
||||||
|
{ userId: playerB.toString() },
|
||||||
|
],
|
||||||
|
auctionTeams: [
|
||||||
|
{
|
||||||
|
teamIndex: 1,
|
||||||
|
captainUserId: captain.toString(),
|
||||||
|
remainingBidPointAmount: 1000,
|
||||||
|
members: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
currentAuctionPlayerOrderIndex: 0,
|
||||||
|
currentAuctionRound: 1,
|
||||||
|
passedAuctionPlayerQueue: [],
|
||||||
|
maximumBidRoundDurationSeconds: 600,
|
||||||
|
bidRoundExtensionSeconds: 5,
|
||||||
|
initialTeamBidPointAmount: 1000,
|
||||||
|
topTierAuctionPriorityThreshold: 'NONE',
|
||||||
|
manualTopTierAuctionPriorityUserIds: [],
|
||||||
|
};
|
||||||
|
await models.match.collection.insertOne({
|
||||||
|
_id: matchId,
|
||||||
|
communityId,
|
||||||
|
title: 'Isolated cluster test',
|
||||||
|
participants: [],
|
||||||
|
teams: [],
|
||||||
|
status: 'team-building',
|
||||||
|
scheduledAt: new Date(),
|
||||||
|
});
|
||||||
|
await models.auction.collection.insertOne({
|
||||||
|
_id: auctionId,
|
||||||
|
matchId,
|
||||||
|
auctionEngineVersion: 'redis-v1',
|
||||||
|
auctionLifecycleId: lifecycle,
|
||||||
|
phase: 'presenting',
|
||||||
|
revision: 0,
|
||||||
|
currentAuctionHistorySequence: 0,
|
||||||
|
chatMessages: [],
|
||||||
|
...structure,
|
||||||
|
});
|
||||||
|
const commands = new AuctionRedisCommandService(
|
||||||
|
new AuctionRedisService(redis),
|
||||||
|
);
|
||||||
|
await commands.hydrateState(matchId.toString(), {
|
||||||
|
matchId: matchId.toString(),
|
||||||
|
auctionLifecycleId: lifecycle,
|
||||||
|
phase: 'presenting',
|
||||||
|
revision: 0,
|
||||||
|
auctionHistorySequence: 0,
|
||||||
|
maximumBidRoundDurationSeconds: 600,
|
||||||
|
structureStateJson: JSON.stringify(structure),
|
||||||
|
liveStatePatch: {
|
||||||
|
currentAuctionPlayerOrderIndex: 0,
|
||||||
|
currentBidRoundPlayerUserId: playerA.toString(),
|
||||||
|
currentBidRoundId: 'initial',
|
||||||
|
currentHighestBidPointAmount: 0,
|
||||||
|
currentHighestBidderTeamIndex: -1,
|
||||||
|
maximumBidRoundDurationSeconds: 600,
|
||||||
|
bidRoundExtensionSeconds: 5,
|
||||||
|
'team:1:captainUserId': captain.toString(),
|
||||||
|
'team:1:remainingBidPointAmount': 1000,
|
||||||
|
'team:1:assignedAuctionTeamMemberCount': 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const fixture = {
|
||||||
|
matchId: matchId.toString(),
|
||||||
|
communityId: communityId.toString(),
|
||||||
|
auctionId: auctionId.toString(),
|
||||||
|
lifecycle,
|
||||||
|
mongoVersion: build.version,
|
||||||
|
redisVersion: (await redis.info('server')).match(
|
||||||
|
/redis_version:([^\r\n]+)/,
|
||||||
|
)?.[1],
|
||||||
|
};
|
||||||
|
await writeFile('/results/fixture.json', JSON.stringify(fixture, null, 2));
|
||||||
|
console.log(
|
||||||
|
`Initialized isolated MongoDB ${build.version} and Redis ${fixture.redisVersion}`,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await connection.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function verify() {
|
||||||
|
const fixture = JSON.parse(await readFile('/results/fixture.json', 'utf8'));
|
||||||
|
report.versions = {
|
||||||
|
node: process.version,
|
||||||
|
mongo: fixture.mongoVersion,
|
||||||
|
redis: fixture.redisVersion,
|
||||||
|
};
|
||||||
|
const route = `/communities/${fixture.communityId}/matches/${fixture.matchId}/auction`;
|
||||||
|
const url = `http://lb:8080${route}`;
|
||||||
|
await until(
|
||||||
|
async () => (await http('http://api-a:3000/_cluster/health')).body.node,
|
||||||
|
'API A startup',
|
||||||
|
60_000,
|
||||||
|
);
|
||||||
|
await until(
|
||||||
|
async () => (await http('http://api-b:3000/_cluster/health')).body.node,
|
||||||
|
'API B startup',
|
||||||
|
60_000,
|
||||||
|
);
|
||||||
|
const routedNodes = new Set();
|
||||||
|
for (let i = 0; i < 4; i += 1) routedNodes.add((await http(url)).node);
|
||||||
|
assert.deepEqual([...routedNodes].sort(), ['api-a', 'api-b']);
|
||||||
|
pass('Nginx routes real HTTP requests to both API processes');
|
||||||
|
|
||||||
|
async function connectViewer() {
|
||||||
|
const socket = io('http://lb:8080/auction', {
|
||||||
|
transports: ['websocket'],
|
||||||
|
autoConnect: false,
|
||||||
|
reconnection: false,
|
||||||
|
});
|
||||||
|
const viewer = {
|
||||||
|
socket,
|
||||||
|
state: null,
|
||||||
|
states: [],
|
||||||
|
histories: [],
|
||||||
|
errors: [],
|
||||||
|
};
|
||||||
|
socket.on('state', (state) => {
|
||||||
|
viewer.state = state;
|
||||||
|
viewer.states.push(state);
|
||||||
|
});
|
||||||
|
socket.on('error', (error) => viewer.errors.push(error));
|
||||||
|
socket.on('connect_error', (error) => viewer.errors.push(error.message));
|
||||||
|
socket.on('history-updated', (event) => {
|
||||||
|
void http(`${url}/history`)
|
||||||
|
.then(({ body, node }) => viewer.histories.push({ event, body, node }))
|
||||||
|
.catch((error) => viewer.errors.push(error.message));
|
||||||
|
});
|
||||||
|
cleanup.push(() => socket.disconnect());
|
||||||
|
socket.connect();
|
||||||
|
await until(
|
||||||
|
() => socket.connected,
|
||||||
|
`socket connect: ${viewer.errors.join(',')}`,
|
||||||
|
);
|
||||||
|
const ack = await socket
|
||||||
|
.timeout(5000)
|
||||||
|
.emitWithAck('join', { matchId: fixture.matchId });
|
||||||
|
assert.equal(ack.ok, true);
|
||||||
|
assert.equal(viewer.state.matchId, fixture.matchId);
|
||||||
|
assert.equal(viewer.state.auctionEngineVersion, 'redis-v1');
|
||||||
|
const health = await Promise.all(
|
||||||
|
['api-a', 'api-b'].map((node) =>
|
||||||
|
http(`http://${node}:3000/_cluster/health`),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
viewer.node = health.find((entry) =>
|
||||||
|
entry.body.sockets.includes(socket.id),
|
||||||
|
).body.node;
|
||||||
|
return viewer;
|
||||||
|
}
|
||||||
|
const one = await connectViewer(),
|
||||||
|
two = await connectViewer();
|
||||||
|
assert.notEqual(one.node, two.node);
|
||||||
|
const a = one.node === 'api-a' ? one : two;
|
||||||
|
const b = one.node === 'api-b' ? one : two;
|
||||||
|
const initialRevision = a.state.revision;
|
||||||
|
const revisionFor = (sequence) => initialRevision + sequence;
|
||||||
|
assert.equal(b.state.revision, initialRevision);
|
||||||
|
pass('WebSocket viewers join different nodes through Nginx', {
|
||||||
|
nodes: [one.node, two.node],
|
||||||
|
});
|
||||||
|
|
||||||
|
const sequences = async () =>
|
||||||
|
(await http(`${url}/history`)).body.map((entry) => entry.sequence);
|
||||||
|
const action = async (viewer, name) => {
|
||||||
|
const ack = await viewer.socket
|
||||||
|
.timeout(5000)
|
||||||
|
.emitWithAck(name, { auctionActionRequestId: randomUUID() });
|
||||||
|
assert.equal(ack.ok, true, JSON.stringify(ack));
|
||||||
|
};
|
||||||
|
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
|
||||||
|
await action(a, 'open-bidding');
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
a.state.revision === revisionFor(1) &&
|
||||||
|
b.state.revision === revisionFor(1),
|
||||||
|
'cross-node full state',
|
||||||
|
);
|
||||||
|
assert.equal(a.histories.length + b.histories.length, 0);
|
||||||
|
await http('http://api-a:3000/_cluster/hold-commit', {});
|
||||||
|
const pendingDrain = http('http://api-a:3000/_cluster/drain', {});
|
||||||
|
await until(
|
||||||
|
async () =>
|
||||||
|
(await http('http://api-a:3000/_cluster/health')).body.commitsWaiting ===
|
||||||
|
1,
|
||||||
|
'real Mongo commit barrier',
|
||||||
|
);
|
||||||
|
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
|
||||||
|
assert.equal(a.histories.length + b.histories.length, 0);
|
||||||
|
await http('http://api-a:3000/_cluster/release-commit', {});
|
||||||
|
assert.equal((await pendingDrain).body.applied, 1);
|
||||||
|
await until(
|
||||||
|
() => a.histories.length === 1 && b.histories.length === 1,
|
||||||
|
'post-commit history event on both nodes',
|
||||||
|
);
|
||||||
|
for (const viewer of [a, b])
|
||||||
|
assert.deepEqual(
|
||||||
|
viewer.histories[0].body.map((entry) => entry.sequence),
|
||||||
|
[1],
|
||||||
|
);
|
||||||
|
pass(
|
||||||
|
'History notification crosses nodes only after a real Mongo transaction commits',
|
||||||
|
);
|
||||||
|
|
||||||
|
await action(a, 'passed');
|
||||||
|
await action(b, 'next');
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
a.state.revision === revisionFor(3) &&
|
||||||
|
b.state.revision === revisionFor(3),
|
||||||
|
'state fanout in both directions',
|
||||||
|
);
|
||||||
|
const keys = await redis.zRange(AUCTION_DURABILITY_RETRY_QUEUE_KEY, 0, -1);
|
||||||
|
const queued = await Promise.all(
|
||||||
|
keys.map(async (key) => ({
|
||||||
|
key,
|
||||||
|
payload: JSON.parse(await redis.hGet(key, 'payloadJson')),
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
const third = queued.find(
|
||||||
|
({ payload }) => payload.auctionHistorySequence === 3,
|
||||||
|
);
|
||||||
|
const second = queued.find(
|
||||||
|
({ payload }) => payload.auctionHistorySequence === 2,
|
||||||
|
);
|
||||||
|
assert.ok(third && second);
|
||||||
|
await http('http://api-b:3000/_cluster/apply', third.payload);
|
||||||
|
assert.deepEqual(await sequences(), [3, 1]);
|
||||||
|
const previousEvents = b.histories.length;
|
||||||
|
await http('http://api-a:3000/_cluster/apply', second.payload);
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
b.histories.length > previousEvents &&
|
||||||
|
b.histories.at(-1).body.some((entry) => entry.sequence === 2),
|
||||||
|
'late append history refresh',
|
||||||
|
);
|
||||||
|
assert.deepEqual(await sequences(), [3, 2, 1]);
|
||||||
|
for (const item of queued) {
|
||||||
|
await redis.zRem(AUCTION_DURABILITY_RETRY_QUEUE_KEY, item.key);
|
||||||
|
await redis.del(item.key);
|
||||||
|
}
|
||||||
|
pass(
|
||||||
|
'A delayed history append refreshes another node with the same maximum sequence',
|
||||||
|
{ maximumSequence: 3 },
|
||||||
|
);
|
||||||
|
|
||||||
|
const undo = await http(`${url}/undo`, {
|
||||||
|
sequence: 1,
|
||||||
|
auctionActionRequestId: randomUUID(),
|
||||||
|
});
|
||||||
|
assert.equal(undo.body.auctionEngineVersion, 'redis-v1');
|
||||||
|
assert.equal(undo.body.revision, revisionFor(4));
|
||||||
|
assert.deepEqual(await sequences(), [3, 2, 1]);
|
||||||
|
await http('http://api-b:3000/_cluster/drain', {});
|
||||||
|
await until(
|
||||||
|
async () => JSON.stringify(await sequences()) === '[4,1]',
|
||||||
|
'durable Undo history',
|
||||||
|
);
|
||||||
|
await until(
|
||||||
|
() => b.histories.at(-1)?.body[0]?.sequence === 4,
|
||||||
|
'Undo history event',
|
||||||
|
);
|
||||||
|
pass('Undo increases sequence and replaces history on every node', {
|
||||||
|
sequence: 4,
|
||||||
|
history: [4, 1],
|
||||||
|
});
|
||||||
|
|
||||||
|
await until(
|
||||||
|
async () =>
|
||||||
|
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
|
||||||
|
'Redis subscriptions',
|
||||||
|
);
|
||||||
|
await http('http://redis-proxy:8081/block', {});
|
||||||
|
console.log(
|
||||||
|
'FAULT Pub/Sub TCP connections blocked; Redis commands and HTTP stay available',
|
||||||
|
);
|
||||||
|
await delay(500);
|
||||||
|
assert.equal(
|
||||||
|
(await http('http://api-a:3000/_cluster/health')).body.node,
|
||||||
|
'api-a',
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
(await http('http://api-b:3000/_cluster/health')).body.node,
|
||||||
|
'api-b',
|
||||||
|
);
|
||||||
|
assert.equal(a.socket.connected && b.socket.connected, true);
|
||||||
|
const timestamp = await b.socket
|
||||||
|
.timeout(3000)
|
||||||
|
.emitWithAck('time-sync', { sequence: 1 });
|
||||||
|
assert.equal(timestamp.ok, true);
|
||||||
|
pass(
|
||||||
|
'Both API processes and socket clock sync survive Pub/Sub-only TCP disconnection',
|
||||||
|
);
|
||||||
|
|
||||||
|
environmentManager.setIsServer(() => false);
|
||||||
|
const queryClient = new QueryClient({
|
||||||
|
defaultOptions: { queries: { retry: false, gcTime: Infinity } },
|
||||||
|
});
|
||||||
|
cleanup.push(() => queryClient.clear());
|
||||||
|
let stateByHttp = b.state,
|
||||||
|
historyByHttp,
|
||||||
|
statePolls = 0,
|
||||||
|
historyPolls = 0;
|
||||||
|
const pollingStarted = Date.now();
|
||||||
|
const stateObserver = new QueryObserver(queryClient, {
|
||||||
|
queryKey: ['auction', fixture.matchId],
|
||||||
|
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
|
||||||
|
queryFn: async () => {
|
||||||
|
statePolls += 1;
|
||||||
|
return (await http(url)).body;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const historyObserver = new QueryObserver(queryClient, {
|
||||||
|
queryKey: ['auction-history', fixture.matchId],
|
||||||
|
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
|
||||||
|
queryFn: async () => {
|
||||||
|
historyPolls += 1;
|
||||||
|
return (await http(`${url}/history`)).body;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
cleanup.push(
|
||||||
|
stateObserver.subscribe((result) => {
|
||||||
|
if (
|
||||||
|
result.data &&
|
||||||
|
shouldApplyPolledAuctionState(stateByHttp, result.data, fixture.matchId)
|
||||||
|
)
|
||||||
|
stateByHttp = mergeAuctionState(stateByHttp, result.data);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
cleanup.push(
|
||||||
|
historyObserver.subscribe((result) => {
|
||||||
|
if (result.data) historyByHttp = result.data;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
stateByHttp.revision === revisionFor(4) &&
|
||||||
|
historyByHttp?.[0]?.sequence === 4,
|
||||||
|
'baseline HTTP reads',
|
||||||
|
);
|
||||||
|
const beforeHistoryEvents = b.histories.length;
|
||||||
|
await action(a, 'open-bidding');
|
||||||
|
await http('http://api-a:3000/_cluster/drain', {});
|
||||||
|
await delay(250);
|
||||||
|
assert.equal(b.state.revision, revisionFor(4));
|
||||||
|
assert.equal(b.histories.length, beforeHistoryEvents);
|
||||||
|
assert.equal(a.state.revision, revisionFor(5));
|
||||||
|
// Do not run heartbeat recovery yet: verify the independent 30-second safety
|
||||||
|
// route even if the client identity checker fails to notice a missed event.
|
||||||
|
console.log(
|
||||||
|
'WAIT Real 30-second HTTP safety polling with Pub/Sub still blocked',
|
||||||
|
);
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
stateByHttp.revision === revisionFor(5) &&
|
||||||
|
historyByHttp?.[0]?.sequence === 5,
|
||||||
|
'30-second HTTP safety fallback',
|
||||||
|
35_000,
|
||||||
|
);
|
||||||
|
assert.ok(Date.now() - pollingStarted >= 29_000);
|
||||||
|
pass(
|
||||||
|
'Real 30-second HTTP polls repair missed state and history through Nginx',
|
||||||
|
{ elapsedMS: Date.now() - pollingStarted, statePolls, historyPolls },
|
||||||
|
);
|
||||||
|
|
||||||
|
const monitor = createAuctionHeartbeatMonitor();
|
||||||
|
const heartbeat = async (sequence) => {
|
||||||
|
const ack = await b.socket
|
||||||
|
.timeout(3000)
|
||||||
|
.emitWithAck('time-sync', { sequence });
|
||||||
|
assert.equal(ack.ok, true);
|
||||||
|
return monitor.inspect(b.state, ack.auctionState);
|
||||||
|
};
|
||||||
|
assert.equal((await heartbeat(2)).needsResync, false);
|
||||||
|
await delay(10_000);
|
||||||
|
assert.equal((await heartbeat(3)).needsResync, true);
|
||||||
|
let resyncs = 0;
|
||||||
|
const requester = createAuctionResyncRequester(
|
||||||
|
async () => {
|
||||||
|
resyncs += 1;
|
||||||
|
return b.socket.timeout(3000).emitWithAck('resync', {});
|
||||||
|
},
|
||||||
|
(state) => {
|
||||||
|
b.state = state;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
await requester.request(revisionFor(5), fixture.lifecycle);
|
||||||
|
assert.equal(b.state.revision, revisionFor(5));
|
||||||
|
assert.equal(resyncs, 1);
|
||||||
|
pass(
|
||||||
|
'Two real heartbeats detect a persistent gap and private socket resync works during Pub/Sub outage',
|
||||||
|
{ resyncs },
|
||||||
|
);
|
||||||
|
|
||||||
|
await http('http://redis-proxy:8081/restore', {});
|
||||||
|
await until(
|
||||||
|
async () =>
|
||||||
|
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
|
||||||
|
'Pub/Sub resubscription',
|
||||||
|
20_000,
|
||||||
|
);
|
||||||
|
await action(a, 'passed');
|
||||||
|
await http('http://api-a:3000/_cluster/drain', {});
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
b.state.revision === revisionFor(6) &&
|
||||||
|
b.histories.at(-1)?.body[0]?.sequence === 6,
|
||||||
|
'events after Pub/Sub restoration',
|
||||||
|
);
|
||||||
|
pass(
|
||||||
|
'Cross-node state and history events recover after restoring the TCP path',
|
||||||
|
);
|
||||||
|
const fault = (await http('http://redis-proxy:8081/status')).body;
|
||||||
|
report.fault = fault;
|
||||||
|
const previousId = b.socket.id;
|
||||||
|
b.socket.disconnect();
|
||||||
|
b.socket.connect();
|
||||||
|
await until(
|
||||||
|
() => b.socket.connected && b.socket.id !== previousId,
|
||||||
|
'socket reconnect through Nginx',
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
(
|
||||||
|
await b.socket
|
||||||
|
.timeout(5000)
|
||||||
|
.emitWithAck('join', { matchId: fixture.matchId })
|
||||||
|
).ok,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
await until(
|
||||||
|
() => b.state.revision === revisionFor(6),
|
||||||
|
'reconnect full state',
|
||||||
|
);
|
||||||
|
pass('A reconnected viewer receives the latest snapshot through Nginx');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (process.argv[2] === 'init') await initialize();
|
||||||
|
else {
|
||||||
|
await verify();
|
||||||
|
report.passed = true;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
report.passed = false;
|
||||||
|
report.failure = { message: error.message, stack: error.stack };
|
||||||
|
console.error(error);
|
||||||
|
process.exitCode = 1;
|
||||||
|
} finally {
|
||||||
|
if (process.argv[2] !== 'init') {
|
||||||
|
await http('http://redis-proxy:8081/restore', {}).catch(() => {});
|
||||||
|
report.finishedAt = new Date().toISOString();
|
||||||
|
await writeFile('/results/report.json', JSON.stringify(report, null, 2));
|
||||||
|
}
|
||||||
|
for (const close of cleanup.reverse()) {
|
||||||
|
try {
|
||||||
|
await close();
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# From the repository root: sh apps/api/test/auction-cluster/run.sh
|
||||||
|
# Requires Docker Compose. No host ports or production credentials are used.
|
||||||
|
# Results and container logs remain in this directory's ignored results/ folder.
|
||||||
|
# Authentication uses a fixture admin; client recovery uses the production
|
||||||
|
# helpers and QueryObserver without rendering React. MongoDB 7 is the default
|
||||||
|
# for the test host's kernel; set MONGO_TEST_IMAGE on a compatible host to match
|
||||||
|
# the production database version.
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-lolympus-auction-cluster-$(date +%s)}"
|
||||||
|
case "$COMPOSE_PROJECT_NAME" in
|
||||||
|
lolympus-auction-cluster-*) ;;
|
||||||
|
*) echo 'Refusing to use a Compose project outside the auction-cluster test prefix.' >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
mkdir -p results
|
||||||
|
cleanup() {
|
||||||
|
docker compose logs --no-color api-a api-b redis-proxy > results/containers.log 2>&1 || true
|
||||||
|
docker compose down --volumes --remove-orphans
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
docker compose build runner
|
||||||
|
docker compose up -d redis mongo redis-proxy
|
||||||
|
docker compose run --rm runner node test/auction-cluster/run.mjs init
|
||||||
|
docker compose up -d api-a api-b lb
|
||||||
|
docker compose run --rm runner node test/auction-cluster/run.mjs verify
|
||||||
Reference in new issue
Block a user