Files
dahyeon 8d741ef9a7 test(auction): add two-node Docker failure integration suite
Exercise Nginx routing, Mongo transaction ordering, history backfills, Undo, Pub/Sub TCP loss, safety polling, resync, and reconnection. Keep fixtures and results isolated from production and git.
2026-11-12 01:49:35 +09:00

642 lines
20 KiB
JavaScript

import assert from 'node:assert/strict';
import { createRequire } from 'node:module';
import { readFile, writeFile } from 'node:fs/promises';
import { request } from 'node:http';
import { setTimeout as delay } from 'node:timers/promises';
import { randomUUID } from 'node:crypto';
import mongoose from 'mongoose';
import { createClient } from 'redis';
const require = createRequire(import.meta.url);
require('ts-node/register/transpile-only');
const {
AuctionRedisService,
} = require('../../src/modules/auction/redis/auction-redis.service.ts');
const {
AuctionRedisCommandService,
} = require('../../src/modules/auction/redis/auction-redis-command.service.ts');
const {
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
} = require('../../src/modules/auction/redis/auction-redis.keys.ts');
const {
MatchSchema,
AuctionSchema,
AuctionHistoryEntrySchema,
AuctionSideEffectApplicationSchema,
} = require('../../src/modules/global/database/schema.ts');
const webRequire = createRequire(
new URL('../../../web/package.json', import.meta.url),
);
const { io } = webRequire('socket.io-client');
const { QueryClient, QueryObserver, environmentManager } = webRequire(
'@tanstack/react-query',
);
const {
createAuctionHeartbeatMonitor,
shouldApplyPolledAuctionState,
mergeAuctionState,
} = await import('../../../web/src/api/auction-state.ts');
const { createAuctionResyncRequester } =
await import('../../../web/src/api/auction-resync.ts');
const { AUCTION_SAFETY_POLL_INTERVAL_MS } =
await import('../../../web/src/api/auction-polling.ts');
if (
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379' ||
!process.env.MONGO_TEST_URL?.startsWith(
'mongodb://mongo:27017/auction_cluster?',
)
) {
throw new Error('Only the isolated Compose services are allowed.');
}
const redis = createClient({ url: 'redis://redis:6379' });
redis.on('error', () => {});
await redis.connect();
const cleanup = [() => (redis.isOpen ? redis.quit() : undefined)];
const report = {
startedAt: new Date().toISOString(),
checks: [],
requests: [],
limitation:
'Real API processes, Nginx, Redis TCP faults and Mongo transactions; fixture identity replaces the external login provider. Client recovery uses production helpers and QueryObserver, without rendering the React page.',
};
async function until(check, description, timeout = 15_000) {
const deadline = Date.now() + timeout;
let lastError;
while (Date.now() < deadline) {
try {
const value = await check();
if (value) return value;
} catch (error) {
lastError = error;
}
await delay(50);
}
throw new Error(
`Timed out: ${description}${lastError ? `: ${lastError.message}` : ''}`,
);
}
async function http(url, body) {
const started = Date.now();
return new Promise((resolve, reject) => {
const req = request(
url,
{
method: body === undefined ? 'GET' : 'POST',
agent: false,
headers: { 'content-type': 'application/json' },
},
(res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
report.requests.push({
url,
node: res.headers['x-cluster-node'],
status: res.statusCode,
durationMS: Date.now() - started,
});
if (res.statusCode < 200 || res.statusCode >= 300)
return reject(new Error(`${url}: ${res.statusCode} ${data}`));
try {
resolve({
body: JSON.parse(data),
node: res.headers['x-cluster-node'],
});
} catch (error) {
reject(error);
}
});
},
);
req.setTimeout(10_000, () =>
req.destroy(new Error(`HTTP timeout: ${url}`)),
);
req.on('error', reject);
req.end(body === undefined ? undefined : JSON.stringify(body));
});
}
const pass = (name, details = {}) => {
report.checks.push({ name, passed: true, ...details });
console.log(`PASS ${name} ${JSON.stringify(details)}`);
};
async function initialize() {
const admin = new mongoose.mongo.MongoClient(
'mongodb://mongo:27017/?directConnection=true',
{ serverSelectionTimeoutMS: 1000 },
);
await until(
async () => {
await admin.connect();
return true;
},
'Mongo startup',
60_000,
);
await admin.db('admin').command({
replSetInitiate: {
_id: 'auction_cluster',
members: [{ _id: 0, host: 'mongo:27017' }],
},
});
await until(
async () =>
(await admin.db('admin').command({ hello: 1 })).isWritablePrimary,
'Mongo primary',
60_000,
);
const build = await admin.db('admin').command({ buildInfo: 1 });
await admin.close();
const connection = await mongoose
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
.asPromise();
try {
const models = {
match: connection.model('match', MatchSchema),
auction: connection.model('auction', AuctionSchema),
history: connection.model('auctionHistory', AuctionHistoryEntrySchema),
sideEffects: connection.model(
'auctionSideEffectApplication',
AuctionSideEffectApplicationSchema,
),
};
await Promise.all(
Object.values(models).map((model) => model.createCollection()),
);
await models.history.createIndexes();
const matchId = new mongoose.Types.ObjectId(),
communityId = new mongoose.Types.ObjectId(),
auctionId = new mongoose.Types.ObjectId();
const lifecycle = `cluster:${matchId}`;
const playerA = new mongoose.Types.ObjectId(),
playerB = new mongoose.Types.ObjectId(),
captain = new mongoose.Types.ObjectId();
const structure = {
auctionPlayerOrder: [
{ userId: playerA.toString() },
{ userId: playerB.toString() },
],
auctionTeams: [
{
teamIndex: 1,
captainUserId: captain.toString(),
remainingBidPointAmount: 1000,
members: [],
},
],
currentAuctionPlayerOrderIndex: 0,
currentAuctionRound: 1,
passedAuctionPlayerQueue: [],
maximumBidRoundDurationSeconds: 600,
bidRoundExtensionSeconds: 5,
initialTeamBidPointAmount: 1000,
topTierAuctionPriorityThreshold: 'NONE',
manualTopTierAuctionPriorityUserIds: [],
};
await models.match.collection.insertOne({
_id: matchId,
communityId,
title: 'Isolated cluster test',
participants: [],
teams: [],
status: 'team-building',
scheduledAt: new Date(),
});
await models.auction.collection.insertOne({
_id: auctionId,
matchId,
auctionEngineVersion: 'redis-v1',
auctionLifecycleId: lifecycle,
phase: 'presenting',
revision: 0,
currentAuctionHistorySequence: 0,
chatMessages: [],
...structure,
});
const commands = new AuctionRedisCommandService(
new AuctionRedisService(redis),
);
await commands.hydrateState(matchId.toString(), {
matchId: matchId.toString(),
auctionLifecycleId: lifecycle,
phase: 'presenting',
revision: 0,
auctionHistorySequence: 0,
maximumBidRoundDurationSeconds: 600,
structureStateJson: JSON.stringify(structure),
liveStatePatch: {
currentAuctionPlayerOrderIndex: 0,
currentBidRoundPlayerUserId: playerA.toString(),
currentBidRoundId: 'initial',
currentHighestBidPointAmount: 0,
currentHighestBidderTeamIndex: -1,
maximumBidRoundDurationSeconds: 600,
bidRoundExtensionSeconds: 5,
'team:1:captainUserId': captain.toString(),
'team:1:remainingBidPointAmount': 1000,
'team:1:assignedAuctionTeamMemberCount': 1,
},
});
const fixture = {
matchId: matchId.toString(),
communityId: communityId.toString(),
auctionId: auctionId.toString(),
lifecycle,
mongoVersion: build.version,
redisVersion: (await redis.info('server')).match(
/redis_version:([^\r\n]+)/,
)?.[1],
};
await writeFile('/results/fixture.json', JSON.stringify(fixture, null, 2));
console.log(
`Initialized isolated MongoDB ${build.version} and Redis ${fixture.redisVersion}`,
);
} finally {
await connection.close();
}
}
async function verify() {
const fixture = JSON.parse(await readFile('/results/fixture.json', 'utf8'));
report.versions = {
node: process.version,
mongo: fixture.mongoVersion,
redis: fixture.redisVersion,
};
const route = `/communities/${fixture.communityId}/matches/${fixture.matchId}/auction`;
const url = `http://lb:8080${route}`;
await until(
async () => (await http('http://api-a:3000/_cluster/health')).body.node,
'API A startup',
60_000,
);
await until(
async () => (await http('http://api-b:3000/_cluster/health')).body.node,
'API B startup',
60_000,
);
const routedNodes = new Set();
for (let i = 0; i < 4; i += 1) routedNodes.add((await http(url)).node);
assert.deepEqual([...routedNodes].sort(), ['api-a', 'api-b']);
pass('Nginx routes real HTTP requests to both API processes');
async function connectViewer() {
const socket = io('http://lb:8080/auction', {
transports: ['websocket'],
autoConnect: false,
reconnection: false,
});
const viewer = {
socket,
state: null,
states: [],
histories: [],
errors: [],
};
socket.on('state', (state) => {
viewer.state = state;
viewer.states.push(state);
});
socket.on('error', (error) => viewer.errors.push(error));
socket.on('connect_error', (error) => viewer.errors.push(error.message));
socket.on('history-updated', (event) => {
void http(`${url}/history`)
.then(({ body, node }) => viewer.histories.push({ event, body, node }))
.catch((error) => viewer.errors.push(error.message));
});
cleanup.push(() => socket.disconnect());
socket.connect();
await until(
() => socket.connected,
`socket connect: ${viewer.errors.join(',')}`,
);
const ack = await socket
.timeout(5000)
.emitWithAck('join', { matchId: fixture.matchId });
assert.equal(ack.ok, true);
assert.equal(viewer.state.matchId, fixture.matchId);
assert.equal(viewer.state.auctionEngineVersion, 'redis-v1');
const health = await Promise.all(
['api-a', 'api-b'].map((node) =>
http(`http://${node}:3000/_cluster/health`),
),
);
viewer.node = health.find((entry) =>
entry.body.sockets.includes(socket.id),
).body.node;
return viewer;
}
const one = await connectViewer(),
two = await connectViewer();
assert.notEqual(one.node, two.node);
const a = one.node === 'api-a' ? one : two;
const b = one.node === 'api-b' ? one : two;
const initialRevision = a.state.revision;
const revisionFor = (sequence) => initialRevision + sequence;
assert.equal(b.state.revision, initialRevision);
pass('WebSocket viewers join different nodes through Nginx', {
nodes: [one.node, two.node],
});
const sequences = async () =>
(await http(`${url}/history`)).body.map((entry) => entry.sequence);
const action = async (viewer, name) => {
const ack = await viewer.socket
.timeout(5000)
.emitWithAck(name, { auctionActionRequestId: randomUUID() });
assert.equal(ack.ok, true, JSON.stringify(ack));
};
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
await action(a, 'open-bidding');
await until(
() =>
a.state.revision === revisionFor(1) &&
b.state.revision === revisionFor(1),
'cross-node full state',
);
assert.equal(a.histories.length + b.histories.length, 0);
await http('http://api-a:3000/_cluster/hold-commit', {});
const pendingDrain = http('http://api-a:3000/_cluster/drain', {});
await until(
async () =>
(await http('http://api-a:3000/_cluster/health')).body.commitsWaiting ===
1,
'real Mongo commit barrier',
);
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
assert.equal(a.histories.length + b.histories.length, 0);
await http('http://api-a:3000/_cluster/release-commit', {});
assert.equal((await pendingDrain).body.applied, 1);
await until(
() => a.histories.length === 1 && b.histories.length === 1,
'post-commit history event on both nodes',
);
for (const viewer of [a, b])
assert.deepEqual(
viewer.histories[0].body.map((entry) => entry.sequence),
[1],
);
pass(
'History notification crosses nodes only after a real Mongo transaction commits',
);
await action(a, 'passed');
await action(b, 'next');
await until(
() =>
a.state.revision === revisionFor(3) &&
b.state.revision === revisionFor(3),
'state fanout in both directions',
);
const keys = await redis.zRange(AUCTION_DURABILITY_RETRY_QUEUE_KEY, 0, -1);
const queued = await Promise.all(
keys.map(async (key) => ({
key,
payload: JSON.parse(await redis.hGet(key, 'payloadJson')),
})),
);
const third = queued.find(
({ payload }) => payload.auctionHistorySequence === 3,
);
const second = queued.find(
({ payload }) => payload.auctionHistorySequence === 2,
);
assert.ok(third && second);
await http('http://api-b:3000/_cluster/apply', third.payload);
assert.deepEqual(await sequences(), [3, 1]);
const previousEvents = b.histories.length;
await http('http://api-a:3000/_cluster/apply', second.payload);
await until(
() =>
b.histories.length > previousEvents &&
b.histories.at(-1).body.some((entry) => entry.sequence === 2),
'late append history refresh',
);
assert.deepEqual(await sequences(), [3, 2, 1]);
for (const item of queued) {
await redis.zRem(AUCTION_DURABILITY_RETRY_QUEUE_KEY, item.key);
await redis.del(item.key);
}
pass(
'A delayed history append refreshes another node with the same maximum sequence',
{ maximumSequence: 3 },
);
const undo = await http(`${url}/undo`, {
sequence: 1,
auctionActionRequestId: randomUUID(),
});
assert.equal(undo.body.auctionEngineVersion, 'redis-v1');
assert.equal(undo.body.revision, revisionFor(4));
assert.deepEqual(await sequences(), [3, 2, 1]);
await http('http://api-b:3000/_cluster/drain', {});
await until(
async () => JSON.stringify(await sequences()) === '[4,1]',
'durable Undo history',
);
await until(
() => b.histories.at(-1)?.body[0]?.sequence === 4,
'Undo history event',
);
pass('Undo increases sequence and replaces history on every node', {
sequence: 4,
history: [4, 1],
});
await until(
async () =>
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
'Redis subscriptions',
);
await http('http://redis-proxy:8081/block', {});
console.log(
'FAULT Pub/Sub TCP connections blocked; Redis commands and HTTP stay available',
);
await delay(500);
assert.equal(
(await http('http://api-a:3000/_cluster/health')).body.node,
'api-a',
);
assert.equal(
(await http('http://api-b:3000/_cluster/health')).body.node,
'api-b',
);
assert.equal(a.socket.connected && b.socket.connected, true);
const timestamp = await b.socket
.timeout(3000)
.emitWithAck('time-sync', { sequence: 1 });
assert.equal(timestamp.ok, true);
pass(
'Both API processes and socket clock sync survive Pub/Sub-only TCP disconnection',
);
environmentManager.setIsServer(() => false);
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false, gcTime: Infinity } },
});
cleanup.push(() => queryClient.clear());
let stateByHttp = b.state,
historyByHttp,
statePolls = 0,
historyPolls = 0;
const pollingStarted = Date.now();
const stateObserver = new QueryObserver(queryClient, {
queryKey: ['auction', fixture.matchId],
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
queryFn: async () => {
statePolls += 1;
return (await http(url)).body;
},
});
const historyObserver = new QueryObserver(queryClient, {
queryKey: ['auction-history', fixture.matchId],
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
queryFn: async () => {
historyPolls += 1;
return (await http(`${url}/history`)).body;
},
});
cleanup.push(
stateObserver.subscribe((result) => {
if (
result.data &&
shouldApplyPolledAuctionState(stateByHttp, result.data, fixture.matchId)
)
stateByHttp = mergeAuctionState(stateByHttp, result.data);
}),
);
cleanup.push(
historyObserver.subscribe((result) => {
if (result.data) historyByHttp = result.data;
}),
);
await until(
() =>
stateByHttp.revision === revisionFor(4) &&
historyByHttp?.[0]?.sequence === 4,
'baseline HTTP reads',
);
const beforeHistoryEvents = b.histories.length;
await action(a, 'open-bidding');
await http('http://api-a:3000/_cluster/drain', {});
await delay(250);
assert.equal(b.state.revision, revisionFor(4));
assert.equal(b.histories.length, beforeHistoryEvents);
assert.equal(a.state.revision, revisionFor(5));
// Do not run heartbeat recovery yet: verify the independent 30-second safety
// route even if the client identity checker fails to notice a missed event.
console.log(
'WAIT Real 30-second HTTP safety polling with Pub/Sub still blocked',
);
await until(
() =>
stateByHttp.revision === revisionFor(5) &&
historyByHttp?.[0]?.sequence === 5,
'30-second HTTP safety fallback',
35_000,
);
assert.ok(Date.now() - pollingStarted >= 29_000);
pass(
'Real 30-second HTTP polls repair missed state and history through Nginx',
{ elapsedMS: Date.now() - pollingStarted, statePolls, historyPolls },
);
const monitor = createAuctionHeartbeatMonitor();
const heartbeat = async (sequence) => {
const ack = await b.socket
.timeout(3000)
.emitWithAck('time-sync', { sequence });
assert.equal(ack.ok, true);
return monitor.inspect(b.state, ack.auctionState);
};
assert.equal((await heartbeat(2)).needsResync, false);
await delay(10_000);
assert.equal((await heartbeat(3)).needsResync, true);
let resyncs = 0;
const requester = createAuctionResyncRequester(
async () => {
resyncs += 1;
return b.socket.timeout(3000).emitWithAck('resync', {});
},
(state) => {
b.state = state;
},
);
await requester.request(revisionFor(5), fixture.lifecycle);
assert.equal(b.state.revision, revisionFor(5));
assert.equal(resyncs, 1);
pass(
'Two real heartbeats detect a persistent gap and private socket resync works during Pub/Sub outage',
{ resyncs },
);
await http('http://redis-proxy:8081/restore', {});
await until(
async () =>
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
'Pub/Sub resubscription',
20_000,
);
await action(a, 'passed');
await http('http://api-a:3000/_cluster/drain', {});
await until(
() =>
b.state.revision === revisionFor(6) &&
b.histories.at(-1)?.body[0]?.sequence === 6,
'events after Pub/Sub restoration',
);
pass(
'Cross-node state and history events recover after restoring the TCP path',
);
const fault = (await http('http://redis-proxy:8081/status')).body;
report.fault = fault;
const previousId = b.socket.id;
b.socket.disconnect();
b.socket.connect();
await until(
() => b.socket.connected && b.socket.id !== previousId,
'socket reconnect through Nginx',
);
assert.equal(
(
await b.socket
.timeout(5000)
.emitWithAck('join', { matchId: fixture.matchId })
).ok,
true,
);
await until(
() => b.state.revision === revisionFor(6),
'reconnect full state',
);
pass('A reconnected viewer receives the latest snapshot through Nginx');
}
try {
if (process.argv[2] === 'init') await initialize();
else {
await verify();
report.passed = true;
}
} catch (error) {
report.passed = false;
report.failure = { message: error.message, stack: error.stack };
console.error(error);
process.exitCode = 1;
} finally {
if (process.argv[2] !== 'init') {
await http('http://redis-proxy:8081/restore', {}).catch(() => {});
report.finishedAt = new Date().toISOString();
await writeFile('/results/report.json', JSON.stringify(report, null, 2));
}
for (const close of cleanup.reverse()) {
try {
await close();
} catch {}
}
}