test(auction): add two-node Docker failure integration suite
Exercise Nginx routing, Mongo transaction ordering, history backfills, Undo, Pub/Sub TCP loss, safety polling, resync, and reconnection. Keep fixtures and results isolated from production and git.
This commit is contained in:
1 parent
23a0a3f8dd
commit
8d741ef9a7
9 files changed
+1024
No files matched your search
@@ -0,0 +1,641 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createRequire } from 'node:module';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { request } from 'node:http';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import mongoose from 'mongoose';
|
||||
import { createClient } from 'redis';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
require('ts-node/register/transpile-only');
|
||||
const {
|
||||
AuctionRedisService,
|
||||
} = require('../../src/modules/auction/redis/auction-redis.service.ts');
|
||||
const {
|
||||
AuctionRedisCommandService,
|
||||
} = require('../../src/modules/auction/redis/auction-redis-command.service.ts');
|
||||
const {
|
||||
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
|
||||
} = require('../../src/modules/auction/redis/auction-redis.keys.ts');
|
||||
const {
|
||||
MatchSchema,
|
||||
AuctionSchema,
|
||||
AuctionHistoryEntrySchema,
|
||||
AuctionSideEffectApplicationSchema,
|
||||
} = require('../../src/modules/global/database/schema.ts');
|
||||
const webRequire = createRequire(
|
||||
new URL('../../../web/package.json', import.meta.url),
|
||||
);
|
||||
const { io } = webRequire('socket.io-client');
|
||||
const { QueryClient, QueryObserver, environmentManager } = webRequire(
|
||||
'@tanstack/react-query',
|
||||
);
|
||||
const {
|
||||
createAuctionHeartbeatMonitor,
|
||||
shouldApplyPolledAuctionState,
|
||||
mergeAuctionState,
|
||||
} = await import('../../../web/src/api/auction-state.ts');
|
||||
const { createAuctionResyncRequester } =
|
||||
await import('../../../web/src/api/auction-resync.ts');
|
||||
const { AUCTION_SAFETY_POLL_INTERVAL_MS } =
|
||||
await import('../../../web/src/api/auction-polling.ts');
|
||||
|
||||
if (
|
||||
process.env.REDIS_TEST_URL !== 'redis://redis-proxy:6379' ||
|
||||
!process.env.MONGO_TEST_URL?.startsWith(
|
||||
'mongodb://mongo:27017/auction_cluster?',
|
||||
)
|
||||
) {
|
||||
throw new Error('Only the isolated Compose services are allowed.');
|
||||
}
|
||||
const redis = createClient({ url: 'redis://redis:6379' });
|
||||
redis.on('error', () => {});
|
||||
await redis.connect();
|
||||
const cleanup = [() => (redis.isOpen ? redis.quit() : undefined)];
|
||||
const report = {
|
||||
startedAt: new Date().toISOString(),
|
||||
checks: [],
|
||||
requests: [],
|
||||
limitation:
|
||||
'Real API processes, Nginx, Redis TCP faults and Mongo transactions; fixture identity replaces the external login provider. Client recovery uses production helpers and QueryObserver, without rendering the React page.',
|
||||
};
|
||||
|
||||
async function until(check, description, timeout = 15_000) {
|
||||
const deadline = Date.now() + timeout;
|
||||
let lastError;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const value = await check();
|
||||
if (value) return value;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
await delay(50);
|
||||
}
|
||||
throw new Error(
|
||||
`Timed out: ${description}${lastError ? `: ${lastError.message}` : ''}`,
|
||||
);
|
||||
}
|
||||
async function http(url, body) {
|
||||
const started = Date.now();
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = request(
|
||||
url,
|
||||
{
|
||||
method: body === undefined ? 'GET' : 'POST',
|
||||
agent: false,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
},
|
||||
(res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
res.on('end', () => {
|
||||
report.requests.push({
|
||||
url,
|
||||
node: res.headers['x-cluster-node'],
|
||||
status: res.statusCode,
|
||||
durationMS: Date.now() - started,
|
||||
});
|
||||
if (res.statusCode < 200 || res.statusCode >= 300)
|
||||
return reject(new Error(`${url}: ${res.statusCode} ${data}`));
|
||||
try {
|
||||
resolve({
|
||||
body: JSON.parse(data),
|
||||
node: res.headers['x-cluster-node'],
|
||||
});
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
req.setTimeout(10_000, () =>
|
||||
req.destroy(new Error(`HTTP timeout: ${url}`)),
|
||||
);
|
||||
req.on('error', reject);
|
||||
req.end(body === undefined ? undefined : JSON.stringify(body));
|
||||
});
|
||||
}
|
||||
const pass = (name, details = {}) => {
|
||||
report.checks.push({ name, passed: true, ...details });
|
||||
console.log(`PASS ${name} ${JSON.stringify(details)}`);
|
||||
};
|
||||
|
||||
async function initialize() {
|
||||
const admin = new mongoose.mongo.MongoClient(
|
||||
'mongodb://mongo:27017/?directConnection=true',
|
||||
{ serverSelectionTimeoutMS: 1000 },
|
||||
);
|
||||
await until(
|
||||
async () => {
|
||||
await admin.connect();
|
||||
return true;
|
||||
},
|
||||
'Mongo startup',
|
||||
60_000,
|
||||
);
|
||||
await admin.db('admin').command({
|
||||
replSetInitiate: {
|
||||
_id: 'auction_cluster',
|
||||
members: [{ _id: 0, host: 'mongo:27017' }],
|
||||
},
|
||||
});
|
||||
await until(
|
||||
async () =>
|
||||
(await admin.db('admin').command({ hello: 1 })).isWritablePrimary,
|
||||
'Mongo primary',
|
||||
60_000,
|
||||
);
|
||||
const build = await admin.db('admin').command({ buildInfo: 1 });
|
||||
await admin.close();
|
||||
const connection = await mongoose
|
||||
.createConnection(process.env.MONGO_TEST_URL, { autoIndex: false })
|
||||
.asPromise();
|
||||
try {
|
||||
const models = {
|
||||
match: connection.model('match', MatchSchema),
|
||||
auction: connection.model('auction', AuctionSchema),
|
||||
history: connection.model('auctionHistory', AuctionHistoryEntrySchema),
|
||||
sideEffects: connection.model(
|
||||
'auctionSideEffectApplication',
|
||||
AuctionSideEffectApplicationSchema,
|
||||
),
|
||||
};
|
||||
await Promise.all(
|
||||
Object.values(models).map((model) => model.createCollection()),
|
||||
);
|
||||
await models.history.createIndexes();
|
||||
const matchId = new mongoose.Types.ObjectId(),
|
||||
communityId = new mongoose.Types.ObjectId(),
|
||||
auctionId = new mongoose.Types.ObjectId();
|
||||
const lifecycle = `cluster:${matchId}`;
|
||||
const playerA = new mongoose.Types.ObjectId(),
|
||||
playerB = new mongoose.Types.ObjectId(),
|
||||
captain = new mongoose.Types.ObjectId();
|
||||
const structure = {
|
||||
auctionPlayerOrder: [
|
||||
{ userId: playerA.toString() },
|
||||
{ userId: playerB.toString() },
|
||||
],
|
||||
auctionTeams: [
|
||||
{
|
||||
teamIndex: 1,
|
||||
captainUserId: captain.toString(),
|
||||
remainingBidPointAmount: 1000,
|
||||
members: [],
|
||||
},
|
||||
],
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentAuctionRound: 1,
|
||||
passedAuctionPlayerQueue: [],
|
||||
maximumBidRoundDurationSeconds: 600,
|
||||
bidRoundExtensionSeconds: 5,
|
||||
initialTeamBidPointAmount: 1000,
|
||||
topTierAuctionPriorityThreshold: 'NONE',
|
||||
manualTopTierAuctionPriorityUserIds: [],
|
||||
};
|
||||
await models.match.collection.insertOne({
|
||||
_id: matchId,
|
||||
communityId,
|
||||
title: 'Isolated cluster test',
|
||||
participants: [],
|
||||
teams: [],
|
||||
status: 'team-building',
|
||||
scheduledAt: new Date(),
|
||||
});
|
||||
await models.auction.collection.insertOne({
|
||||
_id: auctionId,
|
||||
matchId,
|
||||
auctionEngineVersion: 'redis-v1',
|
||||
auctionLifecycleId: lifecycle,
|
||||
phase: 'presenting',
|
||||
revision: 0,
|
||||
currentAuctionHistorySequence: 0,
|
||||
chatMessages: [],
|
||||
...structure,
|
||||
});
|
||||
const commands = new AuctionRedisCommandService(
|
||||
new AuctionRedisService(redis),
|
||||
);
|
||||
await commands.hydrateState(matchId.toString(), {
|
||||
matchId: matchId.toString(),
|
||||
auctionLifecycleId: lifecycle,
|
||||
phase: 'presenting',
|
||||
revision: 0,
|
||||
auctionHistorySequence: 0,
|
||||
maximumBidRoundDurationSeconds: 600,
|
||||
structureStateJson: JSON.stringify(structure),
|
||||
liveStatePatch: {
|
||||
currentAuctionPlayerOrderIndex: 0,
|
||||
currentBidRoundPlayerUserId: playerA.toString(),
|
||||
currentBidRoundId: 'initial',
|
||||
currentHighestBidPointAmount: 0,
|
||||
currentHighestBidderTeamIndex: -1,
|
||||
maximumBidRoundDurationSeconds: 600,
|
||||
bidRoundExtensionSeconds: 5,
|
||||
'team:1:captainUserId': captain.toString(),
|
||||
'team:1:remainingBidPointAmount': 1000,
|
||||
'team:1:assignedAuctionTeamMemberCount': 1,
|
||||
},
|
||||
});
|
||||
const fixture = {
|
||||
matchId: matchId.toString(),
|
||||
communityId: communityId.toString(),
|
||||
auctionId: auctionId.toString(),
|
||||
lifecycle,
|
||||
mongoVersion: build.version,
|
||||
redisVersion: (await redis.info('server')).match(
|
||||
/redis_version:([^\r\n]+)/,
|
||||
)?.[1],
|
||||
};
|
||||
await writeFile('/results/fixture.json', JSON.stringify(fixture, null, 2));
|
||||
console.log(
|
||||
`Initialized isolated MongoDB ${build.version} and Redis ${fixture.redisVersion}`,
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function verify() {
|
||||
const fixture = JSON.parse(await readFile('/results/fixture.json', 'utf8'));
|
||||
report.versions = {
|
||||
node: process.version,
|
||||
mongo: fixture.mongoVersion,
|
||||
redis: fixture.redisVersion,
|
||||
};
|
||||
const route = `/communities/${fixture.communityId}/matches/${fixture.matchId}/auction`;
|
||||
const url = `http://lb:8080${route}`;
|
||||
await until(
|
||||
async () => (await http('http://api-a:3000/_cluster/health')).body.node,
|
||||
'API A startup',
|
||||
60_000,
|
||||
);
|
||||
await until(
|
||||
async () => (await http('http://api-b:3000/_cluster/health')).body.node,
|
||||
'API B startup',
|
||||
60_000,
|
||||
);
|
||||
const routedNodes = new Set();
|
||||
for (let i = 0; i < 4; i += 1) routedNodes.add((await http(url)).node);
|
||||
assert.deepEqual([...routedNodes].sort(), ['api-a', 'api-b']);
|
||||
pass('Nginx routes real HTTP requests to both API processes');
|
||||
|
||||
async function connectViewer() {
|
||||
const socket = io('http://lb:8080/auction', {
|
||||
transports: ['websocket'],
|
||||
autoConnect: false,
|
||||
reconnection: false,
|
||||
});
|
||||
const viewer = {
|
||||
socket,
|
||||
state: null,
|
||||
states: [],
|
||||
histories: [],
|
||||
errors: [],
|
||||
};
|
||||
socket.on('state', (state) => {
|
||||
viewer.state = state;
|
||||
viewer.states.push(state);
|
||||
});
|
||||
socket.on('error', (error) => viewer.errors.push(error));
|
||||
socket.on('connect_error', (error) => viewer.errors.push(error.message));
|
||||
socket.on('history-updated', (event) => {
|
||||
void http(`${url}/history`)
|
||||
.then(({ body, node }) => viewer.histories.push({ event, body, node }))
|
||||
.catch((error) => viewer.errors.push(error.message));
|
||||
});
|
||||
cleanup.push(() => socket.disconnect());
|
||||
socket.connect();
|
||||
await until(
|
||||
() => socket.connected,
|
||||
`socket connect: ${viewer.errors.join(',')}`,
|
||||
);
|
||||
const ack = await socket
|
||||
.timeout(5000)
|
||||
.emitWithAck('join', { matchId: fixture.matchId });
|
||||
assert.equal(ack.ok, true);
|
||||
assert.equal(viewer.state.matchId, fixture.matchId);
|
||||
assert.equal(viewer.state.auctionEngineVersion, 'redis-v1');
|
||||
const health = await Promise.all(
|
||||
['api-a', 'api-b'].map((node) =>
|
||||
http(`http://${node}:3000/_cluster/health`),
|
||||
),
|
||||
);
|
||||
viewer.node = health.find((entry) =>
|
||||
entry.body.sockets.includes(socket.id),
|
||||
).body.node;
|
||||
return viewer;
|
||||
}
|
||||
const one = await connectViewer(),
|
||||
two = await connectViewer();
|
||||
assert.notEqual(one.node, two.node);
|
||||
const a = one.node === 'api-a' ? one : two;
|
||||
const b = one.node === 'api-b' ? one : two;
|
||||
const initialRevision = a.state.revision;
|
||||
const revisionFor = (sequence) => initialRevision + sequence;
|
||||
assert.equal(b.state.revision, initialRevision);
|
||||
pass('WebSocket viewers join different nodes through Nginx', {
|
||||
nodes: [one.node, two.node],
|
||||
});
|
||||
|
||||
const sequences = async () =>
|
||||
(await http(`${url}/history`)).body.map((entry) => entry.sequence);
|
||||
const action = async (viewer, name) => {
|
||||
const ack = await viewer.socket
|
||||
.timeout(5000)
|
||||
.emitWithAck(name, { auctionActionRequestId: randomUUID() });
|
||||
assert.equal(ack.ok, true, JSON.stringify(ack));
|
||||
};
|
||||
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
|
||||
await action(a, 'open-bidding');
|
||||
await until(
|
||||
() =>
|
||||
a.state.revision === revisionFor(1) &&
|
||||
b.state.revision === revisionFor(1),
|
||||
'cross-node full state',
|
||||
);
|
||||
assert.equal(a.histories.length + b.histories.length, 0);
|
||||
await http('http://api-a:3000/_cluster/hold-commit', {});
|
||||
const pendingDrain = http('http://api-a:3000/_cluster/drain', {});
|
||||
await until(
|
||||
async () =>
|
||||
(await http('http://api-a:3000/_cluster/health')).body.commitsWaiting ===
|
||||
1,
|
||||
'real Mongo commit barrier',
|
||||
);
|
||||
for (let i = 0; i < 4; i += 1) assert.deepEqual(await sequences(), []);
|
||||
assert.equal(a.histories.length + b.histories.length, 0);
|
||||
await http('http://api-a:3000/_cluster/release-commit', {});
|
||||
assert.equal((await pendingDrain).body.applied, 1);
|
||||
await until(
|
||||
() => a.histories.length === 1 && b.histories.length === 1,
|
||||
'post-commit history event on both nodes',
|
||||
);
|
||||
for (const viewer of [a, b])
|
||||
assert.deepEqual(
|
||||
viewer.histories[0].body.map((entry) => entry.sequence),
|
||||
[1],
|
||||
);
|
||||
pass(
|
||||
'History notification crosses nodes only after a real Mongo transaction commits',
|
||||
);
|
||||
|
||||
await action(a, 'passed');
|
||||
await action(b, 'next');
|
||||
await until(
|
||||
() =>
|
||||
a.state.revision === revisionFor(3) &&
|
||||
b.state.revision === revisionFor(3),
|
||||
'state fanout in both directions',
|
||||
);
|
||||
const keys = await redis.zRange(AUCTION_DURABILITY_RETRY_QUEUE_KEY, 0, -1);
|
||||
const queued = await Promise.all(
|
||||
keys.map(async (key) => ({
|
||||
key,
|
||||
payload: JSON.parse(await redis.hGet(key, 'payloadJson')),
|
||||
})),
|
||||
);
|
||||
const third = queued.find(
|
||||
({ payload }) => payload.auctionHistorySequence === 3,
|
||||
);
|
||||
const second = queued.find(
|
||||
({ payload }) => payload.auctionHistorySequence === 2,
|
||||
);
|
||||
assert.ok(third && second);
|
||||
await http('http://api-b:3000/_cluster/apply', third.payload);
|
||||
assert.deepEqual(await sequences(), [3, 1]);
|
||||
const previousEvents = b.histories.length;
|
||||
await http('http://api-a:3000/_cluster/apply', second.payload);
|
||||
await until(
|
||||
() =>
|
||||
b.histories.length > previousEvents &&
|
||||
b.histories.at(-1).body.some((entry) => entry.sequence === 2),
|
||||
'late append history refresh',
|
||||
);
|
||||
assert.deepEqual(await sequences(), [3, 2, 1]);
|
||||
for (const item of queued) {
|
||||
await redis.zRem(AUCTION_DURABILITY_RETRY_QUEUE_KEY, item.key);
|
||||
await redis.del(item.key);
|
||||
}
|
||||
pass(
|
||||
'A delayed history append refreshes another node with the same maximum sequence',
|
||||
{ maximumSequence: 3 },
|
||||
);
|
||||
|
||||
const undo = await http(`${url}/undo`, {
|
||||
sequence: 1,
|
||||
auctionActionRequestId: randomUUID(),
|
||||
});
|
||||
assert.equal(undo.body.auctionEngineVersion, 'redis-v1');
|
||||
assert.equal(undo.body.revision, revisionFor(4));
|
||||
assert.deepEqual(await sequences(), [3, 2, 1]);
|
||||
await http('http://api-b:3000/_cluster/drain', {});
|
||||
await until(
|
||||
async () => JSON.stringify(await sequences()) === '[4,1]',
|
||||
'durable Undo history',
|
||||
);
|
||||
await until(
|
||||
() => b.histories.at(-1)?.body[0]?.sequence === 4,
|
||||
'Undo history event',
|
||||
);
|
||||
pass('Undo increases sequence and replaces history on every node', {
|
||||
sequence: 4,
|
||||
history: [4, 1],
|
||||
});
|
||||
|
||||
await until(
|
||||
async () =>
|
||||
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
|
||||
'Redis subscriptions',
|
||||
);
|
||||
await http('http://redis-proxy:8081/block', {});
|
||||
console.log(
|
||||
'FAULT Pub/Sub TCP connections blocked; Redis commands and HTTP stay available',
|
||||
);
|
||||
await delay(500);
|
||||
assert.equal(
|
||||
(await http('http://api-a:3000/_cluster/health')).body.node,
|
||||
'api-a',
|
||||
);
|
||||
assert.equal(
|
||||
(await http('http://api-b:3000/_cluster/health')).body.node,
|
||||
'api-b',
|
||||
);
|
||||
assert.equal(a.socket.connected && b.socket.connected, true);
|
||||
const timestamp = await b.socket
|
||||
.timeout(3000)
|
||||
.emitWithAck('time-sync', { sequence: 1 });
|
||||
assert.equal(timestamp.ok, true);
|
||||
pass(
|
||||
'Both API processes and socket clock sync survive Pub/Sub-only TCP disconnection',
|
||||
);
|
||||
|
||||
environmentManager.setIsServer(() => false);
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: { queries: { retry: false, gcTime: Infinity } },
|
||||
});
|
||||
cleanup.push(() => queryClient.clear());
|
||||
let stateByHttp = b.state,
|
||||
historyByHttp,
|
||||
statePolls = 0,
|
||||
historyPolls = 0;
|
||||
const pollingStarted = Date.now();
|
||||
const stateObserver = new QueryObserver(queryClient, {
|
||||
queryKey: ['auction', fixture.matchId],
|
||||
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
|
||||
queryFn: async () => {
|
||||
statePolls += 1;
|
||||
return (await http(url)).body;
|
||||
},
|
||||
});
|
||||
const historyObserver = new QueryObserver(queryClient, {
|
||||
queryKey: ['auction-history', fixture.matchId],
|
||||
refetchInterval: AUCTION_SAFETY_POLL_INTERVAL_MS,
|
||||
queryFn: async () => {
|
||||
historyPolls += 1;
|
||||
return (await http(`${url}/history`)).body;
|
||||
},
|
||||
});
|
||||
cleanup.push(
|
||||
stateObserver.subscribe((result) => {
|
||||
if (
|
||||
result.data &&
|
||||
shouldApplyPolledAuctionState(stateByHttp, result.data, fixture.matchId)
|
||||
)
|
||||
stateByHttp = mergeAuctionState(stateByHttp, result.data);
|
||||
}),
|
||||
);
|
||||
cleanup.push(
|
||||
historyObserver.subscribe((result) => {
|
||||
if (result.data) historyByHttp = result.data;
|
||||
}),
|
||||
);
|
||||
await until(
|
||||
() =>
|
||||
stateByHttp.revision === revisionFor(4) &&
|
||||
historyByHttp?.[0]?.sequence === 4,
|
||||
'baseline HTTP reads',
|
||||
);
|
||||
const beforeHistoryEvents = b.histories.length;
|
||||
await action(a, 'open-bidding');
|
||||
await http('http://api-a:3000/_cluster/drain', {});
|
||||
await delay(250);
|
||||
assert.equal(b.state.revision, revisionFor(4));
|
||||
assert.equal(b.histories.length, beforeHistoryEvents);
|
||||
assert.equal(a.state.revision, revisionFor(5));
|
||||
// Do not run heartbeat recovery yet: verify the independent 30-second safety
|
||||
// route even if the client identity checker fails to notice a missed event.
|
||||
console.log(
|
||||
'WAIT Real 30-second HTTP safety polling with Pub/Sub still blocked',
|
||||
);
|
||||
await until(
|
||||
() =>
|
||||
stateByHttp.revision === revisionFor(5) &&
|
||||
historyByHttp?.[0]?.sequence === 5,
|
||||
'30-second HTTP safety fallback',
|
||||
35_000,
|
||||
);
|
||||
assert.ok(Date.now() - pollingStarted >= 29_000);
|
||||
pass(
|
||||
'Real 30-second HTTP polls repair missed state and history through Nginx',
|
||||
{ elapsedMS: Date.now() - pollingStarted, statePolls, historyPolls },
|
||||
);
|
||||
|
||||
const monitor = createAuctionHeartbeatMonitor();
|
||||
const heartbeat = async (sequence) => {
|
||||
const ack = await b.socket
|
||||
.timeout(3000)
|
||||
.emitWithAck('time-sync', { sequence });
|
||||
assert.equal(ack.ok, true);
|
||||
return monitor.inspect(b.state, ack.auctionState);
|
||||
};
|
||||
assert.equal((await heartbeat(2)).needsResync, false);
|
||||
await delay(10_000);
|
||||
assert.equal((await heartbeat(3)).needsResync, true);
|
||||
let resyncs = 0;
|
||||
const requester = createAuctionResyncRequester(
|
||||
async () => {
|
||||
resyncs += 1;
|
||||
return b.socket.timeout(3000).emitWithAck('resync', {});
|
||||
},
|
||||
(state) => {
|
||||
b.state = state;
|
||||
},
|
||||
);
|
||||
await requester.request(revisionFor(5), fixture.lifecycle);
|
||||
assert.equal(b.state.revision, revisionFor(5));
|
||||
assert.equal(resyncs, 1);
|
||||
pass(
|
||||
'Two real heartbeats detect a persistent gap and private socket resync works during Pub/Sub outage',
|
||||
{ resyncs },
|
||||
);
|
||||
|
||||
await http('http://redis-proxy:8081/restore', {});
|
||||
await until(
|
||||
async () =>
|
||||
(await http('http://redis-proxy:8081/status')).body.subscribers >= 2,
|
||||
'Pub/Sub resubscription',
|
||||
20_000,
|
||||
);
|
||||
await action(a, 'passed');
|
||||
await http('http://api-a:3000/_cluster/drain', {});
|
||||
await until(
|
||||
() =>
|
||||
b.state.revision === revisionFor(6) &&
|
||||
b.histories.at(-1)?.body[0]?.sequence === 6,
|
||||
'events after Pub/Sub restoration',
|
||||
);
|
||||
pass(
|
||||
'Cross-node state and history events recover after restoring the TCP path',
|
||||
);
|
||||
const fault = (await http('http://redis-proxy:8081/status')).body;
|
||||
report.fault = fault;
|
||||
const previousId = b.socket.id;
|
||||
b.socket.disconnect();
|
||||
b.socket.connect();
|
||||
await until(
|
||||
() => b.socket.connected && b.socket.id !== previousId,
|
||||
'socket reconnect through Nginx',
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await b.socket
|
||||
.timeout(5000)
|
||||
.emitWithAck('join', { matchId: fixture.matchId })
|
||||
).ok,
|
||||
true,
|
||||
);
|
||||
await until(
|
||||
() => b.state.revision === revisionFor(6),
|
||||
'reconnect full state',
|
||||
);
|
||||
pass('A reconnected viewer receives the latest snapshot through Nginx');
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.argv[2] === 'init') await initialize();
|
||||
else {
|
||||
await verify();
|
||||
report.passed = true;
|
||||
}
|
||||
} catch (error) {
|
||||
report.passed = false;
|
||||
report.failure = { message: error.message, stack: error.stack };
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
if (process.argv[2] !== 'init') {
|
||||
await http('http://redis-proxy:8081/restore', {}).catch(() => {});
|
||||
report.finishedAt = new Date().toISOString();
|
||||
await writeFile('/results/report.json', JSON.stringify(report, null, 2));
|
||||
}
|
||||
for (const close of cleanup.reverse()) {
|
||||
try {
|
||||
await close();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user