fix: harden result polling and Redis 8.10 operations

Preserve tournament result jobs through lock contention and transient failures,
atomically schedule retries before acknowledgement, and publish initial polls
after releasing the preparation lock.

Fix Redis connection blocking, durability Hash cleanup, duplicate auction
request ordering, and scheduled-job promotion. Use native Redis 8.10 stream,
lock, counter, and field-expiring cache operations with invalidation fencing.

Repair 90 test type errors and add a repeatable API typecheck command.

Validation: 704 API tests, 129 web tests, 31 real Redis integration tests,
3 dependency checks, workspace type checks and builds, and changed-file lint.
This commit is contained in:
dahyeon committed 2026-11-09 17:33:13 +09:00
1 parent 0c4c4239ec
commit 4b8ccf0077
49 files changed
+2214 -371

No files matched your search

+7
View File
@@ -0,0 +1,7 @@
{
"rootDir": "..",
"testEnvironment": "node",
"testRegex": "test/.*\\.integration-spec\\.ts$",
"transform": { "^.+\\.ts$": "ts-jest" },
"testTimeout": 15000
}
@@ -0,0 +1,662 @@
import { createClient } from 'redis';
import { RedisService } from '../src/modules/global/redis/redis.service';
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service';
import { AuctionService } from '../src/modules/auction/auction.service';
import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys';
import { RiotJobQueueService } from '../src/modules/riot-job-queue/riot-job-queue.service';
import {
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_STREAM,
RIOT_JOB_SCHEDULED_ZSET,
} from '../src/modules/riot-job-queue/riot-job-queue.types';
import { buildAuctionReserveActionRequestScriptInvocation } from '../src/modules/auction/redis/auction-redis.reserve-action';
import { buildAuctionApplyTransitionScriptInvocation } from '../src/modules/auction/redis/auction-redis.apply-transition';
import { DistributedLockService } from '../src/modules/global/redis/distributed-lock.service';
import { BackgroundRefreshScheduler } from '../src/modules/riot-background-refresh/background-refresh.scheduler';
import { setTimeout as delay } from 'node:timers/promises';
import {
CommunityStatsCacheKey,
CommunityStatsCacheService,
} from '../src/modules/community-stats/community-stats-cache.service';
import { CommunityStatsService } from '../src/modules/community-stats/community-stats.service';
import { PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT } from '../src/modules/riot-job-queue/riot-job-queue.scripts';
import mongoose from 'mongoose';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new RedisService(client);
const auctionRedis = new AuctionRedisService(client);
const queue = new RiotJobQueueService({} as never, redis);
beforeAll(async () => {
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) {
throw new Error(
'Run via pnpm --filter api test:redis to create an isolated Redis process.',
);
}
client.on('error', () => {});
await client.connect();
});
describe('Redis native operations', () => {
it('serves normal commands while a dedicated stream connection is blocked', async () => {
const reader = await redis.createDedicatedConnection();
try {
await redis.createConsumerGroup('stream', 'workers', '0');
const pending = reader.readGroup('stream', 'workers', 'reader', '>', {
blockMs: 2000,
count: 1,
});
void pending.catch(() => {}); // A failed assertion must still allow connection cleanup.
let isBlocked = false;
for (let i = 0; i < 100; i++) {
const clients = await client.sendCommand<string>(['CLIENT', 'LIST']);
if (/flags=b\b.*cmd=xreadgroup/.test(clients)) {
isBlocked = true;
break;
}
await delay(5);
}
expect(isBlocked).toBe(true);
await redis.setString('cache', 'available');
expect(await redis.getString('cache')).toBe('available');
const id = await redis.appendStream('stream', { job: 'wake' });
expect((await pending)?.[0].messages[0].id).toBe(id);
} finally {
if (reader.raw().isOpen) reader.raw().destroy();
}
});
it('claims abandoned pending messages before reading new work with the same idle threshold', async () => {
await redis.createConsumerGroup('stream', 'workers', '0');
const oldId = await redis.appendStream('stream', { job: 'old' });
await redis.readGroup('stream', 'workers', 'dead-worker', '>', {
count: 1,
});
const newId = await redis.appendStream('stream', { job: 'new' });
await client.sendCommand([
'XCLAIM',
'stream',
'workers',
'dead-worker',
'0',
oldId,
'IDLE',
'1800001',
]);
const recovered = await redis.readGroup(
'stream',
'workers',
'live-worker',
'>',
{ count: 1, claimMinIdleMs: 1800000 },
);
expect(recovered?.[0].messages[0]).toEqual(
expect.objectContaining({ id: oldId, message: { job: 'old' } }),
);
const fresh = await redis.readGroup(
'stream',
'workers',
'live-worker',
'>',
{ count: 1, claimMinIdleMs: 1800000 },
);
expect(fresh?.[0].messages[0].id).toBe(newId);
expect(
await redis.readGroup('stream', 'workers', 'other-worker', '>', {
count: 1,
claimMinIdleMs: 1800000,
}),
).toBeNull();
});
it('acknowledges atomically but retains a message until every group has processed it', async () => {
for (const group of ['workers', 'audit'])
await redis.createConsumerGroup('stream', group, '0');
const id = await redis.appendStream('stream', { job: 'one' });
await redis.readGroup('stream', 'workers', 'worker', '>', { count: 1 });
await redis.acknowledgeAndDeleteStream('stream', 'workers', id);
expect(await client.xLen('stream')).toBe(1);
expect((await client.xPending('stream', 'workers')).pending).toBe(0);
await redis.readGroup('stream', 'audit', 'auditor', '>', { count: 1 });
await redis.acknowledgeAndDeleteStream('stream', 'audit', id);
expect(await client.xLen('stream')).toBe(0);
await expect(
redis.acknowledgeAndDeleteStream('stream', 'workers', id),
).resolves.toBeUndefined();
});
it('never trims unread or pending jobs even when MAXLEN is exceeded', async () => {
await redis.createConsumerGroup('stream', 'workers', '0');
const first = await redis.appendStream(
'stream',
{ job: 'pending' },
{ maxLen: 1 },
);
await redis.readGroup('stream', 'workers', 'worker', '>', { count: 1 });
for (let i = 0; i < 220; i++)
await redis.appendStream('stream', { job: String(i) }, { maxLen: 1 });
expect(await client.xLen('stream')).toBe(221);
expect((await client.xRange('stream', first, first))[0].message).toEqual({
job: 'pending',
});
});
it('does not release, renew, or recreate a lock belonging to another owner', async () => {
const locks = new DistributedLockService(redis);
const owner = await locks.acquireLock({
lockKey: 'lock',
lockOwnershipToken: 'old',
lockExpirationDurationMilliseconds: 1000,
});
expect(owner).not.toBeNull();
expect(await locks.extendLockIfOwned(owner!, 4000)).toBe(true);
expect(await client.pTTL('lock')).toBeGreaterThan(3000);
await client.set('lock', 'new', { PX: 5000 });
expect(await locks.releaseLockIfOwned(owner!)).toBe(false);
expect(await auctionRedis.releaseTokenLock('lock', 'old')).toBe(false);
expect(await locks.extendLockIfOwned(owner!, 9000)).toBe(false);
expect(await client.get('lock')).toBe('new');
expect(await auctionRedis.releaseTokenLock('lock', 'new')).toBe(true);
expect(await locks.extendLockIfOwned(owner!)).toBe(false);
expect(await client.exists('lock')).toBe(0);
});
it('does not clear the dedup token of a newer refresh job', async () => {
const job = {
jobId: 'old',
puuid: 'player',
queueGroup: 'summoners-rift',
} as const;
const key = 'riot:job:dedup:player:summoners-rift';
await redis.setJson(
'riot:job:status:old',
{ ...job, state: 'processing' },
86400,
);
await redis.setString(key, 'new', 300);
await queue.markCompleted(job as never);
expect(await redis.getString(key)).toBe('new');
await queue.markFailed(job as never, new Error('failure'));
expect(await redis.getString(key)).toBe('new');
await redis.setString(key, 'old', 300);
await queue.markCompleted(job as never);
expect(await redis.getString(key)).toBeNull();
});
it('increments failure counts without losing concurrent updates and refreshes the TTL', async () => {
const scheduler = Object.assign(
Object.create(BackgroundRefreshScheduler.prototype),
{
redis,
logger: { warn: jest.fn() },
},
) as {
bumpFailureCounter(puuid: string): Promise<void>;
clearFailureCounter(puuid: string): Promise<void>;
};
await client.set('riot:bg:fail:player', '1', { EX: 10 });
await Promise.all(
Array.from({ length: 10 }, () => scheduler.bumpFailureCounter('player')),
);
expect(await client.get('riot:bg:fail:player')).toBe('11');
expect(await client.ttl('riot:bg:fail:player')).toBeGreaterThan(86390);
await scheduler.clearFailureCounter('player');
expect(await client.exists('riot:bg:fail:player')).toBe(0);
});
});
beforeEach(async () => {
await client.flushDb();
});
afterAll(async () => {
if (client.isOpen) await client.quit();
});
function scheduledJob(priority = 'high', suffix = '1') {
return JSON.stringify({
kind: 'poll-tournament-result',
jobId: `job-${suffix}`,
matchId: 'match-1',
attempt: 1,
expandedToAllPuuids: false,
enqueuedAt: Date.now(),
priority,
});
}
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>((accept) => {
resolve = accept;
});
return { promise, resolve };
}
describe('Community Hash response cache', () => {
const cache = new CommunityStatsCacheService(redis);
it('does not share an old in-flight detail request after invalidation or overwrite the fresh response', async () => {
const service = new CommunityStatsService(
{} as never,
cache,
{} as never,
{} as never,
);
const communityId = new mongoose.Types.ObjectId();
const matchId = new mongoose.Types.ObjectId();
const dataset = (title: string) => ({
matches: [
{
matchId: matchId.toString(),
title,
scheduledAt: null,
status: 'finished',
gameCount: 0,
games: [],
teams: [],
seriesResults: [],
},
],
detailsByMatchId: new Map(),
});
const oldDataset = deferred<ReturnType<typeof dataset>>();
const oldStarted = deferred<void>();
const build = jest
.spyOn(
service as unknown as {
buildBaseDataset(): Promise<ReturnType<typeof dataset>>;
},
'buildBaseDataset',
)
.mockImplementationOnce(() => {
oldStarted.resolve();
return oldDataset.promise;
})
.mockResolvedValue(dataset('fresh'));
const oldRequest = service.getMatchDetail({ communityId, matchId });
await oldStarted.promise;
await cache.invalidateCommunity(communityId.toString());
try {
const fresh = await service.getMatchDetail({ communityId, matchId });
expect(fresh.match.title).toBe('fresh');
expect(build).toHaveBeenCalledTimes(2);
} finally {
oldDataset.resolve(dataset('old'));
await oldRequest;
}
expect((await oldRequest).match.title).toBe('old');
expect(
(await service.getMatchDetail({ communityId, matchId })).match.title,
).toBe('fresh');
expect(build).toHaveBeenCalledTimes(2);
});
it('preserves independent normal, detail and incomplete-detail TTLs', async () => {
const normal = new CommunityStatsCacheKey('community', 'matches', 'page-1');
const detail = new CommunityStatsCacheKey(
'community',
'matchDetail',
'match-1',
);
const incomplete = new CommunityStatsCacheKey(
'community',
'matchDetail',
'match-2',
);
for (const key of [normal, detail, incomplete])
expect(await cache.get(key)).toBeNull();
await cache.set(normal, { matches: [] });
await cache.setDetail(detail, { games: [1] });
await cache.set(incomplete, { games: [] }, 60);
for (const [key, ttl] of [
[normal, 300],
[detail, 86400],
[incomplete, 60],
] as const) {
const [remaining] = await client.sendCommand<number[]>([
'HTTL',
key.hashKey,
'FIELDS',
'1',
key.field,
]);
expect(remaining).toBeGreaterThan(ttl - 3);
expect(remaining).toBeLessThanOrEqual(ttl);
}
expect(await cache.get(detail)).toEqual({ games: [1] });
expect(await client.ttl(detail.hashKey)).toBe(-1);
await client.sendCommand([
'HPEXPIRE',
incomplete.hashKey,
'20',
'FIELDS',
'1',
incomplete.field,
]);
await delay(50);
expect(await cache.get(incomplete)).toBeNull();
expect(await cache.get(detail)).toEqual({ games: [1] });
});
it('invalidates all scopes for one community while preserving other communities', async () => {
const entries = (
[
'matches',
'leaderboard',
'teamInsights',
'matchDetail',
'userInternal',
] as const
).map((scope) => new CommunityStatsCacheKey('one', scope, 'field'));
const other = new CommunityStatsCacheKey('two', 'matches', 'field');
for (const key of [...entries, other]) {
await cache.get(key);
await cache.set(key, { fresh: true });
}
expect(await cache.invalidateCommunity('one')).toBe(5);
for (const key of entries) expect(await cache.get(key)).toBeNull();
expect(await cache.get(other)).toEqual({ fresh: true });
expect(await cache.invalidateCommunity('one')).toBe(0);
});
it('rejects stale fills and separates in-flight identities across invalidation', async () => {
const old = new CommunityStatsCacheKey('one', 'matches', 'page');
await cache.get(old);
const oldIdentity = old.identity;
await cache.invalidateCommunity('one');
const fresh = new CommunityStatsCacheKey('one', 'matches', 'page');
await cache.get(fresh);
expect(fresh.identity).not.toBe(oldIdentity);
await cache.set(fresh, { version: 2 });
await cache.set(old, { version: 1 });
expect(await cache.get(fresh)).toEqual({ version: 2 });
await cache.invalidateCommunity('one');
await cache.set(old, { version: 0 });
expect(await client.exists(old.hashKey)).toBe(0);
});
it('treats malformed data as a miss and refreshes only the written field TTL', async () => {
const key = new CommunityStatsCacheKey('one', 'matches', 'page');
await cache.get(key);
await client.hSet(key.hashKey, key.field, 'broken-json');
expect(await cache.get(key)).toBeNull();
await cache.set(key, { ok: true }, 1);
await cache.set(key, { ok: true }, 300);
const [ttl] = await client.sendCommand<number[]>([
'HTTL',
key.hashKey,
'FIELDS',
'1',
key.field,
]);
expect(ttl).toBeGreaterThan(297);
expect(await cache.get(key)).toEqual({ ok: true });
});
});
describe('Redis correctness regressions', () => {
it('rechecks the due timestamp after selection so rescheduled work is not promoted early', async () => {
const job = scheduledJob();
const selectedAt = Date.now();
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
score: selectedAt + 60000,
value: job,
});
expect(
await client.sendCommand([
'EVAL',
PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT,
'2',
RIOT_JOB_SCHEDULED_ZSET,
RIOT_JOB_HIGH_STREAM,
job,
String(selectedAt),
]),
).toBe(0);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('discards malformed scheduled jobs and keeps the requested promotion batch limit', async () => {
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, [
{ score: 0, value: 'invalid-json' },
{ score: 0, value: JSON.stringify({ kind: 'unsupported' }) },
...Array.from({ length: 3 }, (_, i) => ({
score: 1,
value: scheduledJob('high', String(i)),
})),
]);
expect(await queue.promoteDueScheduledJobs(2)).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(3);
expect(await queue.promoteDueScheduledJobs(2)).toBe(2);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(2);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('keeps a scheduled job when destination append fails', async () => {
const job = scheduledJob();
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
score: Date.now() - 1,
value: job,
});
await client.set(RIOT_JOB_HIGH_STREAM, 'wrong-type');
await expect(queue.promoteDueScheduledJobs()).rejects.toThrow('WRONGTYPE');
expect(await client.zScore(RIOT_JOB_SCHEDULED_ZSET, job)).not.toBeNull();
await client.del(RIOT_JOB_HIGH_STREAM);
expect(await queue.promoteDueScheduledJobs()).toBe(1);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(1);
});
it('promotes each due job once under concurrent workers and leaves future jobs scheduled', async () => {
const jobs = Array.from({ length: 12 }, (_, i) =>
scheduledJob(i % 2 ? 'low' : 'high', String(i)),
);
await client.zAdd(
RIOT_JOB_SCHEDULED_ZSET,
jobs.map((value) => ({ score: Date.now() - 1, value })),
);
const future = scheduledJob('high', 'future');
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
score: Date.now() + 60_000,
value: future,
});
const counts = await Promise.all(
Array.from({ length: 4 }, () => queue.promoteDueScheduledJobs(5)),
);
await queue.promoteDueScheduledJobs();
expect(counts.reduce((sum, value) => sum + value, 0)).toBeLessThanOrEqual(
12,
);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(6);
expect(await client.xLen(RIOT_JOB_LOW_STREAM)).toBe(6);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('cleans only the requested lifecycle Hash payloads and queue references', async () => {
const keys = buildAuctionRedisKeys('match-1');
const oldKey = keys.durabilityPayloadKey('old');
const newKey = keys.durabilityPayloadKey('new');
for (const [key, lifecycle] of [
[oldKey, 'old-life'],
[newKey, 'new-life'],
]) {
await client.hSet(key, {
auctionLifecycleId: lifecycle,
payloadJson: JSON.stringify({ auctionLifecycleId: lifecycle }),
});
await client.zAdd(keys.durabilityRetryQueueKey, { score: 1, value: key });
await client.zAdd(keys.durabilityDeadLetterQueueKey, {
score: 1,
value: key,
});
}
const service = Object.assign(Object.create(AuctionService.prototype), {
auctionRedisService: auctionRedis,
}) as {
clearRedisDurabilityPayloadsForLifecycle(
keys: ReturnType<typeof buildAuctionRedisKeys>,
lifecycle: string,
): Promise<void>;
};
await service.clearRedisDurabilityPayloadsForLifecycle(keys, 'old-life');
expect(await client.exists(oldKey)).toBe(0);
expect(await client.exists(newKey)).toBe(1);
expect(await client.zRange(keys.durabilityRetryQueueKey, 0, -1)).toEqual([
newKey,
]);
expect(
await client.zRange(keys.durabilityDeadLetterQueueKey, 0, -1),
).toEqual([newKey]);
});
it('supports legacy JSON cleanup and preserves malformed or foreign payloads until an unconditional cleanup', async () => {
const keys = buildAuctionRedisKeys('legacy');
const legacy = keys.durabilityPayloadKey('old');
const foreign = keys.durabilityPayloadKey('foreign');
const malformed = keys.durabilityPayloadKey('malformed');
await client.set(
legacy,
JSON.stringify({ auctionLifecycleId: 'old-life' }),
);
await client.set(
foreign,
JSON.stringify({ auctionLifecycleId: 'new-life' }),
);
await client.set(malformed, 'broken-json');
await client.zAdd(
keys.durabilityRetryQueueKey,
[legacy, foreign, malformed].map((value) => ({ score: 1, value })),
);
const service = Object.assign(Object.create(AuctionService.prototype), {
auctionRedisService: auctionRedis,
}) as {
clearRedisDurabilityPayloadsForLifecycle(
keys: ReturnType<typeof buildAuctionRedisKeys>,
lifecycle?: string,
): Promise<void>;
};
await service.clearRedisDurabilityPayloadsForLifecycle(keys, 'old-life');
expect(await client.exists(legacy)).toBe(0);
expect(await client.exists([foreign, malformed])).toBe(2);
await service.clearRedisDurabilityPayloadsForLifecycle(keys);
expect(await client.exists([foreign, malformed])).toBe(0);
expect(await client.zCard(keys.durabilityRetryQueueKey)).toBe(0);
});
it('preserves the 500-request cap, replay window, and fingerprint conflict checks', async () => {
const keys = buildAuctionRedisKeys('retention');
const reserve = async (id: string, revision = 1) => {
const invocation = buildAuctionReserveActionRequestScriptInvocation(
keys,
{
auctionActionRequestId: id,
auctionActionType: 'next',
auctionStateTransitionId: `transition-${id}`,
clientObservedAuctionLifecycleId: 'life',
clientObservedPhase: 'sold',
clientObservedRevision: revision,
leaseDurationMS: 15000,
maximumStoredAuctionActionRequestResultCount: 500,
minimumAuctionActionRequestResultReplayWindowMS: 300000,
processingOwnerToken: 'owner',
},
);
return auctionRedis.evalScript<string[]>(
invocation.script,
invocation.keys,
invocation.values,
);
};
for (let i = 0; i < 500; i++)
expect((await reserve(String(i)))[0]).toBe('RESERVED');
expect((await reserve('0', 2)).slice(0, 2)).toEqual([
'REJECTED',
'DUPLICATE_ACTION_REQUEST_CONFLICT',
]);
expect((await reserve('501')).slice(0, 2)).toEqual([
'REJECTED',
'ACTION_REQUEST_RESULT_LIMIT',
]);
expect(await client.lLen(keys.actionRequestResultOrderKey)).toBe(500);
expect((await reserve('0'))[0]).toBe('RESERVED');
expect(await client.lLen(keys.actionRequestResultOrderKey)).toBe(500);
await client.hSet(
keys.actionRequestResultsKey,
'life:0:createdAtTimestampMS',
String(Date.now() - 300001),
);
expect((await reserve('501'))[0]).toBe('RESERVED');
expect(await client.lLen(keys.actionRequestResultOrderKey)).toBe(500);
expect(
await client.hExists(keys.actionRequestResultsKey, 'life:0:fingerprint'),
).toBe(0);
});
it('counts one order entry per reserved and applied action and still replays it', async () => {
const keys = buildAuctionRedisKeys('match-1');
const commands = new AuctionRedisCommandService(auctionRedis);
await commands.hydrateState('match-1', {
matchId: 'match-1',
auctionHistorySequence: 0,
auctionLifecycleId: 'life',
phase: 'sold',
revision: 0,
maximumBidRoundDurationSeconds: 30,
structureStateJson: '{}',
});
const reservation = buildAuctionReserveActionRequestScriptInvocation(keys, {
auctionActionRequestId: 'request',
auctionActionType: 'next',
auctionStateTransitionId: 'transition',
clientObservedAuctionLifecycleId: 'life',
clientObservedPhase: 'sold',
clientObservedRevision: 1,
leaseDurationMS: 15000,
maximumStoredAuctionActionRequestResultCount: 500,
minimumAuctionActionRequestResultReplayWindowMS: 300000,
processingOwnerToken: 'owner',
});
await auctionRedis.evalScript(
reservation.script,
reservation.keys,
reservation.values,
);
const apply = buildAuctionApplyTransitionScriptInvocation(keys, {
actionRequest: {
scopeKey: 'life:request',
processingOwnerToken: 'owner',
observedProcessingLeaseVersion: 1,
},
afterAuctionStructureStateJson: '{}',
auctionHistorySequence: 1,
auctionLifecycleId: 'life',
auctionStateTransitionId: 'transition',
broadcastStateRevision: 2,
deadline: { mode: 'none' },
durabilityPayloadJson: '{}',
expectedPhase: 'sold',
expectedRevision: 1,
liveStatePatch: { phase: 'sold' },
matchId: 'match-1',
shouldBroadcastState: false,
});
const result = await auctionRedis.evalScript<string[]>(
apply.script,
apply.keys,
apply.values,
);
expect(result[0]).toBe('OK');
expect(
await client.lRange(keys.actionRequestResultOrderKey, 0, -1),
).toEqual(['life:request']);
const replay = await auctionRedis.evalScript<string[]>(
reservation.script,
reservation.keys,
reservation.values,
);
expect(replay.slice(0, 2)).toEqual(['REPLAY', 'applied']);
});
});
+120
View File
@@ -0,0 +1,120 @@
import { spawn } from 'node:child_process';
import { createRequire } from 'node:module';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, relative, resolve } from 'node:path';
import { createServer } from 'node:net';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { createClient } from 'redis';
const require = createRequire(import.meta.url);
const listener = createServer();
await new Promise((accept, reject) => {
listener.once('error', reject);
listener.listen(0, '127.0.0.1', accept);
});
const port = listener.address().port;
await new Promise((accept) => listener.close(accept));
const directory = await mkdtemp(join(tmpdir(), 'lolympus-redis-test-'));
const password = randomUUID();
const redis = spawn(
process.env.REDIS_TEST_SERVER || 'redis-server',
[
'--bind',
'127.0.0.1',
'--port',
String(port),
'--protected-mode',
'yes',
'--requirepass',
password,
'--save',
'',
'--appendonly',
'no',
'--dir',
directory.replaceAll('\\', '/'),
],
{ windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] },
);
let startupError;
const redisClosed = new Promise((accept) => redis.once('close', accept));
let output = '';
redis.on('error', (error) => {
startupError = error;
});
redis.stdout.on('data', (data) => {
output += data;
});
redis.stderr.on('data', (data) => {
output += data;
});
const url = `redis://:${password}@127.0.0.1:${port}`;
const client = createClient({
url,
socket: { reconnectStrategy: false, connectTimeout: 500 },
});
client.on('error', () => {});
try {
let connected = false;
for (let attempt = 0; attempt < 100; attempt++) {
if (startupError) throw startupError;
if (redis.exitCode !== null) throw new Error(`Redis exited: ${output}`);
try {
await client.connect();
connected = true;
break;
} catch {
await delay(100);
}
}
if (!connected) throw new Error(`Redis did not start: ${output}`);
const version = (await client.info('server')).match(
/redis_version:([^\r\n]+)/,
)?.[1];
if (!version?.startsWith('8.10.'))
throw new Error(`Expected Redis 8.10.x, got ${version}`);
console.log(
`Integration tests using isolated Redis ${version} on loopback port ${port}`,
);
const jest = spawn(
process.execPath,
[
require.resolve('jest/bin/jest'),
'--config',
'test/jest-redis.json',
'--runInBand',
...process.argv.slice(2),
],
{
cwd: resolve(import.meta.dirname, '..'),
windowsHide: true,
stdio: 'inherit',
env: {
...process.env,
NODE_ENV: 'test',
REDIS_TEST_URL: url,
REDIS_TEST_ISOLATED: '1',
},
},
);
process.exitCode = await new Promise((accept, reject) => {
jest.once('error', reject);
jest.once('exit', (code) => accept(code ?? 1));
});
} finally {
if (client.isOpen) {
await client.sendCommand(['SHUTDOWN', 'NOSAVE']).catch(() => {});
if (client.isOpen) client.destroy();
}
if (redis.pid && redis.exitCode === null) redis.kill();
await redisClosed;
const cleanupTarget = relative(resolve(tmpdir()), resolve(directory));
if (!/^lolympus-redis-test-[^\\/]+$/.test(cleanupTarget)) {
throw new Error(
'Refusing to clean up a directory outside the isolated test workspace.',
);
}
await rm(directory, { recursive: true, force: true });
}
@@ -0,0 +1,284 @@
import { createClient } from 'redis';
import { RedisService } from '../src/modules/global/redis/redis.service';
import { RiotJobQueueService } from '../src/modules/riot-job-queue/riot-job-queue.service';
import {
RIOT_JOB_GROUP,
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_STREAM,
RIOT_JOB_SCHEDULED_ZSET,
} from '../src/modules/riot-job-queue/riot-job-queue.types';
import type { PollTournamentResultJob } from '../src/modules/tournament-result/tournament-result.types';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new RedisService(client);
const queue = new RiotJobQueueService({} as never, redis);
beforeAll(async () => {
const url = process.env.REDIS_TEST_URL;
if (
process.env.REDIS_TEST_ISOLATED !== '1' ||
!url ||
!['127.0.0.1', 'localhost', '[::1]'].includes(new URL(url).hostname)
) {
throw new Error('Run with the isolated loopback Redis integration runner.');
}
client.on('error', () => {});
await client.connect();
});
beforeEach(async () => {
await client.flushDb();
});
afterAll(async () => {
if (client.isOpen) await client.quit();
});
describe('Tournament result retry delivery on Redis', () => {
it('replaces a pending poll with one durable retry and ignores repeated transfer requests', async () => {
const job = pollJob();
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
const before = Date.now();
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
const after = Date.now();
const scheduled = await client.zRangeWithScores(
RIOT_JOB_SCHEDULED_ZSET,
0,
-1,
);
expect(scheduled).toHaveLength(1);
expect(JSON.parse(scheduled[0].value)).toEqual({ ...job, retryCount: 1 });
expect(scheduled[0].score).toBeGreaterThanOrEqual(before + 30_000);
expect(scheduled[0].score).toBeLessThanOrEqual(after + 30_000);
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(0);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(0);
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
expect(
await client.zRangeWithScores(RIOT_JOB_SCHEDULED_ZSET, 0, -1),
).toEqual(scheduled);
});
it.each([
[1, 60_000],
[2, 120_000],
[3, 240_000],
[4, 300_000],
[20, 300_000],
])(
'backs off retry count %i by %i ms without changing the poll attempt',
async (retryCount, delayMs) => {
const job = { ...pollJob(), retryCount };
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
const before = Date.now();
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
const after = Date.now();
const scheduled = await client.zRangeWithScores(
RIOT_JOB_SCHEDULED_ZSET,
0,
-1,
);
expect(scheduled).toHaveLength(1);
expect(JSON.parse(scheduled[0].value)).toEqual({
...job,
retryCount: retryCount + 1,
});
expect(scheduled[0].score).toBeGreaterThanOrEqual(before + delayMs);
expect(scheduled[0].score).toBeLessThanOrEqual(after + delayMs);
},
);
it('keeps the original pending when retry persistence fails and can recover afterward', async () => {
const job = pollJob();
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
await client.set(RIOT_JOB_SCHEDULED_ZSET, 'wrong-type');
await expect(
queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
}),
).rejects.toThrow();
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(1);
expect(await client.xRange(RIOT_JOB_HIGH_STREAM, entryId, entryId)).toEqual(
[{ id: entryId, message: { job: JSON.stringify(job) } }],
);
await client.del(RIOT_JOB_SCHEDULED_ZSET);
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('retains the original for another consumer group until that group acknowledges it', async () => {
const job = pollJob();
const stream = RIOT_JOB_HIGH_STREAM;
await redis.createConsumerGroup(stream, 'audit', '0');
const entryId = await pendingPoll(stream, job);
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0);
expect(await client.xLen(stream)).toBe(1);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
const auditEntry = await redis.readGroup(stream, 'audit', 'auditor', '>', {
count: 1,
});
expect(auditEntry?.[0].messages[0].id).toBe(entryId);
await redis.acknowledgeAndDeleteStream(stream, 'audit', entryId);
expect(await client.xLen(stream)).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it.each([
['high', RIOT_JOB_HIGH_STREAM],
['low', RIOT_JOB_LOW_STREAM],
] as const)(
'promotes a due %s retry to its original priority and allows successful consumption',
async (priority, stream) => {
const job = { ...pollJob(), priority };
const entryId = await pendingPoll(stream, job);
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
const [scheduled] = await client.zRangeWithScores(
RIOT_JOB_SCHEDULED_ZSET,
0,
-1,
);
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
value: scheduled.value,
score: Date.now() - 1,
});
expect(await queue.promoteDueScheduledJobs()).toBe(1);
const delivery = await redis.readGroup(
stream,
RIOT_JOB_GROUP,
'recovered-worker',
'>',
{ count: 1 },
);
const retriedEntry = delivery?.[0].messages[0];
expect(retriedEntry).toBeDefined();
expect(JSON.parse(retriedEntry!.message.job)).toEqual({
...job,
retryCount: 1,
});
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0);
const otherStream =
priority === 'high' ? RIOT_JOB_LOW_STREAM : RIOT_JOB_HIGH_STREAM;
expect(await client.xLen(otherStream)).toBe(0);
await redis.acknowledgeAndDeleteStream(
stream,
RIOT_JOB_GROUP,
retriedEntry!.id,
);
expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0);
expect(await client.xLen(stream)).toBe(0);
},
);
it('retains pending work during a disconnected retry client and recovers with a healthy connection', async () => {
const job = pollJob();
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
const offlineClient = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const offlineQueue = new RiotJobQueueService(
{} as never,
new RedisService(offlineClient),
);
await expect(
offlineQueue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
}),
).rejects.toThrow();
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(1);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(1);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0);
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
});
function pollJob(): PollTournamentResultJob {
return {
kind: 'poll-tournament-result',
jobId: 'original-result-job',
matchId: '6aeecacdffed1eb545b8c0f3',
attempt: 7,
expandedToAllPuuids: true,
enqueuedAt: 1_794_053_383_203,
priority: 'high',
};
}
async function pendingPoll(
stream: string,
job: PollTournamentResultJob,
): Promise<string> {
await redis.createConsumerGroup(stream, RIOT_JOB_GROUP, '0');
const entryId = await redis.appendStream(stream, {
job: JSON.stringify(job),
});
const delivery = await redis.readGroup(
stream,
RIOT_JOB_GROUP,
'original-worker',
'>',
{ count: 1 },
);
expect(delivery?.[0].messages[0].id).toBe(entryId);
return entryId;
}