Preserve tournament result jobs through lock contention and transient failures, atomically schedule retries before acknowledgement, and publish initial polls after releasing the preparation lock. Fix Redis connection blocking, durability Hash cleanup, duplicate auction request ordering, and scheduled-job promotion. Use native Redis 8.10 stream, lock, counter, and field-expiring cache operations with invalidation fencing. Repair 90 test type errors and add a repeatable API typecheck command. Validation: 704 API tests, 129 web tests, 31 real Redis integration tests, 3 dependency checks, workspace type checks and builds, and changed-file lint.
285 lines
8.7 KiB
TypeScript
285 lines
8.7 KiB
TypeScript
import { createClient } from 'redis';
|
|
import { RedisService } from '../src/modules/global/redis/redis.service';
|
|
import { RiotJobQueueService } from '../src/modules/riot-job-queue/riot-job-queue.service';
|
|
import {
|
|
RIOT_JOB_GROUP,
|
|
RIOT_JOB_HIGH_STREAM,
|
|
RIOT_JOB_LOW_STREAM,
|
|
RIOT_JOB_SCHEDULED_ZSET,
|
|
} from '../src/modules/riot-job-queue/riot-job-queue.types';
|
|
import type { PollTournamentResultJob } from '../src/modules/tournament-result/tournament-result.types';
|
|
|
|
const client = createClient({
|
|
url: process.env.REDIS_TEST_URL,
|
|
socket: { reconnectStrategy: false },
|
|
});
|
|
const redis = new RedisService(client);
|
|
const queue = new RiotJobQueueService({} as never, redis);
|
|
|
|
beforeAll(async () => {
|
|
const url = process.env.REDIS_TEST_URL;
|
|
if (
|
|
process.env.REDIS_TEST_ISOLATED !== '1' ||
|
|
!url ||
|
|
!['127.0.0.1', 'localhost', '[::1]'].includes(new URL(url).hostname)
|
|
) {
|
|
throw new Error('Run with the isolated loopback Redis integration runner.');
|
|
}
|
|
client.on('error', () => {});
|
|
await client.connect();
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
await client.flushDb();
|
|
});
|
|
|
|
afterAll(async () => {
|
|
if (client.isOpen) await client.quit();
|
|
});
|
|
|
|
describe('Tournament result retry delivery on Redis', () => {
|
|
it('replaces a pending poll with one durable retry and ignores repeated transfer requests', async () => {
|
|
const job = pollJob();
|
|
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
|
|
const before = Date.now();
|
|
|
|
await queue.rescheduleTournamentResultPoll({
|
|
stream: RIOT_JOB_HIGH_STREAM,
|
|
entryId,
|
|
job,
|
|
});
|
|
|
|
const after = Date.now();
|
|
const scheduled = await client.zRangeWithScores(
|
|
RIOT_JOB_SCHEDULED_ZSET,
|
|
0,
|
|
-1,
|
|
);
|
|
expect(scheduled).toHaveLength(1);
|
|
expect(JSON.parse(scheduled[0].value)).toEqual({ ...job, retryCount: 1 });
|
|
expect(scheduled[0].score).toBeGreaterThanOrEqual(before + 30_000);
|
|
expect(scheduled[0].score).toBeLessThanOrEqual(after + 30_000);
|
|
expect(
|
|
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
|
|
).toBe(0);
|
|
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(0);
|
|
|
|
await queue.rescheduleTournamentResultPoll({
|
|
stream: RIOT_JOB_HIGH_STREAM,
|
|
entryId,
|
|
job,
|
|
});
|
|
|
|
expect(
|
|
await client.zRangeWithScores(RIOT_JOB_SCHEDULED_ZSET, 0, -1),
|
|
).toEqual(scheduled);
|
|
});
|
|
|
|
it.each([
|
|
[1, 60_000],
|
|
[2, 120_000],
|
|
[3, 240_000],
|
|
[4, 300_000],
|
|
[20, 300_000],
|
|
])(
|
|
'backs off retry count %i by %i ms without changing the poll attempt',
|
|
async (retryCount, delayMs) => {
|
|
const job = { ...pollJob(), retryCount };
|
|
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
|
|
const before = Date.now();
|
|
|
|
await queue.rescheduleTournamentResultPoll({
|
|
stream: RIOT_JOB_HIGH_STREAM,
|
|
entryId,
|
|
job,
|
|
});
|
|
|
|
const after = Date.now();
|
|
const scheduled = await client.zRangeWithScores(
|
|
RIOT_JOB_SCHEDULED_ZSET,
|
|
0,
|
|
-1,
|
|
);
|
|
expect(scheduled).toHaveLength(1);
|
|
expect(JSON.parse(scheduled[0].value)).toEqual({
|
|
...job,
|
|
retryCount: retryCount + 1,
|
|
});
|
|
expect(scheduled[0].score).toBeGreaterThanOrEqual(before + delayMs);
|
|
expect(scheduled[0].score).toBeLessThanOrEqual(after + delayMs);
|
|
},
|
|
);
|
|
|
|
it('keeps the original pending when retry persistence fails and can recover afterward', async () => {
|
|
const job = pollJob();
|
|
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
|
|
await client.set(RIOT_JOB_SCHEDULED_ZSET, 'wrong-type');
|
|
|
|
await expect(
|
|
queue.rescheduleTournamentResultPoll({
|
|
stream: RIOT_JOB_HIGH_STREAM,
|
|
entryId,
|
|
job,
|
|
}),
|
|
).rejects.toThrow();
|
|
|
|
expect(
|
|
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
|
|
).toBe(1);
|
|
expect(await client.xRange(RIOT_JOB_HIGH_STREAM, entryId, entryId)).toEqual(
|
|
[{ id: entryId, message: { job: JSON.stringify(job) } }],
|
|
);
|
|
await client.del(RIOT_JOB_SCHEDULED_ZSET);
|
|
|
|
await queue.rescheduleTournamentResultPoll({
|
|
stream: RIOT_JOB_HIGH_STREAM,
|
|
entryId,
|
|
job,
|
|
});
|
|
|
|
expect(
|
|
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
|
|
).toBe(0);
|
|
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
|
|
});
|
|
|
|
it('retains the original for another consumer group until that group acknowledges it', async () => {
|
|
const job = pollJob();
|
|
const stream = RIOT_JOB_HIGH_STREAM;
|
|
await redis.createConsumerGroup(stream, 'audit', '0');
|
|
const entryId = await pendingPoll(stream, job);
|
|
|
|
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
|
|
|
|
expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0);
|
|
expect(await client.xLen(stream)).toBe(1);
|
|
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
|
|
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
|
|
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
|
|
|
|
const auditEntry = await redis.readGroup(stream, 'audit', 'auditor', '>', {
|
|
count: 1,
|
|
});
|
|
expect(auditEntry?.[0].messages[0].id).toBe(entryId);
|
|
await redis.acknowledgeAndDeleteStream(stream, 'audit', entryId);
|
|
expect(await client.xLen(stream)).toBe(0);
|
|
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
|
|
});
|
|
|
|
it.each([
|
|
['high', RIOT_JOB_HIGH_STREAM],
|
|
['low', RIOT_JOB_LOW_STREAM],
|
|
] as const)(
|
|
'promotes a due %s retry to its original priority and allows successful consumption',
|
|
async (priority, stream) => {
|
|
const job = { ...pollJob(), priority };
|
|
const entryId = await pendingPoll(stream, job);
|
|
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
|
|
const [scheduled] = await client.zRangeWithScores(
|
|
RIOT_JOB_SCHEDULED_ZSET,
|
|
0,
|
|
-1,
|
|
);
|
|
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
|
|
value: scheduled.value,
|
|
score: Date.now() - 1,
|
|
});
|
|
|
|
expect(await queue.promoteDueScheduledJobs()).toBe(1);
|
|
|
|
const delivery = await redis.readGroup(
|
|
stream,
|
|
RIOT_JOB_GROUP,
|
|
'recovered-worker',
|
|
'>',
|
|
{ count: 1 },
|
|
);
|
|
const retriedEntry = delivery?.[0].messages[0];
|
|
expect(retriedEntry).toBeDefined();
|
|
expect(JSON.parse(retriedEntry!.message.job)).toEqual({
|
|
...job,
|
|
retryCount: 1,
|
|
});
|
|
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0);
|
|
const otherStream =
|
|
priority === 'high' ? RIOT_JOB_LOW_STREAM : RIOT_JOB_HIGH_STREAM;
|
|
expect(await client.xLen(otherStream)).toBe(0);
|
|
|
|
await redis.acknowledgeAndDeleteStream(
|
|
stream,
|
|
RIOT_JOB_GROUP,
|
|
retriedEntry!.id,
|
|
);
|
|
expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0);
|
|
expect(await client.xLen(stream)).toBe(0);
|
|
},
|
|
);
|
|
|
|
it('retains pending work during a disconnected retry client and recovers with a healthy connection', async () => {
|
|
const job = pollJob();
|
|
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
|
|
const offlineClient = createClient({
|
|
url: process.env.REDIS_TEST_URL,
|
|
socket: { reconnectStrategy: false },
|
|
});
|
|
const offlineQueue = new RiotJobQueueService(
|
|
{} as never,
|
|
new RedisService(offlineClient),
|
|
);
|
|
|
|
await expect(
|
|
offlineQueue.rescheduleTournamentResultPoll({
|
|
stream: RIOT_JOB_HIGH_STREAM,
|
|
entryId,
|
|
job,
|
|
}),
|
|
).rejects.toThrow();
|
|
|
|
expect(
|
|
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
|
|
).toBe(1);
|
|
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(1);
|
|
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0);
|
|
await queue.rescheduleTournamentResultPoll({
|
|
stream: RIOT_JOB_HIGH_STREAM,
|
|
entryId,
|
|
job,
|
|
});
|
|
expect(
|
|
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
|
|
).toBe(0);
|
|
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
|
|
});
|
|
});
|
|
|
|
function pollJob(): PollTournamentResultJob {
|
|
return {
|
|
kind: 'poll-tournament-result',
|
|
jobId: 'original-result-job',
|
|
matchId: '6aeecacdffed1eb545b8c0f3',
|
|
attempt: 7,
|
|
expandedToAllPuuids: true,
|
|
enqueuedAt: 1_794_053_383_203,
|
|
priority: 'high',
|
|
};
|
|
}
|
|
|
|
async function pendingPoll(
|
|
stream: string,
|
|
job: PollTournamentResultJob,
|
|
): Promise<string> {
|
|
await redis.createConsumerGroup(stream, RIOT_JOB_GROUP, '0');
|
|
const entryId = await redis.appendStream(stream, {
|
|
job: JSON.stringify(job),
|
|
});
|
|
const delivery = await redis.readGroup(
|
|
stream,
|
|
RIOT_JOB_GROUP,
|
|
'original-worker',
|
|
'>',
|
|
{ count: 1 },
|
|
);
|
|
expect(delivery?.[0].messages[0].id).toBe(entryId);
|
|
return entryId;
|
|
}
|