import { createClient } from 'redis'; import { RedisService } from '../src/modules/global/redis/redis.service'; import { RiotJobQueueService } from '../src/modules/riot-job-queue/riot-job-queue.service'; import { RIOT_JOB_GROUP, RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM, RIOT_JOB_SCHEDULED_ZSET, } from '../src/modules/riot-job-queue/riot-job-queue.types'; import type { PollTournamentResultJob } from '../src/modules/tournament-result/tournament-result.types'; const client = createClient({ url: process.env.REDIS_TEST_URL, socket: { reconnectStrategy: false }, }); const redis = new RedisService(client); const queue = new RiotJobQueueService({} as never, redis); beforeAll(async () => { const url = process.env.REDIS_TEST_URL; if ( process.env.REDIS_TEST_ISOLATED !== '1' || !url || !['127.0.0.1', 'localhost', '[::1]'].includes(new URL(url).hostname) ) { throw new Error('Run with the isolated loopback Redis integration runner.'); } client.on('error', () => {}); await client.connect(); }); beforeEach(async () => { await client.flushDb(); }); afterAll(async () => { if (client.isOpen) await client.quit(); }); describe('Tournament result retry delivery on Redis', () => { it('replaces a pending poll with one durable retry and ignores repeated transfer requests', async () => { const job = pollJob(); const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job); const before = Date.now(); await queue.rescheduleTournamentResultPoll({ stream: RIOT_JOB_HIGH_STREAM, entryId, job, }); const after = Date.now(); const scheduled = await client.zRangeWithScores( RIOT_JOB_SCHEDULED_ZSET, 0, -1, ); expect(scheduled).toHaveLength(1); expect(JSON.parse(scheduled[0].value)).toEqual({ ...job, retryCount: 1 }); expect(scheduled[0].score).toBeGreaterThanOrEqual(before + 30_000); expect(scheduled[0].score).toBeLessThanOrEqual(after + 30_000); expect( (await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending, ).toBe(0); expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(0); await queue.rescheduleTournamentResultPoll({ stream: RIOT_JOB_HIGH_STREAM, entryId, job, }); expect( await client.zRangeWithScores(RIOT_JOB_SCHEDULED_ZSET, 0, -1), ).toEqual(scheduled); }); it.each([ [1, 60_000], [2, 120_000], [3, 240_000], [4, 300_000], [20, 300_000], ])( 'backs off retry count %i by %i ms without changing the poll attempt', async (retryCount, delayMs) => { const job = { ...pollJob(), retryCount }; const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job); const before = Date.now(); await queue.rescheduleTournamentResultPoll({ stream: RIOT_JOB_HIGH_STREAM, entryId, job, }); const after = Date.now(); const scheduled = await client.zRangeWithScores( RIOT_JOB_SCHEDULED_ZSET, 0, -1, ); expect(scheduled).toHaveLength(1); expect(JSON.parse(scheduled[0].value)).toEqual({ ...job, retryCount: retryCount + 1, }); expect(scheduled[0].score).toBeGreaterThanOrEqual(before + delayMs); expect(scheduled[0].score).toBeLessThanOrEqual(after + delayMs); }, ); it('keeps the original pending when retry persistence fails and can recover afterward', async () => { const job = pollJob(); const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job); await client.set(RIOT_JOB_SCHEDULED_ZSET, 'wrong-type'); await expect( queue.rescheduleTournamentResultPoll({ stream: RIOT_JOB_HIGH_STREAM, entryId, job, }), ).rejects.toThrow(); expect( (await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending, ).toBe(1); expect(await client.xRange(RIOT_JOB_HIGH_STREAM, entryId, entryId)).toEqual( [{ id: entryId, message: { job: JSON.stringify(job) } }], ); await client.del(RIOT_JOB_SCHEDULED_ZSET); await queue.rescheduleTournamentResultPoll({ stream: RIOT_JOB_HIGH_STREAM, entryId, job, }); expect( (await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending, ).toBe(0); expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1); }); it('retains the original for another consumer group until that group acknowledges it', async () => { const job = pollJob(); const stream = RIOT_JOB_HIGH_STREAM; await redis.createConsumerGroup(stream, 'audit', '0'); const entryId = await pendingPoll(stream, job); await queue.rescheduleTournamentResultPoll({ stream, entryId, job }); expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0); expect(await client.xLen(stream)).toBe(1); expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1); await queue.rescheduleTournamentResultPoll({ stream, entryId, job }); expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1); const auditEntry = await redis.readGroup(stream, 'audit', 'auditor', '>', { count: 1, }); expect(auditEntry?.[0].messages[0].id).toBe(entryId); await redis.acknowledgeAndDeleteStream(stream, 'audit', entryId); expect(await client.xLen(stream)).toBe(0); expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1); }); it.each([ ['high', RIOT_JOB_HIGH_STREAM], ['low', RIOT_JOB_LOW_STREAM], ] as const)( 'promotes a due %s retry to its original priority and allows successful consumption', async (priority, stream) => { const job = { ...pollJob(), priority }; const entryId = await pendingPoll(stream, job); await queue.rescheduleTournamentResultPoll({ stream, entryId, job }); const [scheduled] = await client.zRangeWithScores( RIOT_JOB_SCHEDULED_ZSET, 0, -1, ); await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, { value: scheduled.value, score: Date.now() - 1, }); expect(await queue.promoteDueScheduledJobs()).toBe(1); const delivery = await redis.readGroup( stream, RIOT_JOB_GROUP, 'recovered-worker', '>', { count: 1 }, ); const retriedEntry = delivery?.[0].messages[0]; expect(retriedEntry).toBeDefined(); expect(JSON.parse(retriedEntry!.message.job)).toEqual({ ...job, retryCount: 1, }); expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0); const otherStream = priority === 'high' ? RIOT_JOB_LOW_STREAM : RIOT_JOB_HIGH_STREAM; expect(await client.xLen(otherStream)).toBe(0); await redis.acknowledgeAndDeleteStream( stream, RIOT_JOB_GROUP, retriedEntry!.id, ); expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0); expect(await client.xLen(stream)).toBe(0); }, ); it('retains pending work during a disconnected retry client and recovers with a healthy connection', async () => { const job = pollJob(); const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job); const offlineClient = createClient({ url: process.env.REDIS_TEST_URL, socket: { reconnectStrategy: false }, }); const offlineQueue = new RiotJobQueueService( {} as never, new RedisService(offlineClient), ); await expect( offlineQueue.rescheduleTournamentResultPoll({ stream: RIOT_JOB_HIGH_STREAM, entryId, job, }), ).rejects.toThrow(); expect( (await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending, ).toBe(1); expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(1); expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0); await queue.rescheduleTournamentResultPoll({ stream: RIOT_JOB_HIGH_STREAM, entryId, job, }); expect( (await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending, ).toBe(0); expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1); }); }); function pollJob(): PollTournamentResultJob { return { kind: 'poll-tournament-result', jobId: 'original-result-job', matchId: '6aeecacdffed1eb545b8c0f3', attempt: 7, expandedToAllPuuids: true, enqueuedAt: 1_794_053_383_203, priority: 'high', }; } async function pendingPoll( stream: string, job: PollTournamentResultJob, ): Promise { await redis.createConsumerGroup(stream, RIOT_JOB_GROUP, '0'); const entryId = await redis.appendStream(stream, { job: JSON.stringify(job), }); const delivery = await redis.readGroup( stream, RIOT_JOB_GROUP, 'original-worker', '>', { count: 1 }, ); expect(delivery?.[0].messages[0].id).toBe(entryId); return entryId; }