Files
lolympus-tournament-support/apps/api/src/modules/riot-job-queue/riot-job-worker.poll-retry.spec.ts
T
dahyeon 4b8ccf0077 fix: harden result polling and Redis 8.10 operations
Preserve tournament result jobs through lock contention and transient failures,
atomically schedule retries before acknowledgement, and publish initial polls
after releasing the preparation lock.

Fix Redis connection blocking, durability Hash cleanup, duplicate auction
request ordering, and scheduled-job promotion. Use native Redis 8.10 stream,
lock, counter, and field-expiring cache operations with invalidation fencing.

Repair 90 test type errors and add a repeatable API typecheck command.

Validation: 704 API tests, 129 web tests, 31 real Redis integration tests,
3 dependency checks, workspace type checks and builds, and changed-file lint.
2026-11-09 17:33:13 +09:00

160 lines
4.8 KiB
TypeScript

import type { StreamEntry } from '../global/redis/redis.service';
import type { PollTournamentResultJob } from '../tournament-result/tournament-result.types';
import { RiotJobWorkerService } from './riot-job-worker.service';
import { RIOT_JOB_HIGH_STREAM } from './riot-job-queue.types';
const job: PollTournamentResultJob = {
kind: 'poll-tournament-result',
jobId: 'poll-1',
matchId: 'match-1',
attempt: 3,
expandedToAllPuuids: true,
enqueuedAt: 1,
priority: 'high',
};
function harness() {
const entry: StreamEntry = {
id: '1-0',
message: { job: JSON.stringify(job) },
};
const pending = new Set([entry.id]);
const scheduled: PollTournamentResultJob[] = [];
const processPollJob = jest.fn().mockResolvedValue(undefined);
const acknowledge = jest.fn().mockImplementation(async () => {
pending.delete(entry.id);
await Promise.resolve();
});
const reschedule = jest
.fn()
.mockImplementation(
async (params: {
stream: string;
entryId: string;
job: PollTournamentResultJob;
}) => {
scheduled.push(params.job);
pending.delete(params.entryId);
await Promise.resolve();
},
);
const locks = {
runWithRenewedLock: jest
.fn()
.mockImplementation(
async (_options: unknown, operation: () => Promise<unknown>) =>
operation(),
),
};
const worker = new RiotJobWorkerService(
{ acknowledgeAndDeleteStream: acknowledge } as never,
locks as never,
{ rescheduleTournamentResultPoll: reschedule } as never,
{} as never,
{} as never,
{} as never,
{ processPollJob } as never,
{} as never,
{} as never,
);
const process = () =>
(
worker as unknown as {
processEntry(stream: string, entry: StreamEntry): Promise<void>;
}
).processEntry(RIOT_JOB_HIGH_STREAM, entry);
return {
entry,
pending,
scheduled,
processPollJob,
acknowledge,
reschedule,
locks,
process,
};
}
describe('Tournament result worker retry delivery', () => {
it.each(['match lock unavailable', 'database write failed'])(
'preserves failed work in delayed retry instead of completing it: %s',
async (reason) => {
const h = harness();
h.processPollJob.mockRejectedValue(new Error(reason));
await h.process();
expect(h.scheduled).toEqual([job]);
expect(h.pending.size).toBe(0);
expect(h.acknowledge).not.toHaveBeenCalled();
},
);
it('leaves the original pending when saving the delayed retry fails', async () => {
const h = harness();
h.processPollJob.mockRejectedValue(new Error('database write failed'));
h.reschedule.mockRejectedValue(new Error('Redis unavailable'));
await expect(h.process()).rejects.toThrow('Redis unavailable');
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.scheduled).toEqual([]);
expect(h.acknowledge).not.toHaveBeenCalled();
});
it('retains the original while retry persistence is still in flight', async () => {
const h = harness();
h.processPollJob.mockRejectedValue(new Error('match lock unavailable'));
let releaseRetry!: () => void;
let retryStarted!: () => void;
const started = new Promise<void>((resolve) => {
retryStarted = resolve;
});
const retry = new Promise<void>((resolve) => {
releaseRetry = resolve;
});
h.reschedule.mockImplementation(async () => {
retryStarted();
await retry;
h.scheduled.push(job);
h.pending.delete(h.entry.id);
});
const processing = h.process();
// Race against completion so the buggy worker fails instead of hanging.
await Promise.race([started, processing]);
try {
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.acknowledge).not.toHaveBeenCalled();
} finally {
releaseRetry();
await processing;
}
});
it('acknowledges successfully processed work without scheduling retry', async () => {
const h = harness();
await h.process();
expect(h.pending.size).toBe(0);
expect(h.scheduled).toEqual([]);
expect(h.acknowledge).toHaveBeenCalledTimes(1);
});
it('does not acknowledge work owned by another job worker', async () => {
const h = harness();
h.locks.runWithRenewedLock.mockResolvedValue(null);
await h.process();
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.processPollJob).not.toHaveBeenCalled();
expect(h.scheduled).toEqual([]);
});
it('leaves a successful job pending if its acknowledgement fails', async () => {
const h = harness();
h.acknowledge.mockRejectedValue(new Error('acknowledgement unavailable'));
await expect(h.process()).rejects.toThrow('acknowledgement unavailable');
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.scheduled).toEqual([]);
});
});