fix: harden result polling and Redis 8.10 operations

Preserve tournament result jobs through lock contention and transient failures,
atomically schedule retries before acknowledgement, and publish initial polls
after releasing the preparation lock.

Fix Redis connection blocking, durability Hash cleanup, duplicate auction
request ordering, and scheduled-job promotion. Use native Redis 8.10 stream,
lock, counter, and field-expiring cache operations with invalidation fencing.

Repair 90 test type errors and add a repeatable API typecheck command.

Validation: 704 API tests, 129 web tests, 31 real Redis integration tests,
3 dependency checks, workspace type checks and builds, and changed-file lint.
This commit is contained in:
dahyeon committed 2026-11-09 17:33:13 +09:00
1 parent 0c4c4239ec
commit 4b8ccf0077
49 files changed
+2214 -371

No files matched your search

@@ -1,12 +1,17 @@
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { RedisService, StreamEntry } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
import { RiotRecentMatchesService } from '../riot/riot-recent-matches.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { RIOT_JOB_GROUP, RiotRefreshSubJob } from './riot-job-queue.types';
import {
RIOT_JOB_GROUP,
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_STREAM,
RiotRefreshSubJob,
} from './riot-job-queue.types';
import { RiotJobQueueService } from './riot-job-queue.service';
import { RiotJobWorkerService } from './riot-job-worker.service';
import { RiotRefreshGateway } from './riot-refresh.gateway';
@@ -16,9 +21,17 @@ function execResult<T>(value: T) {
}
function makeWorker() {
const readerClient = { isOpen: true, destroy: jest.fn() };
const streamReader = {
raw: jest.fn().mockReturnValue(readerClient),
readGroup: jest.fn().mockResolvedValue(null),
} as unknown as jest.Mocked<RedisService>;
const redisService = {
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
acknowledgeAndDeleteStream: jest.fn().mockResolvedValue(undefined),
createConsumerGroup: jest.fn().mockResolvedValue(undefined),
createDedicatedConnection: jest.fn().mockResolvedValue(streamReader),
raw: jest.fn(),
readGroup: jest.fn(),
} as unknown as jest.Mocked<RedisService>;
const distributedLockService = {
runWithRenewedLock: jest
@@ -33,6 +46,7 @@ function makeWorker() {
),
} as unknown as jest.Mocked<DistributedLockService>;
const jobQueueService = {
promoteDueScheduledJobs: jest.fn().mockResolvedValue(0),
markProcessing: jest.fn().mockResolvedValue(undefined),
markCompleted: jest.fn().mockResolvedValue({
parentJobId: 'parent-1',
@@ -87,10 +101,106 @@ function makeWorker() {
connectionService,
refreshGateway,
enrichmentService,
streamReader,
readerClient,
};
}
type WorkerInternals = {
stopped: boolean;
loop(): Promise<void>;
tryBecomeLeader(): Promise<boolean>;
readOne(
stream: string,
blockMs: number,
): Promise<{ stream: string; message: StreamEntry } | null>;
processEntry(stream: string, entry: StreamEntry): Promise<void>;
};
describe('RiotJobWorkerService', () => {
it('connects a dedicated reader after creating groups and only closes that reader', async () => {
const { worker, redisService, streamReader, readerClient } = makeWorker();
const internal = worker as unknown as WorkerInternals;
jest.spyOn(internal, 'loop').mockResolvedValue(undefined);
await worker.onModuleInit();
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_GROUP,
);
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
RIOT_JOB_LOW_STREAM,
RIOT_JOB_GROUP,
);
expect(
redisService.createConsumerGroup.mock.invocationCallOrder[1],
).toBeLessThan(
redisService.createDedicatedConnection.mock.invocationCallOrder[0],
);
await internal.readOne(RIOT_JOB_HIGH_STREAM, 100);
expect(streamReader.readGroup).toHaveBeenCalledWith(
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_GROUP,
expect.any(String),
'>',
{ count: 1, blockMs: 100, claimMinIdleMs: 1800000 },
);
expect(redisService.readGroup).not.toHaveBeenCalled();
await worker.onModuleDestroy();
expect(readerClient.destroy).toHaveBeenCalledTimes(1);
expect(redisService.raw).not.toHaveBeenCalled();
expect(internal.stopped).toBe(true);
});
it.each([true, false])(
'preserves high-before-low consumption (high available: %s)',
async (highAvailable) => {
const { worker, jobQueueService } = makeWorker();
const internal = worker as unknown as WorkerInternals;
const entry = { id: '1-0', message: { job: '{}' } };
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
const read = jest
.spyOn(internal, 'readOne')
.mockImplementation(async (stream) => {
if (stream === RIOT_JOB_HIGH_STREAM && !highAvailable) return null;
return { stream, message: entry };
});
const process = jest
.spyOn(internal, 'processEntry')
.mockImplementation(async () => {
internal.stopped = true;
});
await internal.loop();
expect(jobQueueService.promoteDueScheduledJobs).toHaveBeenCalledTimes(1);
expect(read.mock.calls).toEqual(
highAvailable
? [[RIOT_JOB_HIGH_STREAM, 100]]
: [
[RIOT_JOB_HIGH_STREAM, 100],
[RIOT_JOB_LOW_STREAM, 1000],
],
);
expect(process).toHaveBeenCalledWith(
highAvailable ? RIOT_JOB_HIGH_STREAM : RIOT_JOB_LOW_STREAM,
entry,
);
},
);
it('does not start processing a message returned during shutdown', async () => {
const { worker } = makeWorker();
const internal = worker as unknown as WorkerInternals;
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
jest.spyOn(internal, 'readOne').mockImplementation(async (stream) => {
await worker.onModuleDestroy();
return { stream, message: { id: '1-0', message: {} } };
});
const process = jest.spyOn(internal, 'processEntry');
await internal.loop();
expect(process).not.toHaveBeenCalled();
});
it('acknowledges and deletes invalid stream entries', async () => {
const { worker, redisService } = makeWorker();
@@ -106,15 +216,11 @@ describe('RiotJobWorkerService', () => {
message: { job: 'not-json' },
});
expect(redisService.acknowledgeStream).toHaveBeenCalledWith(
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
'riot:queue:high',
RIOT_JOB_GROUP,
'bad-1',
);
expect(redisService.deleteStreamEntries).toHaveBeenCalledWith(
'riot:queue:high',
'bad-1',
);
});
it('refreshes Riot identity and league entries while processing user history refresh jobs', async () => {
@@ -165,19 +271,15 @@ describe('RiotJobWorkerService', () => {
expect(
enrichmentService.refreshRiotIdentity.mock.invocationCallOrder[0],
).toBeLessThan(jobQueueService.markCompleted.mock.invocationCallOrder[0]);
expect(redisService.acknowledgeStream).toHaveBeenCalledWith(
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
'riot:queue:high',
RIOT_JOB_GROUP,
'1-0',
);
expect(redisService.deleteStreamEntries).toHaveBeenCalledWith(
'riot:queue:high',
'1-0',
);
expect(
redisService.acknowledgeStream.mock.invocationCallOrder[0],
jobQueueService.markCompleted.mock.invocationCallOrder[0],
).toBeLessThan(
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
redisService.acknowledgeAndDeleteStream.mock.invocationCallOrder[0],
);
});
@@ -249,7 +351,6 @@ describe('RiotJobWorkerService', () => {
message: { job: JSON.stringify(job) },
});
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
expect(redisService.acknowledgeAndDeleteStream).not.toHaveBeenCalled();
});
});