Files
lolympus-tournament-support/apps/api/src/modules/riot-job-queue/riot-job-worker.service.spec.ts
T
dahyeon 4b8ccf0077 fix: harden result polling and Redis 8.10 operations
Preserve tournament result jobs through lock contention and transient failures,
atomically schedule retries before acknowledgement, and publish initial polls
after releasing the preparation lock.

Fix Redis connection blocking, durability Hash cleanup, duplicate auction
request ordering, and scheduled-job promotion. Use native Redis 8.10 stream,
lock, counter, and field-expiring cache operations with invalidation fencing.

Repair 90 test type errors and add a repeatable API typecheck command.

Validation: 704 API tests, 129 web tests, 31 real Redis integration tests,
3 dependency checks, workspace type checks and builds, and changed-file lint.
2026-11-09 17:33:13 +09:00

357 lines
12 KiB
TypeScript

import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService, StreamEntry } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
import { RiotRecentMatchesService } from '../riot/riot-recent-matches.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import {
RIOT_JOB_GROUP,
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_STREAM,
RiotRefreshSubJob,
} from './riot-job-queue.types';
import { RiotJobQueueService } from './riot-job-queue.service';
import { RiotJobWorkerService } from './riot-job-worker.service';
import { RiotRefreshGateway } from './riot-refresh.gateway';
function execResult<T>(value: T) {
return { exec: jest.fn().mockResolvedValue(value) };
}
function makeWorker() {
const readerClient = { isOpen: true, destroy: jest.fn() };
const streamReader = {
raw: jest.fn().mockReturnValue(readerClient),
readGroup: jest.fn().mockResolvedValue(null),
} as unknown as jest.Mocked<RedisService>;
const redisService = {
acknowledgeAndDeleteStream: jest.fn().mockResolvedValue(undefined),
createConsumerGroup: jest.fn().mockResolvedValue(undefined),
createDedicatedConnection: jest.fn().mockResolvedValue(streamReader),
raw: jest.fn(),
readGroup: jest.fn(),
} as unknown as jest.Mocked<RedisService>;
const distributedLockService = {
runWithRenewedLock: jest
.fn()
.mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
} as unknown as jest.Mocked<DistributedLockService>;
const jobQueueService = {
promoteDueScheduledJobs: jest.fn().mockResolvedValue(0),
markProcessing: jest.fn().mockResolvedValue(undefined),
markCompleted: jest.fn().mockResolvedValue({
parentJobId: 'parent-1',
jobId: 'job-1',
queueGroup: 'summoners-rift',
puuid: 'puuid-1',
state: 'completed',
}),
markFailed: jest.fn(),
} as unknown as jest.Mocked<RiotJobQueueService>;
const recentMatchesService = {
ensureFreshForQueue: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RiotRecentMatchesService>;
const userModel = {
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
};
const connectionService = {
getModel: jest.fn().mockResolvedValue(userModel),
} as unknown as jest.Mocked<ConnectionService>;
const refreshGateway = {
broadcastRefreshEvent: jest.fn(),
} as unknown as jest.Mocked<RiotRefreshGateway>;
const tournamentResultService = {
processPollJob: jest.fn(),
} as unknown as jest.Mocked<TournamentResultService>;
const legacyImportService = {
processMemberJob: jest.fn(),
} as unknown as jest.Mocked<LegacyImportService>;
const enrichmentService = {
refreshRiotIdentity: jest.fn().mockResolvedValue('unchanged'),
refreshLeagueEntries: jest.fn().mockResolvedValue('updated'),
} as unknown as jest.Mocked<RiotAccountEnrichmentService>;
const worker = new RiotJobWorkerService(
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
refreshGateway,
tournamentResultService,
legacyImportService,
enrichmentService,
);
return {
worker,
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
refreshGateway,
enrichmentService,
streamReader,
readerClient,
};
}
type WorkerInternals = {
stopped: boolean;
loop(): Promise<void>;
tryBecomeLeader(): Promise<boolean>;
readOne(
stream: string,
blockMs: number,
): Promise<{ stream: string; message: StreamEntry } | null>;
processEntry(stream: string, entry: StreamEntry): Promise<void>;
};
describe('RiotJobWorkerService', () => {
it('connects a dedicated reader after creating groups and only closes that reader', async () => {
const { worker, redisService, streamReader, readerClient } = makeWorker();
const internal = worker as unknown as WorkerInternals;
jest.spyOn(internal, 'loop').mockResolvedValue(undefined);
await worker.onModuleInit();
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_GROUP,
);
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
RIOT_JOB_LOW_STREAM,
RIOT_JOB_GROUP,
);
expect(
redisService.createConsumerGroup.mock.invocationCallOrder[1],
).toBeLessThan(
redisService.createDedicatedConnection.mock.invocationCallOrder[0],
);
await internal.readOne(RIOT_JOB_HIGH_STREAM, 100);
expect(streamReader.readGroup).toHaveBeenCalledWith(
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_GROUP,
expect.any(String),
'>',
{ count: 1, blockMs: 100, claimMinIdleMs: 1800000 },
);
expect(redisService.readGroup).not.toHaveBeenCalled();
await worker.onModuleDestroy();
expect(readerClient.destroy).toHaveBeenCalledTimes(1);
expect(redisService.raw).not.toHaveBeenCalled();
expect(internal.stopped).toBe(true);
});
it.each([true, false])(
'preserves high-before-low consumption (high available: %s)',
async (highAvailable) => {
const { worker, jobQueueService } = makeWorker();
const internal = worker as unknown as WorkerInternals;
const entry = { id: '1-0', message: { job: '{}' } };
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
const read = jest
.spyOn(internal, 'readOne')
.mockImplementation(async (stream) => {
if (stream === RIOT_JOB_HIGH_STREAM && !highAvailable) return null;
return { stream, message: entry };
});
const process = jest
.spyOn(internal, 'processEntry')
.mockImplementation(async () => {
internal.stopped = true;
});
await internal.loop();
expect(jobQueueService.promoteDueScheduledJobs).toHaveBeenCalledTimes(1);
expect(read.mock.calls).toEqual(
highAvailable
? [[RIOT_JOB_HIGH_STREAM, 100]]
: [
[RIOT_JOB_HIGH_STREAM, 100],
[RIOT_JOB_LOW_STREAM, 1000],
],
);
expect(process).toHaveBeenCalledWith(
highAvailable ? RIOT_JOB_HIGH_STREAM : RIOT_JOB_LOW_STREAM,
entry,
);
},
);
it('does not start processing a message returned during shutdown', async () => {
const { worker } = makeWorker();
const internal = worker as unknown as WorkerInternals;
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
jest.spyOn(internal, 'readOne').mockImplementation(async (stream) => {
await worker.onModuleDestroy();
return { stream, message: { id: '1-0', message: {} } };
});
const process = jest.spyOn(internal, 'processEntry');
await internal.loop();
expect(process).not.toHaveBeenCalled();
});
it('acknowledges and deletes invalid stream entries', async () => {
const { worker, redisService } = makeWorker();
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: 'bad-1',
message: { job: 'not-json' },
});
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
'riot:queue:high',
RIOT_JOB_GROUP,
'bad-1',
);
});
it('refreshes Riot identity and league entries while processing user history refresh jobs', async () => {
const {
worker,
enrichmentService,
recentMatchesService,
redisService,
jobQueueService,
} = makeWorker();
const userId = new mongoose.Types.ObjectId().toString();
const job: RiotRefreshSubJob = {
jobId: 'job-1',
parentJobId: 'parent-1',
userId,
puuid: 'puuid-1',
region: 'asia',
platform: 'kr',
queueGroup: 'summoners-rift',
queueIds: [420, 430],
requestedAt: Date.now(),
priority: 'high',
};
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: '1-0',
message: { job: JSON.stringify(job) },
});
expect(recentMatchesService.ensureFreshForQueue).toHaveBeenCalledTimes(2);
expect(enrichmentService.refreshRiotIdentity).toHaveBeenCalledWith({
userId: expect.any(mongoose.Types.ObjectId),
puuid: 'puuid-1',
region: 'asia',
});
expect(enrichmentService.refreshLeagueEntries).toHaveBeenCalledWith({
userId: expect.any(mongoose.Types.ObjectId),
puuid: 'puuid-1',
platform: 'kr',
});
expect(
enrichmentService.refreshRiotIdentity.mock.invocationCallOrder[0],
).toBeLessThan(jobQueueService.markCompleted.mock.invocationCallOrder[0]);
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
'riot:queue:high',
RIOT_JOB_GROUP,
'1-0',
);
expect(
jobQueueService.markCompleted.mock.invocationCallOrder[0],
).toBeLessThan(
redisService.acknowledgeAndDeleteStream.mock.invocationCallOrder[0],
);
});
it('completes successful match refresh work when Riot identity lookup throws', async () => {
const { worker, enrichmentService, jobQueueService, refreshGateway } =
makeWorker();
enrichmentService.refreshRiotIdentity.mockRejectedValueOnce(
new Error('Account-V1 unavailable'),
);
const job: RiotRefreshSubJob = {
jobId: 'job-1',
parentJobId: 'parent-1',
userId: new mongoose.Types.ObjectId().toString(),
puuid: 'puuid-1',
region: 'asia',
platform: 'kr',
queueGroup: 'summoners-rift',
queueIds: [420],
requestedAt: Date.now(),
priority: 'high',
};
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: '1-0',
message: { job: JSON.stringify(job) },
});
expect(enrichmentService.refreshRiotIdentity).toHaveBeenCalledTimes(1);
expect(jobQueueService.markCompleted).toHaveBeenCalledWith(job);
expect(jobQueueService.markFailed).not.toHaveBeenCalled();
expect(refreshGateway.broadcastRefreshEvent).toHaveBeenCalledWith(
expect.objectContaining({ state: 'completed' }),
);
});
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
const { worker, distributedLockService, redisService } = makeWorker();
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
const userId = new mongoose.Types.ObjectId().toString();
const job: RiotRefreshSubJob = {
jobId: 'job-1',
parentJobId: 'parent-1',
userId,
puuid: 'puuid-1',
region: 'asia',
platform: 'kr',
queueGroup: 'summoners-rift',
queueIds: [420],
requestedAt: Date.now(),
priority: 'high',
};
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: '1-0',
message: { job: JSON.stringify(job) },
});
expect(redisService.acknowledgeAndDeleteStream).not.toHaveBeenCalled();
});
});