fix: harden result polling and Redis 8.10 operations

Preserve tournament result jobs through lock contention and transient failures,
atomically schedule retries before acknowledgement, and publish initial polls
after releasing the preparation lock.

Fix Redis connection blocking, durability Hash cleanup, duplicate auction
request ordering, and scheduled-job promotion. Use native Redis 8.10 stream,
lock, counter, and field-expiring cache operations with invalidation fencing.

Repair 90 test type errors and add a repeatable API typecheck command.

Validation: 704 API tests, 129 web tests, 31 real Redis integration tests,
3 dependency checks, workspace type checks and builds, and changed-file lint.
This commit is contained in:
dahyeon committed 2026-11-09 17:33:13 +09:00
1 parent 0c4c4239ec
commit 4b8ccf0077
49 files changed
+2214 -371

No files matched your search

+2
View File
@@ -22,6 +22,8 @@
"start:debug": "nest start --debug --watch",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
"typecheck": "tsc --noEmit --incremental false",
"test:redis": "node test/run-redis-tests.mjs",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
@@ -2103,7 +2103,7 @@ describe('AuctionService', () => {
});
expect(auction.currentBid).toBe(970);
expect(auction.currentBidderTeamIndex).toBe(1);
expect(auction).toHaveProperty('currentBidderTeamIndex', 1);
expect(auction.bids).toEqual([
expect.objectContaining({
teamIndex: 1,
@@ -2397,7 +2397,7 @@ describe('AuctionService', () => {
await service.next(matchId);
expect(auction.phase).toBe('complete');
expect(auction.completedAt).toBeInstanceOf(Date);
expect(auction).toHaveProperty('completedAt', expect.any(Date));
expect(auction.save).toHaveBeenCalledTimes(1);
expect(match.save).toHaveBeenCalledTimes(2);
expect(match.teams).toEqual([
@@ -39,6 +39,7 @@ import {
stringifyAuctionRedisStructureState,
} from './redis/auction-redis-projection';
import { AuctionRedisService } from './redis/auction-redis.service';
import { AUCTION_CLEANUP_DURABILITY_SCRIPT } from './redis/auction-redis.cleanup';
import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service';
import { shouldHydrateRedisAuctionState } from './redis/auction-redis.state';
// [AUCTION-DEBUG] 경매 "중복 멤버" 조사용 임시 계측. 조사 종료 후 제거 대상.
@@ -2970,54 +2971,16 @@ export class AuctionService {
const payloadKeys = Array.isArray(reply[1]) ? reply[1].map(String) : [];
if (payloadKeys.length === 0) continue;
const values = await this.auctionRedisService
.raw()
.sendCommand(['MGET', ...payloadKeys]);
const payloadKeysToDelete = payloadKeys.filter((payloadKey, index) =>
this.shouldDeleteRedisDurabilityPayload(
Array.isArray(values) ? values[index] : undefined,
expectedAuctionLifecycleId,
payloadKey,
),
);
if (payloadKeysToDelete.length === 0) continue;
await this.auctionRedisService
.raw()
.sendCommand([
'ZREM',
await this.auctionRedisService.evalScript(
AUCTION_CLEANUP_DURABILITY_SCRIPT,
[
keys.durabilityRetryQueueKey,
...payloadKeysToDelete,
]);
await this.auctionRedisService
.raw()
.sendCommand([
'ZREM',
keys.durabilityDeadLetterQueueKey,
...payloadKeysToDelete,
]);
await this.auctionRedisService
.raw()
.sendCommand(['DEL', ...payloadKeysToDelete]);
} while (cursor !== '0');
}
private shouldDeleteRedisDurabilityPayload(
payloadValue: unknown,
expectedAuctionLifecycleId: string | undefined,
payloadKey: string,
): boolean {
if (!expectedAuctionLifecycleId) return true;
if (typeof payloadValue !== 'string') return false;
try {
const payload = JSON.parse(payloadValue) as Record<string, unknown>;
return payload.auctionLifecycleId === expectedAuctionLifecycleId;
} catch {
this.logger.warn(
`Failed to parse auction durability payload ${payloadKey}.`,
...payloadKeys,
],
[expectedAuctionLifecycleId ?? ''],
);
return false;
}
} while (cursor !== '0');
}
private enqueueBackgroundOrderEnrichment(params: {
@@ -317,7 +317,7 @@ if actionRequestScopeKey ~= '' then
actionRequestPrefix .. 'responseServerTimestampMS',
tostring(currentRedisTimestampMS)
)
redis.call('RPUSH', auctionActionRequestResultOrderKey, actionRequestScopeKey)
-- The reservation already inserted this request into the order list.
end
return {'OK', auctionStateTransitionId, tostring(newRevision), tostring(currentRedisTimestampMS)}
@@ -0,0 +1,27 @@
import { defineAuctionRedisScript } from './auction-redis.scripts';
// Compare the lifecycle and remove the payload and its indexes atomically.
export const AUCTION_CLEANUP_DURABILITY_SCRIPT = defineAuctionRedisScript(
'auction.cleanupDurability',
`
local deleted = 0
for i = 3, #KEYS do
local key = KEYS[i]
local keyType = redis.call('TYPE', key).ok
local matches = ARGV[1] == ''
if not matches and keyType == 'hash' then
matches = redis.call('HGET', key, 'auctionLifecycleId') == ARGV[1]
elseif not matches and keyType == 'string' then
-- Retain compatibility with payloads written before the Hash format.
local ok, payload = pcall(cjson.decode, redis.call('GET', key))
matches = ok and type(payload) == 'table' and payload.auctionLifecycleId == ARGV[1]
end
if matches then
redis.call('ZREM', KEYS[1], key)
redis.call('ZREM', KEYS[2], key)
deleted = deleted + redis.call('UNLINK', key)
end
end
return deleted
`,
);
@@ -129,13 +129,7 @@ export class AuctionRedisService implements OnModuleDestroy {
}
public async releaseTokenLock(key: string, token: string): Promise<boolean> {
const result = await this.client.sendCommand([
'EVAL',
'if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end',
'1',
key,
token,
]);
const result = await this.client.sendCommand(['DELEX', key, 'IFEQ', token]);
return Number(result) === 1;
}
}
@@ -4,6 +4,7 @@ import { ConnectionService } from '../global/database/connection.service';
import { IdentityDocument, UserDocument } from '../global/database/schema';
import { UserResolverGuard } from './user-resolver.guard';
import './auth-request';
import type { Request } from 'express';
function createContext(request: Record<string, unknown>): ExecutionContext {
return {
@@ -24,7 +25,7 @@ describe('UserResolverGuard', () => {
const connectionService = {
getModel: jest.fn().mockResolvedValue({ findOne }),
} as unknown as ConnectionService;
const request = {
const request: Pick<Request, 'params' | 'identity' | 'user'> = {
params: { communityId: communityId.toString() },
identity: { _id: identityId } as IdentityDocument,
};
@@ -1,8 +1,67 @@
import { Injectable, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { RedisService } from '../global/redis/redis.service';
const COMMUNITY_STATS_TTL_SECONDS = 5 * 60;
const COMMUNITY_STATS_DETAIL_TTL_SECONDS = 24 * 60 * 60;
const CACHE_SCOPES = [
'matches',
'leaderboard',
'teamInsights',
'matchDetail',
'userInternal',
] as const;
type CacheScope = (typeof CACHE_SCOPES)[number];
function communityPrefix(communityId: string): string {
return `riot:communitystats:v2:{${communityId}}`;
}
export class CommunityStatsCacheKey {
public readonly generationKey: string;
public readonly hashKey: string;
public generation?: string;
constructor(
communityId: string,
scope: CacheScope,
public readonly field: string,
) {
const prefix = communityPrefix(communityId);
this.generationKey = `${prefix}:generation`;
this.hashKey = `${prefix}:${scope}`;
}
public get identity(): string {
return JSON.stringify([this.hashKey, this.generation, this.field]);
}
}
const READ_CACHE_SCRIPT = `
local generation = redis.call('GET', KEYS[1])
if not generation then
generation = ARGV[1]
redis.call('SET', KEYS[1], generation)
end
return {generation, redis.call('HGET', KEYS[2], ARGV[2])}
`;
const WRITE_CACHE_SCRIPT = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
redis.call('HSETEX', KEYS[2], 'EX', ARGV[4], 'FIELDS', 1, ARGV[2], ARGV[3])
return 1
`;
const INVALIDATE_CACHE_SCRIPT = `
-- A fill started before invalidation cannot publish into the new generation.
redis.call('SET', KEYS[1], ARGV[1])
local deleted = 0
for i = 2, #KEYS do
deleted = deleted + redis.call('HLEN', KEYS[i])
redis.call('UNLINK', KEYS[i])
end
return deleted
`;
@Injectable()
export class CommunityStatsCacheService {
@@ -10,61 +69,72 @@ export class CommunityStatsCacheService {
constructor(private readonly redisService: RedisService) {}
public get<T>(key: string): Promise<T | null> {
return this.redisService.getJson<T>(key);
public async get<T>(key: CommunityStatsCacheKey): Promise<T | null> {
const reply = await this.redisService
.raw()
.sendCommand<
[string, string | null]
>(['EVAL', READ_CACHE_SCRIPT, '2', key.generationKey, key.hashKey, randomUUID(), key.field]);
key.generation = reply[0];
if (!reply[1]) return null;
try {
return JSON.parse(reply[1]) as T;
} catch {
this.logger.warn(
`Failed to parse community stats cache field in ${key.hashKey}.`,
);
return null;
}
}
public set(
key: string,
public async set(
key: CommunityStatsCacheKey,
value: unknown,
ttlSeconds = COMMUNITY_STATS_TTL_SECONDS,
): Promise<void> {
return this.redisService.setJson(key, value, ttlSeconds);
if (!key.generation)
throw new Error(
'Read the cache generation before starting a cache fill.',
);
if (!Number.isInteger(ttlSeconds) || ttlSeconds <= 0)
throw new Error('Cache TTL must be a positive integer.');
await this.redisService
.raw()
.sendCommand([
'EVAL',
WRITE_CACHE_SCRIPT,
'2',
key.generationKey,
key.hashKey,
key.generation,
key.field,
JSON.stringify(value),
String(ttlSeconds),
]);
}
public setDetail(key: string, value: unknown): Promise<void> {
public setDetail(key: CommunityStatsCacheKey, value: unknown): Promise<void> {
return this.set(key, value, COMMUNITY_STATS_DETAIL_TTL_SECONDS);
}
public async invalidateCommunity(communityId: string): Promise<number> {
const prefix = `riot:communitystats:`;
const match = `${prefix}*${communityId}*`;
const client = this.redisService.raw();
let cursor = '0';
let deleted = 0;
do {
const reply = await client.sendCommand([
'SCAN',
cursor,
'MATCH',
match,
'COUNT',
'100',
]);
if (!Array.isArray(reply)) {
throw new Error('Redis SCAN returned an invalid response.');
}
const values = reply as unknown[];
const nextCursor = values[0];
const rawKeys = values[1];
if (typeof nextCursor !== 'string') {
throw new Error('Redis SCAN returned an invalid cursor.');
}
cursor = nextCursor;
const keys = Array.isArray(rawKeys)
? (rawKeys as unknown[]).filter(
(key): key is string => typeof key === 'string',
)
: [];
if (keys.length > 0) {
deleted += await client.del(keys);
}
} while (cursor !== '0');
const prefix = communityPrefix(communityId);
const deleted = Number(
await this.redisService
.raw()
.sendCommand([
'EVAL',
INVALIDATE_CACHE_SCRIPT,
String(CACHE_SCOPES.length + 1),
`${prefix}:generation`,
...CACHE_SCOPES.map((scope) => `${prefix}:${scope}`),
randomUUID(),
]),
);
if (deleted > 0) {
this.logger.debug(
`Invalidated ${deleted} community stats cache keys for ${communityId}.`,
`Invalidated ${deleted} community stats cache entries for ${communityId}.`,
);
}
return deleted;
@@ -1,6 +1,9 @@
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import { CommunityStatsCacheService } from './community-stats-cache.service';
import {
CommunityStatsCacheKey,
CommunityStatsCacheService,
} from './community-stats-cache.service';
import { CommunityStatsService } from './community-stats.service';
import type {
InternalMatchCardView,
@@ -281,7 +284,7 @@ describe('CommunityStatsService', () => {
expect(setDetail).toHaveBeenCalledTimes(2);
expect(setDetail).toHaveBeenNthCalledWith(
2,
expect.stringContaining(':r4:'),
expect.objectContaining({ field: expect.stringContaining(':r4:') }),
invalidCheckpointResponse,
);
});
@@ -338,7 +341,7 @@ describe('CommunityStatsService', () => {
const set = jest
.fn()
.mockImplementation(
(_key: string, value: InternalMatchDetailResponse) => {
(_key: CommunityStatsCacheKey, value: InternalMatchDetailResponse) => {
cachedResponse = value;
return Promise.resolve();
},
@@ -346,7 +349,7 @@ describe('CommunityStatsService', () => {
const setDetail = jest
.fn()
.mockImplementation(
(_key: string, value: InternalMatchDetailResponse) => {
(_key: CommunityStatsCacheKey, value: InternalMatchDetailResponse) => {
cachedResponse = value;
return Promise.resolve();
},
@@ -377,7 +380,7 @@ describe('CommunityStatsService', () => {
expect(backedOff).toBe(fallback);
expect(buildAndUpsert).toHaveBeenCalledTimes(1);
expect(set).toHaveBeenCalledWith(
expect.stringContaining(':r4:'),
expect.objectContaining({ field: expect.stringContaining(':r4:') }),
fallback,
60,
);
@@ -389,7 +392,7 @@ describe('CommunityStatsService', () => {
expect(buildAndUpsert).toHaveBeenCalledTimes(2);
expect(setDetail).toHaveBeenCalledTimes(1);
expect(setDetail).toHaveBeenCalledWith(
expect.stringContaining(':r4:'),
expect.objectContaining({ field: expect.stringContaining(':r4:') }),
recovered,
);
});
@@ -547,7 +550,9 @@ describe('CommunityStatsService', () => {
).toEqual([matchId.toString()]);
expect(buildBaseDataset).not.toHaveBeenCalled();
expect(cacheService.set).toHaveBeenCalledWith(
expect.stringContaining(`userInternal:v3:${userId.toString()}`),
expect.objectContaining({
field: expect.stringContaining(`userInternal:v3:${userId.toString()}`),
}),
response,
);
});
@@ -11,7 +11,10 @@ import type {
HistorySeasonDto,
RecentMatchView,
} from '../match-history/match-history.types';
import { CommunityStatsCacheService } from './community-stats-cache.service';
import {
CommunityStatsCacheKey,
CommunityStatsCacheService,
} from './community-stats-cache.service';
import type {
CommunityStatsFilters,
InternalMatchCardView,
@@ -219,21 +222,25 @@ export class CommunityStatsService {
communityId: mongoose.Types.ObjectId;
matchId: mongoose.Types.ObjectId;
}): Promise<InternalMatchDetailResponse> {
const key = `riot:communitystats:matchDetail:v${MATCH_STATS_BUILDER_VERSION}:r${MATCH_DETAIL_RESPONSE_CACHE_VERSION}:${params.communityId.toString()}:${params.matchId.toString()}`;
const key = new CommunityStatsCacheKey(
params.communityId.toString(),
'matchDetail',
`v${MATCH_STATS_BUILDER_VERSION}:r${MATCH_DETAIL_RESPONSE_CACHE_VERSION}:${params.matchId.toString()}`,
);
const cached =
await this.cacheService.get<InternalMatchDetailResponse>(key);
if (cached && isCompleteMatchDetailResponse(cached)) return cached;
const inFlight = this.matchDetailInFlight.get(key);
const inFlight = this.matchDetailInFlight.get(key.identity);
if (inFlight) return inFlight;
const promise = this.buildMatchDetailResponse(params, key);
this.matchDetailInFlight.set(key, promise);
this.matchDetailInFlight.set(key.identity, promise);
try {
return await promise;
} finally {
if (this.matchDetailInFlight.get(key) === promise) {
this.matchDetailInFlight.delete(key);
if (this.matchDetailInFlight.get(key.identity) === promise) {
this.matchDetailInFlight.delete(key.identity);
}
}
}
@@ -243,7 +250,7 @@ export class CommunityStatsService {
communityId: mongoose.Types.ObjectId;
matchId: mongoose.Types.ObjectId;
},
key: string,
key: CommunityStatsCacheKey,
): Promise<InternalMatchDetailResponse> {
const dataset = await this.buildBaseDataset(params.communityId, {
seasonId: 'all',
@@ -346,16 +353,16 @@ export class CommunityStatsService {
// Auction viewers can all request the same next player's internal history
// at once. Coalesce local cache misses so one player change does one DB read.
const inFlight = this.userInternalHistoryInFlight.get(key);
const inFlight = this.userInternalHistoryInFlight.get(key.identity);
if (inFlight) return inFlight;
const promise = this.buildUserInternalHistoryResponse(params, key);
this.userInternalHistoryInFlight.set(key, promise);
this.userInternalHistoryInFlight.set(key.identity, promise);
try {
return await promise;
} finally {
if (this.userInternalHistoryInFlight.get(key) === promise) {
this.userInternalHistoryInFlight.delete(key);
if (this.userInternalHistoryInFlight.get(key.identity) === promise) {
this.userInternalHistoryInFlight.delete(key.identity);
}
}
}
@@ -428,7 +435,7 @@ export class CommunityStatsService {
userId: mongoose.Types.ObjectId;
filters: CommunityStatsFilters;
},
cacheKey: string,
cacheKey: CommunityStatsCacheKey,
): Promise<UserInternalHistoryResponse> {
const context = await this.resolveContext(
params.communityId,
@@ -1138,18 +1145,31 @@ export class CommunityStatsService {
scope: string,
communityId: mongoose.Types.ObjectId,
filters: CommunityStatsFilters,
): string {
return [
'riot:communitystats',
scope,
): CommunityStatsCacheKey {
const cacheScope = scope.startsWith('userInternal:')
? 'userInternal'
: scope;
if (
cacheScope !== 'matches' &&
cacheScope !== 'leaderboard' &&
cacheScope !== 'teamInsights' &&
cacheScope !== 'userInternal'
) {
throw new Error(`Unsupported community stats cache scope: ${scope}`);
}
return new CommunityStatsCacheKey(
communityId.toString(),
filters.seasonId ?? 'current',
filters.from ?? '',
filters.to ?? '',
filters.championId ?? '',
filters.cursor ?? '',
filters.limit ?? '',
].join(':');
cacheScope,
JSON.stringify([
scope,
filters.seasonId ?? 'current',
filters.from ?? '',
filters.to ?? '',
filters.championId ?? '',
filters.cursor ?? '',
filters.limit ?? '',
]),
);
}
private clampLimit(
@@ -106,6 +106,8 @@ describe('MatchStatsFragmentBuilder', () => {
puuid: 'puuid-2',
gameName: 'Opponent',
profileIconId: 8,
preferredPositions: [],
joinedAt: new Date('2026-05-03T00:00:00.000Z'),
});
match.teams[1].members.push({ userId: redUserId, assignedPosition: 'TOP' });
match.result?.games[0].participants.push({
@@ -269,7 +271,7 @@ describe('MatchStatsFragmentBuilder', () => {
teamId: 200,
win: false,
});
const timeline = {
const timeline: RiotMatchTimelineDto = {
metadata: {
dataVersion: '2',
matchId: 'KR_1',
@@ -280,6 +282,7 @@ describe('MatchStatsFragmentBuilder', () => {
frames: [
{
timestamp: 0,
events: [],
participantFrames: {
'1': {
participantId: 1,
@@ -507,9 +510,7 @@ describe('MatchStatsFragmentBuilder', () => {
const fragment = builder.buildFragment({
match,
rawByRiotMatchId: new Map([['KR_1', raw as RiotMatchDto]]),
timelineByRiotMatchId: new Map([
['KR_1', timeline as RiotMatchTimelineDto],
]),
timelineByRiotMatchId: new Map([['KR_1', timeline]]),
});
expect(fragment.gameDetails[0]).toMatchObject({
@@ -794,9 +795,7 @@ describe('MatchStatsFragmentBuilder', () => {
} as RiotMatchDto,
],
]),
timelineByRiotMatchId: new Map([
['KR_1', timeline as RiotMatchTimelineDto],
]),
timelineByRiotMatchId: new Map([['KR_1', timeline]]),
});
expect(
shortFragment.records.every((record) => record.checkpoints === undefined),
@@ -281,15 +281,26 @@ describe('MatchStatsService', () => {
MatchStatsFragmentBuilder['buildFragment']
>[0];
const buildCalls: BuildParams[] = [];
const builtFragment = {
matchCard: { matchId: matchId.toString() },
const builtFragment: ReturnType<
MatchStatsFragmentBuilder['buildFragment']
> = {
matchCard: {
matchId: matchId.toString(),
title: '내전',
scheduledAt: match.scheduledAt.toISOString(),
status: 'finished',
gameCount: 1,
games: [],
teams: [],
seriesResults: [],
},
records: [],
gameDetails: [],
userIds: [],
championIds: [],
gameCreationMin: 1_000,
gameCreationMax: 2_000,
} as ReturnType<MatchStatsFragmentBuilder['buildFragment']>;
};
const buildFragment = jest.fn((params: BuildParams) => {
buildCalls.push(params);
return builtFragment;
@@ -10,7 +10,7 @@ describe('CompetitionBootstrapService', () => {
it('creates a deterministic paired-Series Competition and immutable snapshots', async () => {
const firstHarness = createHarness();
const firstService = new CompetitionBootstrapService(
firstHarness.connectionService as ConnectionService,
firstHarness.connectionService,
);
const match = matchSnapshot({ gameMode: 'pair', teamCount: 2 });
@@ -88,7 +88,7 @@ describe('CompetitionBootstrapService', () => {
const secondHarness = createHarness();
const secondService = new CompetitionBootstrapService(
secondHarness.connectionService as ConnectionService,
secondHarness.connectionService,
);
await secondService.bootstrap({
...match,
@@ -106,9 +106,7 @@ describe('CompetitionBootstrapService', () => {
it('maps round-robin configuration and preserves a limited round count', async () => {
const harness = createHarness();
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionBootstrapService(harness.connectionService);
await service.bootstrap(
matchSnapshot({
@@ -132,9 +130,7 @@ describe('CompetitionBootstrapService', () => {
it('maps double elimination seeds and stage-specific Series rules', async () => {
const harness = createHarness();
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionBootstrapService(harness.connectionService);
const match = matchSnapshot({
gameMode: 'double-elimination',
teamCount: 4,
@@ -177,14 +173,14 @@ describe('CompetitionBootstrapService', () => {
{ legacyTeamIndex: 4, seedNumber: 3 },
]);
expect(
inserted.seriesGraph.find(({ code }) => code === 'WB_R1_M1').rule,
inserted.seriesGraph.find(({ code }) => code === 'WB_R1_M1')?.rule,
).toEqual({ type: 'first-to', winsRequired: 3, maxGames: 5 });
expect(
inserted.seriesGraph.find(({ code }) => code?.startsWith('LB_')).rule,
inserted.seriesGraph.find(({ code }) => code?.startsWith('LB_'))?.rule,
).toEqual({ type: 'first-to', winsRequired: 1, maxGames: 1 });
expect(
inserted.seriesGraph.find(({ code }) => code === 'GF')?.rule,
).toEqual({ type: 'first-to', winsRequired: 1, maxGames: 1 });
expect(inserted.seriesGraph.find(({ code }) => code === 'GF').rule).toEqual(
{ type: 'first-to', winsRequired: 1, maxGames: 1 },
);
expect(
inserted.seriesGraph.find(({ code }) => code === 'GF_RESET'),
).toMatchObject({
@@ -195,9 +191,7 @@ describe('CompetitionBootstrapService', () => {
it('rejects double elimination until authoritative seeds are persisted', async () => {
const harness = createHarness();
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionBootstrapService(harness.connectionService);
await expect(
service.bootstrap(
@@ -253,7 +247,7 @@ describe('CompetitionBootstrapService', () => {
for (const [label, mutate] of cases) {
const harness = createHarness();
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
harness.connectionService,
);
const match = matchSnapshot({ gameMode: 'pair', teamCount: 2 });
mutate(match);
@@ -272,7 +266,7 @@ describe('CompetitionBootstrapService', () => {
it('rejects bracket mode live but explicitly maps it for legacy migration', async () => {
const liveHarness = createHarness();
const liveService = new CompetitionBootstrapService(
liveHarness.connectionService as ConnectionService,
liveHarness.connectionService,
);
const match = matchSnapshot({ gameMode: 'bracket', teamCount: 2 });
@@ -285,7 +279,7 @@ describe('CompetitionBootstrapService', () => {
const migrationHarness = createHarness();
const migrationService = new CompetitionBootstrapService(
migrationHarness.connectionService as ConnectionService,
migrationHarness.connectionService,
);
await migrationService.bootstrap(match, { source: 'legacy-migration' });
@@ -296,9 +290,7 @@ describe('CompetitionBootstrapService', () => {
it('upserts once and links only an unlinked or already-linked Match in one transaction', async () => {
const harness = createHarness();
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionBootstrapService(harness.connectionService);
const competition = await service.bootstrap(
matchSnapshot({ gameMode: 'pair', teamCount: 2 }),
@@ -342,9 +334,7 @@ describe('CompetitionBootstrapService', () => {
revision: 4,
};
const harness = createHarness({ existingCompetition: existing });
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionBootstrapService(harness.connectionService);
const result = await service.bootstrap(
matchSnapshot({ gameMode: 'pair', teamCount: 2 }),
@@ -372,9 +362,7 @@ describe('CompetitionBootstrapService', () => {
competitionResults: [duplicateKey, winner],
sessionCount: 2,
});
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionBootstrapService(harness.connectionService);
const result = await service.bootstrap(
matchSnapshot({ gameMode: 'pair', teamCount: 2 }),
@@ -391,9 +379,7 @@ describe('CompetitionBootstrapService', () => {
it('aborts when the Match is already linked to a different Competition', async () => {
const harness = createHarness({ matchUpdateResult: { matchedCount: 0 } });
const service = new CompetitionBootstrapService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionBootstrapService(harness.connectionService);
await expect(
service.bootstrap(matchSnapshot({ gameMode: 'pair', teamCount: 2 })),
@@ -508,7 +494,7 @@ function createHarness(
const connectionService = {
createSession: jest.fn(() => Promise.resolve(sessions[sessionIndex++])),
getModel: jest.fn((name: string) => Promise.resolve(models[name])),
};
} as unknown as jest.Mocked<ConnectionService>;
return {
sessions,
@@ -22,9 +22,7 @@ describe('CompetitionCommandService', () => {
it('rejects a stale expected revision and aborts without writing', async () => {
const harness = createHarness({ competitionRevision: 8 });
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
await expect(service.acceptGame(acceptGameCommand())).rejects.toMatchObject(
{
@@ -44,9 +42,7 @@ describe('CompetitionCommandService', () => {
it('uses the expected revision in the persistence CAS and aborts a lost race', async () => {
const harness = createHarness({ casResult: null });
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
await expect(service.acceptGame(acceptGameCommand())).rejects.toMatchObject(
{
@@ -86,9 +82,7 @@ describe('CompetitionCommandService', () => {
return Promise.resolve([receipt]);
},
);
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
const command = acceptGameCommand();
const first = await service.acceptGame(command);
@@ -133,9 +127,7 @@ describe('CompetitionCommandService', () => {
});
},
);
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
await expect(
service.acceptGame(acceptGameCommand()),
@@ -162,9 +154,7 @@ describe('CompetitionCommandService', () => {
revision: 8,
}),
);
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
await expect(service.acceptGame(acceptGameCommand())).rejects.toMatchObject(
{
@@ -180,9 +170,7 @@ describe('CompetitionCommandService', () => {
it('atomically accepts a code-backed game, projects it, and enqueues an event', async () => {
const harness = createHarness();
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
const command = {
...acceptGameCommand(),
tournamentCode: {
@@ -279,9 +267,7 @@ describe('CompetitionCommandService', () => {
it('appends an administrative adjudication through the same command runner', async () => {
const harness = createHarness();
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
const result = await service.adjudicateSeries({
competitionId: COMPETITION_ID,
@@ -316,9 +302,7 @@ describe('CompetitionCommandService', () => {
facts: reversal.facts,
assignments: reversal.assignments,
});
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
const preview = await service.previewSeriesReversal({
competitionId: COMPETITION_ID,
@@ -348,9 +332,7 @@ describe('CompetitionCommandService', () => {
it('rejects a reversal preview at a stale revision', async () => {
const reversal = reversalFixture();
const harness = createHarness({ competition: reversal.competition });
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
await expect(
service.previewSeriesReversal({
@@ -372,9 +354,7 @@ describe('CompetitionCommandService', () => {
facts: reversal.facts,
assignments: reversal.assignments,
});
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
const preview = await service.previewSeriesReversal({
competitionId: COMPETITION_ID,
expectedRevision: 7,
@@ -485,9 +465,7 @@ describe('CompetitionCommandService', () => {
}),
)
.mockResolvedValueOnce([]);
const service = new CompetitionCommandService(
harness.connectionService as ConnectionService,
);
const service = new CompetitionCommandService(harness.connectionService);
const preview = await service.previewSeriesReversal({
competitionId: COMPETITION_ID,
expectedRevision: 7,
@@ -635,7 +613,7 @@ function createHarness(
const connectionService = {
createSession: jest.fn().mockResolvedValue(session),
getModel: jest.fn((name: string) => Promise.resolve(models[name])),
};
} as unknown as jest.Mocked<ConnectionService>;
return {
session,
@@ -160,9 +160,7 @@ const SERIES_ID = 'series-1';
const GAME_SLOT_ID = 'slot-1';
function createService(harness: ReturnType<typeof createHarness>) {
return new CompetitionQueryService(
harness.connectionService as ConnectionService,
);
return new CompetitionQueryService(harness.connectionService);
}
function createHarness(options: { competition?: unknown } = {}) {
@@ -244,7 +242,7 @@ function createHarness(options: { competition?: unknown } = {}) {
factModel,
connectionService: {
getModel: jest.fn((name: string) => Promise.resolve(models[name])),
},
} as unknown as jest.Mocked<ConnectionService>,
};
}
@@ -127,7 +127,7 @@ describe('Competition persistence schemas', () => {
function expectIndexes(
schema: {
indexes(): Array<[Record<string, number>, Record<string, unknown>]>;
indexes(): ReturnType<typeof CompetitionSchema.indexes>;
},
expected: Array<{
keys: Record<string, number>;
@@ -589,9 +589,10 @@ function acceptedGameFactory(competitionId: string) {
};
}
function projectedSeries<
T extends { series: Array<{ code?: string; seriesId: string }> },
>(projection: T, code: string) {
function projectedSeries<T extends { code?: string; seriesId: string }>(
projection: { series: T[] },
code: string,
) {
const series = projection.series.find((candidate) => candidate.code === code);
if (!series) throw new Error(`Missing projected series ${code}`);
return series;
@@ -223,11 +223,21 @@ describe('DakaoUserService', () => {
puuid: 'puuid-1',
gameName: '기존소환사',
tagLine: 'KR1',
profileIconId: 1,
revisionDate: 0,
summonerLevel: 30,
tiers: [],
tierHistory: [],
matchIds: [],
championMasteries: [],
championRecords: {},
isPrimary: true,
createdAt: new Date('2026-05-03T00:00:00.000Z'),
updatedAt: new Date('2026-05-03T00:00:00.000Z'),
},
],
save,
} as Partial<UserDocument>,
} satisfies Partial<UserDocument>,
});
await expect(
@@ -35,7 +35,7 @@ describe('parseCommand', () => {
matchAndRiotId: '69b63bc96da18dfdb218ee8c 프리리#KR1',
});
if (result.kind === 'match-join') {
expect(result.positions.positions).toEqual(['MID', 'TOP']);
expect(result.positions?.positions).toEqual(['MID', 'TOP']);
}
});
@@ -62,9 +62,7 @@ describe('auction Redis schema foundation', () => {
expect(
AuctionHistoryEntrySchema.path('matchSideEffectMutation'),
).toBeDefined();
expect(
AuctionHistoryEntrySchema.path<string>('action').options.enum,
).toEqual(
expect(AuctionHistoryEntrySchema.path('action').options.enum).toEqual(
expect.arrayContaining([
'start',
'open-bidding',
@@ -1,8 +1,10 @@
import { migrateTeamIndex } from './20260502-team-index';
type MigrationDocument = Record<string, unknown> & { _id: unknown };
function createDb(params: {
auctions?: Array<Record<string, unknown>>;
matches?: Array<Record<string, unknown>>;
auctions?: MigrationDocument[];
matches?: MigrationDocument[];
}) {
const collections = {
auction: params.auctions ?? [],
@@ -40,6 +40,14 @@ describe('repair-double-elimination-series-rules buildRepairPlan', () => {
{
gameNumber: 1,
bracketMatchId: 'WB_R2_M1',
riotMatchId: 'KR_1',
blueTeamIndex: 4,
redTeamIndex: 6,
winnerTeamIndex: 4,
winnerSide: 'blue',
gameCreation: Date.parse('2026-05-03T00:00:00.000Z'),
gameDuration: 1800,
participants: [],
},
],
seriesResults: [],
@@ -59,16 +59,15 @@ describe('DistributedLockService', () => {
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'),
'1',
'DELEX',
'distributed-lock:test',
'IFEQ',
'owner-token',
]);
});
it('extends a lock only when the ownership token matches', async () => {
const sendCommand = jest.fn().mockResolvedValue(1);
const sendCommand = jest.fn().mockResolvedValue('OK');
const service = createService(sendCommand);
await expect(
@@ -83,11 +82,12 @@ describe('DistributedLockService', () => {
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('PEXPIRE'),
'1',
'SET',
'distributed-lock:test',
'owner-token',
'IFEQ',
'owner-token',
'PX',
'45000',
]);
});
@@ -2,20 +2,6 @@ import { Injectable, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { RedisService } from './redis.service';
const RELEASE_LOCK_IF_OWNED_SCRIPT = `
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("DEL", KEYS[1])
end
return 0
`;
const EXTEND_LOCK_IF_OWNED_SCRIPT = `
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
end
return 0
`;
export interface DistributedLockOwnership {
lockKey: string;
lockOwnershipToken: string;
@@ -75,10 +61,9 @@ export class DistributedLockService {
const result = await this.redisService
.raw()
.sendCommand([
'EVAL',
RELEASE_LOCK_IF_OWNED_SCRIPT,
'1',
'DELEX',
lockOwnership.lockKey,
'IFEQ',
lockOwnership.lockOwnershipToken,
]);
return Number(result) === 1;
@@ -95,15 +80,10 @@ export class DistributedLockService {
const result = await this.redisService
.raw()
.sendCommand([
'EVAL',
EXTEND_LOCK_IF_OWNED_SCRIPT,
'1',
lockOwnership.lockKey,
lockOwnership.lockOwnershipToken,
String(lockExpirationDurationMilliseconds),
]);
return Number(result) === 1;
.sendCommand<
string | null
>(['SET', lockOwnership.lockKey, lockOwnership.lockOwnershipToken, 'IFEQ', lockOwnership.lockOwnershipToken, 'PX', String(lockExpirationDurationMilliseconds)]);
return result === 'OK';
}
public async runWithLock<TResult>(
@@ -5,6 +5,24 @@ function createRedisService(client: Record<string, unknown>) {
}
describe('RedisService stream pending recovery', () => {
it('destroys a dedicated client when connection startup fails without closing the shared client', async () => {
const error = new Error('connection failed');
const dedicated = {
on: jest.fn(),
connect: jest.fn().mockRejectedValue(error),
isOpen: true,
destroy: jest.fn(),
};
const shared = {
duplicate: jest.fn().mockReturnValue(dedicated),
destroy: jest.fn(),
};
const service = createRedisService(shared);
await expect(service.createDedicatedConnection()).rejects.toBe(error);
expect(dedicated.destroy).toHaveBeenCalledTimes(1);
expect(shared.destroy).not.toHaveBeenCalled();
});
it('normalizes XAUTOCLAIM array replies into stream entries', async () => {
const sendCommand = jest
.fn()
@@ -83,6 +83,20 @@ export class RedisService implements OnModuleDestroy {
return this.client;
}
public async createDedicatedConnection(): Promise<RedisService> {
const client = this.client.duplicate();
client.on('error', (error: unknown) => {
this.logger.warn(`Dedicated Redis connection error: ${String(error)}`);
});
try {
await client.connect();
return new RedisService(client);
} catch (error) {
if (client.isOpen) client.destroy();
throw error;
}
}
public async getJson<T>(key: string): Promise<T | null> {
const raw = await this.client.get(key);
if (!raw) return null;
@@ -142,6 +156,13 @@ export class RedisService implements OnModuleDestroy {
return this.client.del(key);
}
public async deleteStringIfEqual(
key: string,
value: string,
): Promise<number> {
return this.client.sendCommand<number>(['DELEX', key, 'IFEQ', value]);
}
public async appendStream(
key: string,
fields: Record<string, string>,
@@ -158,7 +179,10 @@ export class RedisService implements OnModuleDestroy {
}
: undefined;
return this.client.xAdd(key, '*', fields, trimOptions);
return this.client.xAdd(key, '*', fields, {
...trimOptions,
policy: 'ACKED',
});
}
public async createConsumerGroup(
@@ -174,11 +198,13 @@ export class RedisService implements OnModuleDestroy {
group: string,
consumer: string,
startId: string,
options?: { blockMs?: number; count?: number },
options?: { blockMs?: number; count?: number; claimMinIdleMs?: number },
): Promise<StreamReadResult[] | null> {
const readOptions: { BLOCK?: number; COUNT?: number } = {};
const readOptions: { BLOCK?: number; COUNT?: number; CLAIM?: number } = {};
if (options?.blockMs !== undefined) readOptions.BLOCK = options.blockMs;
if (options?.count !== undefined) readOptions.COUNT = options.count;
if (options?.claimMinIdleMs !== undefined)
readOptions.CLAIM = options.claimMinIdleMs;
const reply = await this.client.xReadGroup(
group,
@@ -244,6 +270,14 @@ export class RedisService implements OnModuleDestroy {
return this.client.xAck(key, group, ids);
}
public async acknowledgeAndDeleteStream(
key: string,
group: string,
ids: string | string[],
): Promise<void> {
await this.client.xAckDel(key, group, ids, 'ACKED');
}
public async deleteStreamEntries(
key: string,
ids: string | string[],
@@ -691,12 +691,20 @@ function createImportFixture() {
gameName: `player-${index + 1}`,
tagLine: 'KR1',
profileIconId: index + 1,
revisionDate: 0,
summonerLevel: 30,
tiers: [],
tierHistory: [],
matchIds: [],
championMasteries: [],
championRecords: {},
isPrimary: true,
createdAt: new Date('2026-07-12T00:00:00.000Z'),
updatedAt: new Date('2026-07-12T00:00:00.000Z'),
},
displayName: `player-${index + 1}`,
preferredPositions: [],
} as LegacyUserSnapshot,
} satisfies LegacyUserSnapshot,
]),
);
const participants = users.map(({ puuid }, index) => ({
@@ -3,6 +3,11 @@ import { presentPublicMatch } from './match-public.presenter';
describe('presentPublicMatch', () => {
it('whitelists recruiting fields and excludes result/code internals', () => {
const doubleElimination: NonNullable<
Parameters<typeof presentPublicMatch>[0]['doubleElimination']
> & { bracketMatches: Array<{ winnerTeamIndex: number }> } = {
bracketMatches: [{ winnerTeamIndex: 3 }],
};
const response = presentPublicMatch({
_id: oid('1'),
communityId: oid('2'),
@@ -60,7 +65,7 @@ describe('presentPublicMatch', () => {
tournamentCodeIds: [oid('6')],
tournamentCodes: [{ code: 'SECRET' }],
result: { winnerTeamIndex: 3, games: [], seriesResults: [] },
doubleElimination: { bracketMatches: [{ winnerTeamIndex: 3 }] },
doubleElimination,
resultPollState: { state: 'polling' },
legacyImportSource: { triggeringPuuid: 'secret' },
});
@@ -473,7 +473,7 @@ describe('MatchService', () => {
userId: targetUserId,
preferredPositions: [
{ position: 'TOP', priority: 1 },
{ position: 'JUNGLE', priority: 2 },
{ position: 'JGL', priority: 2 },
{ position: 'MID', priority: 3 },
],
},
@@ -489,7 +489,7 @@ describe('MatchService', () => {
communityId,
targetUserId,
mainPosition: 'MID',
subPosition: 'SUPPORT',
subPosition: 'SUP',
}),
).resolves.toBe(updatedMatch);
expect(MatchModel.findOneAndUpdate).toHaveBeenCalledWith(
@@ -503,9 +503,9 @@ describe('MatchService', () => {
$set: {
'participants.$[participant].preferredPositions': [
{ position: 'MID', priority: 1 },
{ position: 'SUPPORT', priority: 2 },
{ position: 'SUP', priority: 2 },
{ position: 'TOP', priority: 3 },
{ position: 'JUNGLE', priority: 4 },
{ position: 'JGL', priority: 4 },
],
},
},
@@ -9,7 +9,10 @@ import {
StaleAccountRepository,
} from './stale-account.repository';
type SchedulerInternals = BackgroundRefreshScheduler & {
type SchedulerInternals = Pick<
BackgroundRefreshScheduler,
keyof BackgroundRefreshScheduler
> & {
tick(): Promise<void>;
};
@@ -32,7 +35,7 @@ function candidate(puuid: string): StaleAccountCandidate {
function makeScheduler() {
const rawClient = {
sendCommand: jest.fn().mockResolvedValue(1),
sendCommand: jest.fn().mockResolvedValue([1, 1]),
};
const redis = {
getString: jest.fn().mockResolvedValue(null),
@@ -65,7 +68,7 @@ function makeScheduler() {
rateLimiter,
enrichmentService,
staleAccountRepository,
) as SchedulerInternals;
) as unknown as SchedulerInternals;
return {
scheduler,
@@ -84,7 +87,7 @@ describe('BackgroundRefreshScheduler', () => {
jest.restoreAllMocks();
});
it('schedules ticks without requiring RIOT_BG_REFRESH_ENABLED', () => {
it('schedules ticks without requiring RIOT_BG_REFRESH_ENABLED', async () => {
jest.useFakeTimers();
const setIntervalSpy = jest.spyOn(global, 'setInterval');
const { scheduler } = makeScheduler();
@@ -92,7 +95,7 @@ describe('BackgroundRefreshScheduler', () => {
scheduler.onModuleInit();
expect(setIntervalSpy).toHaveBeenCalledTimes(2);
scheduler.onModuleDestroy();
await scheduler.onModuleDestroy();
});
it('uses the dawn refresh gate ratio inside the KST background window', async () => {
@@ -203,12 +206,11 @@ describe('BackgroundRefreshScheduler', () => {
expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(2);
expect(rawClient.sendCommand).toHaveBeenCalledWith([
'INCR',
'riot:bg:fail:puuid-1',
]);
expect(rawClient.sendCommand).toHaveBeenCalledWith([
'EXPIRE',
'INCREX',
'riot:bg:fail:puuid-1',
'BYINT',
'1',
'EX',
'86400',
]);
});
@@ -228,10 +230,7 @@ describe('BackgroundRefreshScheduler', () => {
puuid: 'puuid-1',
});
expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(1);
expect(rawClient.sendCommand).not.toHaveBeenCalledWith([
'INCR',
'riot:bg:fail:puuid-1',
]);
expect(rawClient.sendCommand).not.toHaveBeenCalled();
});
it('skips candidates already in failure backoff', async () => {
@@ -251,14 +251,16 @@ export class BackgroundRefreshScheduler
private async bumpFailureCounter(puuid: string): Promise<void> {
const key = this.failureBackoffKey(puuid);
const client = this.redis.raw();
const response = await client.sendCommand(['INCR', key]);
await client.sendCommand([
'EXPIRE',
const response = await client.sendCommand([
'INCREX',
key,
'BYINT',
'1',
'EX',
String(FAILURE_BACKOFF_TTL_SECONDS),
]);
const count = Number(response ?? 0);
const count = Number(Array.isArray(response) ? response[0] : 0);
if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) {
this.logger.warn(
`Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`,
@@ -0,0 +1,22 @@
export const PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT = `
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
if not score or tonumber(score) > tonumber(ARGV[2]) then
return 0
end
-- Append before removing: a failed XADD must leave the job retryable.
-- Script atomicity prevents a second promoter from appending the same member.
redis.call('XADD', KEYS[2], 'ACKED', 'MAXLEN', '~', '10000', '*', 'job', ARGV[1])
redis.call('ZREM', KEYS[1], ARGV[1])
return 1
`;
export const RESCHEDULE_TOURNAMENT_RESULT_POLL_SCRIPT = `
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
if #pending == 0 then
return 0
end
-- Persist first: even a command error must never acknowledge unsaved work.
redis.call('ZADD', KEYS[2], ARGV[3], ARGV[4])
redis.call('XACKDEL', KEYS[1], ARGV[1], 'ACKED', 'IDS', 1, ARGV[2])
return 1
`;
@@ -6,6 +6,10 @@ import {
NotFoundException,
} from '@nestjs/common';
import { randomUUID } from 'crypto';
import {
PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT,
RESCHEDULE_TOURNAMENT_RESULT_POLL_SCRIPT,
} from './riot-job-queue.scripts';
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import type {
@@ -23,6 +27,7 @@ import {
EnqueueRefreshResult,
RIOT_JOB_AVG_SECONDS,
RIOT_JOB_DEDUP_TTL_SECONDS,
RIOT_JOB_GROUP,
RIOT_JOB_HIGH_COOLDOWN_SECONDS,
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_COOLDOWN_SECONDS,
@@ -213,13 +218,41 @@ export class RiotJobQueueService {
};
}
public async rescheduleTournamentResultPoll(params: {
stream: string;
entryId: string;
job: PollTournamentResultJob;
}): Promise<void> {
const retryCount = (params.job.retryCount ?? 0) + 1;
const retryJob: PollTournamentResultJob = { ...params.job, retryCount };
const delayMs = Math.min(
30_000 * 2 ** Math.min(retryCount - 1, 4),
300_000,
);
// Keep job identity stable so a recovered delivery shares its processing lock.
await this.redisService
.raw()
.sendCommand([
'EVAL',
RESCHEDULE_TOURNAMENT_RESULT_POLL_SCRIPT,
'2',
params.stream,
RIOT_JOB_SCHEDULED_ZSET,
RIOT_JOB_GROUP,
params.entryId,
String(Date.now() + delayMs),
JSON.stringify(retryJob),
]);
}
public async promoteDueScheduledJobs(limit = 100): Promise<number> {
const client = this.redisService.raw();
const now = Date.now();
const entries = await client.sendCommand<string[]>([
'ZRANGEBYSCORE',
RIOT_JOB_SCHEDULED_ZSET,
'-inf',
String(Date.now()),
String(now),
'LIMIT',
'0',
String(limit),
@@ -227,16 +260,22 @@ export class RiotJobQueueService {
let promoted = 0;
for (const entry of entries) {
const removed = await client.sendCommand<number>([
'ZREM',
RIOT_JOB_SCHEDULED_ZSET,
entry,
]);
if (removed !== 1) continue;
const job = this.parseScheduledJob(entry);
if (!job) continue;
await this.appendJob(job);
promoted += 1;
if (!job) {
await client.zRem(RIOT_JOB_SCHEDULED_ZSET, entry);
continue;
}
promoted += Number(
await client.sendCommand([
'EVAL',
PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT,
'2',
RIOT_JOB_SCHEDULED_ZSET,
this.resolveStream(job.priority),
entry,
String(now),
]),
);
}
return promoted;
@@ -300,7 +339,10 @@ export class RiotJobQueueService {
};
await Promise.all([
this.setStatus(next),
this.redisService.delete(this.dedupKey(job.puuid, job.queueGroup)),
this.redisService.deleteStringIfEqual(
this.dedupKey(job.puuid, job.queueGroup),
job.jobId,
),
]);
return this.toEvent(next);
}
@@ -321,7 +363,10 @@ export class RiotJobQueueService {
};
await Promise.all([
this.setStatus(next),
this.redisService.delete(this.dedupKey(job.puuid, job.queueGroup)),
this.redisService.deleteStringIfEqual(
this.dedupKey(job.puuid, job.queueGroup),
job.jobId,
),
]);
return this.toEvent(next);
}
@@ -0,0 +1,159 @@
import type { StreamEntry } from '../global/redis/redis.service';
import type { PollTournamentResultJob } from '../tournament-result/tournament-result.types';
import { RiotJobWorkerService } from './riot-job-worker.service';
import { RIOT_JOB_HIGH_STREAM } from './riot-job-queue.types';
const job: PollTournamentResultJob = {
kind: 'poll-tournament-result',
jobId: 'poll-1',
matchId: 'match-1',
attempt: 3,
expandedToAllPuuids: true,
enqueuedAt: 1,
priority: 'high',
};
function harness() {
const entry: StreamEntry = {
id: '1-0',
message: { job: JSON.stringify(job) },
};
const pending = new Set([entry.id]);
const scheduled: PollTournamentResultJob[] = [];
const processPollJob = jest.fn().mockResolvedValue(undefined);
const acknowledge = jest.fn().mockImplementation(async () => {
pending.delete(entry.id);
await Promise.resolve();
});
const reschedule = jest
.fn()
.mockImplementation(
async (params: {
stream: string;
entryId: string;
job: PollTournamentResultJob;
}) => {
scheduled.push(params.job);
pending.delete(params.entryId);
await Promise.resolve();
},
);
const locks = {
runWithRenewedLock: jest
.fn()
.mockImplementation(
async (_options: unknown, operation: () => Promise<unknown>) =>
operation(),
),
};
const worker = new RiotJobWorkerService(
{ acknowledgeAndDeleteStream: acknowledge } as never,
locks as never,
{ rescheduleTournamentResultPoll: reschedule } as never,
{} as never,
{} as never,
{} as never,
{ processPollJob } as never,
{} as never,
{} as never,
);
const process = () =>
(
worker as unknown as {
processEntry(stream: string, entry: StreamEntry): Promise<void>;
}
).processEntry(RIOT_JOB_HIGH_STREAM, entry);
return {
entry,
pending,
scheduled,
processPollJob,
acknowledge,
reschedule,
locks,
process,
};
}
describe('Tournament result worker retry delivery', () => {
it.each(['match lock unavailable', 'database write failed'])(
'preserves failed work in delayed retry instead of completing it: %s',
async (reason) => {
const h = harness();
h.processPollJob.mockRejectedValue(new Error(reason));
await h.process();
expect(h.scheduled).toEqual([job]);
expect(h.pending.size).toBe(0);
expect(h.acknowledge).not.toHaveBeenCalled();
},
);
it('leaves the original pending when saving the delayed retry fails', async () => {
const h = harness();
h.processPollJob.mockRejectedValue(new Error('database write failed'));
h.reschedule.mockRejectedValue(new Error('Redis unavailable'));
await expect(h.process()).rejects.toThrow('Redis unavailable');
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.scheduled).toEqual([]);
expect(h.acknowledge).not.toHaveBeenCalled();
});
it('retains the original while retry persistence is still in flight', async () => {
const h = harness();
h.processPollJob.mockRejectedValue(new Error('match lock unavailable'));
let releaseRetry!: () => void;
let retryStarted!: () => void;
const started = new Promise<void>((resolve) => {
retryStarted = resolve;
});
const retry = new Promise<void>((resolve) => {
releaseRetry = resolve;
});
h.reschedule.mockImplementation(async () => {
retryStarted();
await retry;
h.scheduled.push(job);
h.pending.delete(h.entry.id);
});
const processing = h.process();
// Race against completion so the buggy worker fails instead of hanging.
await Promise.race([started, processing]);
try {
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.acknowledge).not.toHaveBeenCalled();
} finally {
releaseRetry();
await processing;
}
});
it('acknowledges successfully processed work without scheduling retry', async () => {
const h = harness();
await h.process();
expect(h.pending.size).toBe(0);
expect(h.scheduled).toEqual([]);
expect(h.acknowledge).toHaveBeenCalledTimes(1);
});
it('does not acknowledge work owned by another job worker', async () => {
const h = harness();
h.locks.runWithRenewedLock.mockResolvedValue(null);
await h.process();
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.processPollJob).not.toHaveBeenCalled();
expect(h.scheduled).toEqual([]);
});
it('leaves a successful job pending if its acknowledgement fails', async () => {
const h = harness();
h.acknowledge.mockRejectedValue(new Error('acknowledgement unavailable'));
await expect(h.process()).rejects.toThrow('acknowledgement unavailable');
expect(h.pending.has(h.entry.id)).toBe(true);
expect(h.scheduled).toEqual([]);
});
});
@@ -1,12 +1,17 @@
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { RedisService, StreamEntry } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
import { RiotRecentMatchesService } from '../riot/riot-recent-matches.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { RIOT_JOB_GROUP, RiotRefreshSubJob } from './riot-job-queue.types';
import {
RIOT_JOB_GROUP,
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_STREAM,
RiotRefreshSubJob,
} from './riot-job-queue.types';
import { RiotJobQueueService } from './riot-job-queue.service';
import { RiotJobWorkerService } from './riot-job-worker.service';
import { RiotRefreshGateway } from './riot-refresh.gateway';
@@ -16,9 +21,17 @@ function execResult<T>(value: T) {
}
function makeWorker() {
const readerClient = { isOpen: true, destroy: jest.fn() };
const streamReader = {
raw: jest.fn().mockReturnValue(readerClient),
readGroup: jest.fn().mockResolvedValue(null),
} as unknown as jest.Mocked<RedisService>;
const redisService = {
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
acknowledgeAndDeleteStream: jest.fn().mockResolvedValue(undefined),
createConsumerGroup: jest.fn().mockResolvedValue(undefined),
createDedicatedConnection: jest.fn().mockResolvedValue(streamReader),
raw: jest.fn(),
readGroup: jest.fn(),
} as unknown as jest.Mocked<RedisService>;
const distributedLockService = {
runWithRenewedLock: jest
@@ -33,6 +46,7 @@ function makeWorker() {
),
} as unknown as jest.Mocked<DistributedLockService>;
const jobQueueService = {
promoteDueScheduledJobs: jest.fn().mockResolvedValue(0),
markProcessing: jest.fn().mockResolvedValue(undefined),
markCompleted: jest.fn().mockResolvedValue({
parentJobId: 'parent-1',
@@ -87,10 +101,106 @@ function makeWorker() {
connectionService,
refreshGateway,
enrichmentService,
streamReader,
readerClient,
};
}
type WorkerInternals = {
stopped: boolean;
loop(): Promise<void>;
tryBecomeLeader(): Promise<boolean>;
readOne(
stream: string,
blockMs: number,
): Promise<{ stream: string; message: StreamEntry } | null>;
processEntry(stream: string, entry: StreamEntry): Promise<void>;
};
describe('RiotJobWorkerService', () => {
it('connects a dedicated reader after creating groups and only closes that reader', async () => {
const { worker, redisService, streamReader, readerClient } = makeWorker();
const internal = worker as unknown as WorkerInternals;
jest.spyOn(internal, 'loop').mockResolvedValue(undefined);
await worker.onModuleInit();
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_GROUP,
);
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
RIOT_JOB_LOW_STREAM,
RIOT_JOB_GROUP,
);
expect(
redisService.createConsumerGroup.mock.invocationCallOrder[1],
).toBeLessThan(
redisService.createDedicatedConnection.mock.invocationCallOrder[0],
);
await internal.readOne(RIOT_JOB_HIGH_STREAM, 100);
expect(streamReader.readGroup).toHaveBeenCalledWith(
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_GROUP,
expect.any(String),
'>',
{ count: 1, blockMs: 100, claimMinIdleMs: 1800000 },
);
expect(redisService.readGroup).not.toHaveBeenCalled();
await worker.onModuleDestroy();
expect(readerClient.destroy).toHaveBeenCalledTimes(1);
expect(redisService.raw).not.toHaveBeenCalled();
expect(internal.stopped).toBe(true);
});
it.each([true, false])(
'preserves high-before-low consumption (high available: %s)',
async (highAvailable) => {
const { worker, jobQueueService } = makeWorker();
const internal = worker as unknown as WorkerInternals;
const entry = { id: '1-0', message: { job: '{}' } };
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
const read = jest
.spyOn(internal, 'readOne')
.mockImplementation(async (stream) => {
if (stream === RIOT_JOB_HIGH_STREAM && !highAvailable) return null;
return { stream, message: entry };
});
const process = jest
.spyOn(internal, 'processEntry')
.mockImplementation(async () => {
internal.stopped = true;
});
await internal.loop();
expect(jobQueueService.promoteDueScheduledJobs).toHaveBeenCalledTimes(1);
expect(read.mock.calls).toEqual(
highAvailable
? [[RIOT_JOB_HIGH_STREAM, 100]]
: [
[RIOT_JOB_HIGH_STREAM, 100],
[RIOT_JOB_LOW_STREAM, 1000],
],
);
expect(process).toHaveBeenCalledWith(
highAvailable ? RIOT_JOB_HIGH_STREAM : RIOT_JOB_LOW_STREAM,
entry,
);
},
);
it('does not start processing a message returned during shutdown', async () => {
const { worker } = makeWorker();
const internal = worker as unknown as WorkerInternals;
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
jest.spyOn(internal, 'readOne').mockImplementation(async (stream) => {
await worker.onModuleDestroy();
return { stream, message: { id: '1-0', message: {} } };
});
const process = jest.spyOn(internal, 'processEntry');
await internal.loop();
expect(process).not.toHaveBeenCalled();
});
it('acknowledges and deletes invalid stream entries', async () => {
const { worker, redisService } = makeWorker();
@@ -106,15 +216,11 @@ describe('RiotJobWorkerService', () => {
message: { job: 'not-json' },
});
expect(redisService.acknowledgeStream).toHaveBeenCalledWith(
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
'riot:queue:high',
RIOT_JOB_GROUP,
'bad-1',
);
expect(redisService.deleteStreamEntries).toHaveBeenCalledWith(
'riot:queue:high',
'bad-1',
);
});
it('refreshes Riot identity and league entries while processing user history refresh jobs', async () => {
@@ -165,19 +271,15 @@ describe('RiotJobWorkerService', () => {
expect(
enrichmentService.refreshRiotIdentity.mock.invocationCallOrder[0],
).toBeLessThan(jobQueueService.markCompleted.mock.invocationCallOrder[0]);
expect(redisService.acknowledgeStream).toHaveBeenCalledWith(
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
'riot:queue:high',
RIOT_JOB_GROUP,
'1-0',
);
expect(redisService.deleteStreamEntries).toHaveBeenCalledWith(
'riot:queue:high',
'1-0',
);
expect(
redisService.acknowledgeStream.mock.invocationCallOrder[0],
jobQueueService.markCompleted.mock.invocationCallOrder[0],
).toBeLessThan(
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
redisService.acknowledgeAndDeleteStream.mock.invocationCallOrder[0],
);
});
@@ -249,7 +351,6 @@ describe('RiotJobWorkerService', () => {
message: { job: JSON.stringify(job) },
});
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
expect(redisService.acknowledgeAndDeleteStream).not.toHaveBeenCalled();
});
});
@@ -33,7 +33,6 @@ const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000;
const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000;
const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000;
const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1;
@Injectable()
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
@@ -41,6 +40,7 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private leaderLockOwnership: DistributedLockOwnership | undefined;
private stopped = false;
private streamReader?: RedisService;
constructor(
private readonly redisService: RedisService,
@@ -56,11 +56,14 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
public async onModuleInit() {
await this.ensureConsumerGroups();
this.streamReader = await this.redisService.createDedicatedConnection();
void this.loop();
}
public async onModuleDestroy() {
this.stopped = true;
// Interrupt a blocking read without delaying shutdown or other Redis users.
if (this.streamReader?.raw().isOpen) this.streamReader.raw().destroy();
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
@@ -78,6 +81,7 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
while (!this.stopped) {
try {
const isLeader = await this.tryBecomeLeader();
if (this.stopped) break;
if (!isLeader) {
await this.sleep(1_000);
continue;
@@ -85,14 +89,14 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
await this.jobQueueService.promoteDueScheduledJobs();
const entry =
(await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ??
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
(await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ??
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
if (this.stopped) break;
if (!entry) continue;
await this.processEntry(entry.stream, entry.message);
} catch (error) {
if (this.stopped) break;
this.logger.warn(
`Riot job worker loop failed: ${
error instanceof Error ? error.message : error
@@ -113,7 +117,7 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
return;
}
const didProcessJob = await this.distributedLockService.runWithRenewedLock(
const outcome = await this.distributedLockService.runWithRenewedLock(
{
lockKey: this.jobProcessingLockKey(job),
lockExpirationDurationMilliseconds:
@@ -122,19 +126,38 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.processJob(job);
return true;
try {
await this.processJob(job);
return 'completed' as const;
} catch (error) {
if (!this.isTournamentResultJob(job)) throw error;
this.logger.warn(
`Retrying tournament result poll job ${job.jobId} for match ${job.matchId}: ${
error instanceof Error ? error.message : String(error)
}`,
);
// The handoff saves retry work before acknowledging the original.
// If it fails, leave the original pending for abandoned-job recovery.
await this.jobQueueService.rescheduleTournamentResultPoll({
stream,
entryId: entry.id,
job,
});
return 'rescheduled' as const;
}
},
);
if (!didProcessJob) {
if (!outcome) {
this.logger.debug(
`Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`,
);
return;
}
await this.acknowledgeAndDelete(stream, entry.id);
if (outcome === 'completed') {
await this.acknowledgeAndDelete(stream, entry.id);
}
}
private async processJob(job: RiotJobPayload): Promise<void> {
@@ -163,12 +186,7 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
this.refreshGateway.broadcastRefreshEvent(event);
} catch (error) {
if (this.isTournamentResultJob(job)) {
this.logger.warn(
`Tournament result poll job failed ${job.jobId}: ${
error instanceof Error ? error.message : error
}`,
);
return;
throw error;
}
if (this.isLegacyImportJob(job)) {
this.logger.warn(
@@ -193,23 +211,11 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
stream: string,
id: string,
): Promise<void> {
await this.redisService.acknowledgeStream(stream, RIOT_JOB_GROUP, id);
await this.deleteAcknowledgedEntry(stream, id);
}
private async deleteAcknowledgedEntry(
stream: string,
id: string,
): Promise<void> {
try {
await this.redisService.deleteStreamEntries(stream, id);
} catch (error) {
this.logger.warn(
`Failed to delete acknowledged Riot job stream entry ${stream} ${id}: ${
error instanceof Error ? error.message : error
}`,
);
}
await this.redisService.acknowledgeAndDeleteStream(
stream,
RIOT_JOB_GROUP,
id,
);
}
private async refreshRiotIdentity(job: RiotRefreshSubJob): Promise<void> {
@@ -294,32 +300,23 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
stream: string,
blockMs: number,
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.readGroup(
if (!this.streamReader)
throw new Error('Riot stream reader is not connected.');
const result = await this.streamReader.readGroup(
stream,
RIOT_JOB_GROUP,
this.consumerIdentifier,
'>',
{ count: 1, blockMs },
{
count: 1,
blockMs,
claimMinIdleMs: STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
},
);
const message = result?.[0]?.messages[0];
return message ? { stream, message } : null;
}
private async readOneStalePendingMessage(
stream: string,
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.claimStalePendingStreamMessages({
key: stream,
group: RIOT_JOB_GROUP,
consumer: this.consumerIdentifier,
stalePendingMessageMinimumIdleMilliseconds:
STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
count: STALE_PENDING_MESSAGE_CLAIM_COUNT,
});
const message = result.messages[0];
return message ? { stream, message } : null;
}
private async ensureConsumerGroups(): Promise<void> {
await Promise.all(
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
@@ -7,7 +7,10 @@ import mongoose from 'mongoose';
import { TournamentCodeService } from './tournament-code.service';
import { buildDoubleEliminationBracket } from '../double-elimination/double-elimination.util';
type TournamentCodeServiceInternals = TournamentCodeService & {
type TournamentCodeServiceInternals = Pick<
TournamentCodeService,
keyof TournamentCodeService
> & {
resolveRequiredSlots(match: MatchDocument): number;
resolveSeriesRefForGameNumber(
match: MatchDocument,
@@ -0,0 +1,227 @@
import mongoose from 'mongoose';
import type { MatchDocument } from '../global/database/schema';
import type { RiotJobPriority } from '../riot-job-queue/riot-job-queue.types';
import { TournamentResultService } from './tournament-result.service';
import type { PollTournamentResultJob } from './tournament-result.types';
describe('TournamentResultService poll delivery reliability', () => {
it('lets a fast consumer execute the initial poll after the scheduling lock is released', async () => {
const harness = createHarness({ consumeImmediately: true });
await harness.service.scheduleInitialPoll(harness.match._id);
expect(harness.publications[0].lockHeld).toBe(false);
expect(harness.match.resultPollState).toMatchObject({
lastPollOutcome: 'not-found',
lastPolledAt: expect.any(Date),
state: 'polling',
});
expect(harness.publications).toHaveLength(2);
expect(harness.publications[1].job.attempt).toBe(2);
expect(harness.publications[1].deferUntilMs).toEqual(expect.any(Number));
});
it('rejects a busy match lock so the caller can retain the poll for retry', async () => {
const harness = createHarness();
harness.locks.rejectAcquisition = true;
await expect(harness.service.processPollJob(harness.job)).rejects.toThrow();
expect(harness.match.resultPollState).toBeUndefined();
expect(harness.publications).toEqual([]);
});
it('does not publish a poll when the scheduling lock cannot be acquired', async () => {
const harness = createHarness();
harness.locks.rejectAcquisition = true;
await expect(
harness.service.scheduleInitialPoll(harness.match._id),
).resolves.toBeNull();
expect(harness.publications).toEqual([]);
expect(harness.match.resultPollState).toBeUndefined();
});
it('does not publish a poll until teams have been built', async () => {
const harness = createHarness();
harness.match.teamBuiltAt = undefined;
await expect(
harness.service.scheduleInitialPoll(harness.match._id),
).resolves.toBeNull();
expect(harness.publications).toEqual([]);
});
it('does not publish a poll for a completed Competition', async () => {
const harness = createHarness({ completed: true });
await expect(
harness.service.scheduleInitialPoll(harness.match._id),
).resolves.toBeNull();
expect(harness.publications).toEqual([]);
});
it('does not publish a poll when saving the prepared match fails', async () => {
const harness = createHarness();
harness.match.save.mockRejectedValueOnce(new Error('Database unavailable'));
await expect(
harness.service.scheduleInitialPoll(harness.match._id),
).rejects.toThrow('Database unavailable');
expect(harness.publications).toEqual([]);
expect(harness.locks.held.size).toBe(0);
});
});
class InMemoryLocks {
readonly held = new Set<string>();
rejectAcquisition = false;
async runWithLock<T>(
options: { lockKey: string },
operation: () => Promise<T>,
): Promise<T | null> {
if (this.rejectAcquisition || this.held.has(options.lockKey)) return null;
this.held.add(options.lockKey);
try {
return await operation();
} finally {
this.held.delete(options.lockKey);
}
}
runWithRenewedLock<T>(
options: { lockKey: string },
operation: () => Promise<T>,
): Promise<T | null> {
return this.runWithLock(options, operation);
}
}
type PollRequest = {
matchId: string;
attempt: number;
expandedToAllPuuids: boolean;
priority: RiotJobPriority;
deferUntilMs?: number;
};
function createHarness(
options: { consumeImmediately?: boolean; completed?: boolean } = {},
) {
const users = Array.from({ length: 10 }, (_, index) => ({
userId: new mongoose.Types.ObjectId(),
puuid: `poll-reliability-puuid-${index}`,
}));
const codeId = new mongoose.Types.ObjectId();
const match = {
_id: new mongoose.Types.ObjectId(),
communityId: new mongoose.Types.ObjectId(),
status: 'team-built',
teamBuiltAt: new Date(),
tournamentCodeIds: [codeId],
participants: users,
teams: [users.slice(0, 5), users.slice(5)].map((members, index) => ({
teamIndex: index + 1,
captainUserId: members[0].userId,
members: members.map(({ userId }) => ({ userId })),
})),
save: jest.fn().mockResolvedValue(undefined),
} as unknown as MatchDocument & { save: jest.Mock };
const phase = options.completed ? 'completed' : 'ready';
const competition = {
_id: new mongoose.Types.ObjectId(),
revision: 0,
phase,
teams: match.teams.map((team) => ({
teamId: new mongoose.Types.ObjectId(),
members: team.members,
})),
projection: { phase, series: [], gameSlots: [], games: [] },
};
const locks = new InMemoryLocks();
const publications: Array<{
job: PollTournamentResultJob;
lockHeld: boolean;
deferUntilMs?: number;
}> = [];
const matchLockKey = `distributed-lock:tournament-result:process-poll-job:match:${match._id.toString()}`;
const queue = {
enqueueTournamentResultPoll: async (request: PollRequest) => {
const job: PollTournamentResultJob = {
kind: 'poll-tournament-result',
jobId: `poll-delivery-${publications.length + 1}`,
matchId: request.matchId,
attempt: request.attempt,
expandedToAllPuuids: request.expandedToAllPuuids,
priority: request.priority,
enqueuedAt: Date.now(),
};
publications.push({
job,
lockHeld: locks.held.has(matchLockKey),
deferUntilMs: request.deferUntilMs,
});
if (options.consumeImmediately && request.deferUntilMs === undefined) {
await service.processPollJob(job);
}
return job;
},
};
const connection = {
getModel: (name: string) => {
switch (name) {
case 'match':
return { findById: () => Promise.resolve(match) };
case 'tournamentCode':
return {
find: () => ({
sort: () => ({
exec: () =>
Promise.resolve([
{ _id: codeId, state: 'reserved', code: 'TOURNAMENT-1' },
]),
}),
}),
};
case 'community':
return {
findById: () => ({
select: () => ({
lean: () => ({
exec: () => Promise.resolve({ riotRegion: 'asia' }),
}),
}),
}),
};
default:
throw new Error(`Unexpected model ${name}`);
}
},
};
const service = new TournamentResultService(
connection as never,
{ getMatchIdsByPuuid: () => Promise.resolve([]) } as never,
queue as never,
{} as never,
{} as never,
locks as never,
{ bootstrap: () => Promise.resolve(competition) } as never,
{} as never,
{ provisionReadyAssignments: () => Promise.resolve(undefined) } as never,
);
const job: PollTournamentResultJob = {
kind: 'poll-tournament-result',
jobId: 'retry-after-contention',
matchId: match._id.toString(),
attempt: 1,
expandedToAllPuuids: false,
priority: 'high',
enqueuedAt: Date.now(),
};
return { service, match, job, locks, publications };
}
@@ -11,6 +11,10 @@ describe('TournamentResultService Competition polling', () => {
});
await harness.service.processPollJob({
kind: 'poll-tournament-result',
jobId: 'poll-job-1',
enqueuedAt: Date.parse('2026-07-12T10:00:00.000Z'),
priority: 'high',
matchId: harness.match._id.toString(),
attempt: 1,
expandedToAllPuuids: true,
@@ -62,6 +66,10 @@ describe('TournamentResultService Competition polling', () => {
});
await harness.service.processPollJob({
kind: 'poll-tournament-result',
jobId: 'poll-job-1',
enqueuedAt: Date.parse('2026-07-12T10:00:00.000Z'),
priority: 'high',
matchId: harness.match._id.toString(),
attempt: 1,
expandedToAllPuuids: true,
@@ -99,6 +107,10 @@ describe('TournamentResultService Competition polling', () => {
});
await harness.service.processPollJob({
kind: 'poll-tournament-result',
jobId: 'poll-job-1',
enqueuedAt: Date.parse('2026-07-12T10:00:00.000Z'),
priority: 'high',
matchId: harness.match._id.toString(),
attempt: 1,
expandedToAllPuuids: true,
@@ -107,7 +107,7 @@ export class TournamentResultService {
priority: RiotJobPriority = 'high',
): Promise<EnqueueTournamentResultPollResult | null> {
const matchIdentifier = this.resolveMatchIdentifier(matchOrId);
return this.distributedLockService.runWithLock(
const preparedMatchId = await this.distributedLockService.runWithLock(
{
lockKey: this.tournamentResultPollProcessingLockKey(matchIdentifier),
lockExpirationDurationMilliseconds:
@@ -115,14 +115,22 @@ export class TournamentResultService {
lockAcquireTimeoutMilliseconds: 1_000,
lockRetryDelayMilliseconds: 100,
},
async () => this.scheduleInitialPollWhileLockOwned(matchOrId, priority),
async () => this.prepareInitialPollWhileLockOwned(matchOrId),
);
if (!preparedMatchId) return null;
// A fast worker must not see the job while its preparation owns the same lock.
return this.riotJobQueueService.enqueueTournamentResultPoll({
matchId: preparedMatchId,
attempt: 1,
expandedToAllPuuids: false,
priority,
});
}
private async scheduleInitialPollWhileLockOwned(
private async prepareInitialPollWhileLockOwned(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
priority: RiotJobPriority,
): Promise<EnqueueTournamentResultPollResult | null> {
): Promise<string | null> {
const match = await this.loadMatch(matchOrId);
if (!this.isReadyToPoll(match)) return null;
let competition = await this.ensureCompetition(match);
@@ -143,12 +151,7 @@ export class TournamentResultService {
};
await match.save();
return this.riotJobQueueService.enqueueTournamentResultPoll({
matchId: match._id.toString(),
attempt: 1,
expandedToAllPuuids: false,
priority,
});
return match._id.toString();
}
public async scheduleManualPoll(params: {
@@ -260,8 +263,8 @@ export class TournamentResultService {
},
);
if (didProcess) return;
this.logger.debug(
`Skipped tournament result poll job ${job.jobId}; another worker is processing match ${job.matchId}.`,
throw new Error(
`Tournament result poll job ${job.jobId} could not acquire the processing lock for match ${job.matchId}.`,
);
}
@@ -8,6 +8,8 @@ export interface PollTournamentResultJob {
expandedToAllPuuids: boolean;
enqueuedAt: number;
priority: RiotJobPriority;
/** Delivery failures, separate from the regular polling attempt. */
retryCount?: number;
}
export interface EnqueueTournamentResultPollResult {
@@ -94,7 +94,7 @@ export async function discoverApiRoutes(rootDir: string): Promise<ApiRoute[]> {
routes.set(key, {
method,
route,
controllerFile: path.relative(rootDir, file),
controllerFile: path.relative(rootDir, file).split(path.sep).join('/'),
});
}
}
+7
View File
@@ -0,0 +1,7 @@
{
"rootDir": "..",
"testEnvironment": "node",
"testRegex": "test/.*\\.integration-spec\\.ts$",
"transform": { "^.+\\.ts$": "ts-jest" },
"testTimeout": 15000
}
@@ -0,0 +1,662 @@
import { createClient } from 'redis';
import { RedisService } from '../src/modules/global/redis/redis.service';
import { AuctionRedisService } from '../src/modules/auction/redis/auction-redis.service';
import { AuctionRedisCommandService } from '../src/modules/auction/redis/auction-redis-command.service';
import { AuctionService } from '../src/modules/auction/auction.service';
import { buildAuctionRedisKeys } from '../src/modules/auction/redis/auction-redis.keys';
import { RiotJobQueueService } from '../src/modules/riot-job-queue/riot-job-queue.service';
import {
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_STREAM,
RIOT_JOB_SCHEDULED_ZSET,
} from '../src/modules/riot-job-queue/riot-job-queue.types';
import { buildAuctionReserveActionRequestScriptInvocation } from '../src/modules/auction/redis/auction-redis.reserve-action';
import { buildAuctionApplyTransitionScriptInvocation } from '../src/modules/auction/redis/auction-redis.apply-transition';
import { DistributedLockService } from '../src/modules/global/redis/distributed-lock.service';
import { BackgroundRefreshScheduler } from '../src/modules/riot-background-refresh/background-refresh.scheduler';
import { setTimeout as delay } from 'node:timers/promises';
import {
CommunityStatsCacheKey,
CommunityStatsCacheService,
} from '../src/modules/community-stats/community-stats-cache.service';
import { CommunityStatsService } from '../src/modules/community-stats/community-stats.service';
import { PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT } from '../src/modules/riot-job-queue/riot-job-queue.scripts';
import mongoose from 'mongoose';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new RedisService(client);
const auctionRedis = new AuctionRedisService(client);
const queue = new RiotJobQueueService({} as never, redis);
beforeAll(async () => {
if (process.env.REDIS_TEST_ISOLATED !== '1' || !process.env.REDIS_TEST_URL) {
throw new Error(
'Run via pnpm --filter api test:redis to create an isolated Redis process.',
);
}
client.on('error', () => {});
await client.connect();
});
describe('Redis native operations', () => {
it('serves normal commands while a dedicated stream connection is blocked', async () => {
const reader = await redis.createDedicatedConnection();
try {
await redis.createConsumerGroup('stream', 'workers', '0');
const pending = reader.readGroup('stream', 'workers', 'reader', '>', {
blockMs: 2000,
count: 1,
});
void pending.catch(() => {}); // A failed assertion must still allow connection cleanup.
let isBlocked = false;
for (let i = 0; i < 100; i++) {
const clients = await client.sendCommand<string>(['CLIENT', 'LIST']);
if (/flags=b\b.*cmd=xreadgroup/.test(clients)) {
isBlocked = true;
break;
}
await delay(5);
}
expect(isBlocked).toBe(true);
await redis.setString('cache', 'available');
expect(await redis.getString('cache')).toBe('available');
const id = await redis.appendStream('stream', { job: 'wake' });
expect((await pending)?.[0].messages[0].id).toBe(id);
} finally {
if (reader.raw().isOpen) reader.raw().destroy();
}
});
it('claims abandoned pending messages before reading new work with the same idle threshold', async () => {
await redis.createConsumerGroup('stream', 'workers', '0');
const oldId = await redis.appendStream('stream', { job: 'old' });
await redis.readGroup('stream', 'workers', 'dead-worker', '>', {
count: 1,
});
const newId = await redis.appendStream('stream', { job: 'new' });
await client.sendCommand([
'XCLAIM',
'stream',
'workers',
'dead-worker',
'0',
oldId,
'IDLE',
'1800001',
]);
const recovered = await redis.readGroup(
'stream',
'workers',
'live-worker',
'>',
{ count: 1, claimMinIdleMs: 1800000 },
);
expect(recovered?.[0].messages[0]).toEqual(
expect.objectContaining({ id: oldId, message: { job: 'old' } }),
);
const fresh = await redis.readGroup(
'stream',
'workers',
'live-worker',
'>',
{ count: 1, claimMinIdleMs: 1800000 },
);
expect(fresh?.[0].messages[0].id).toBe(newId);
expect(
await redis.readGroup('stream', 'workers', 'other-worker', '>', {
count: 1,
claimMinIdleMs: 1800000,
}),
).toBeNull();
});
it('acknowledges atomically but retains a message until every group has processed it', async () => {
for (const group of ['workers', 'audit'])
await redis.createConsumerGroup('stream', group, '0');
const id = await redis.appendStream('stream', { job: 'one' });
await redis.readGroup('stream', 'workers', 'worker', '>', { count: 1 });
await redis.acknowledgeAndDeleteStream('stream', 'workers', id);
expect(await client.xLen('stream')).toBe(1);
expect((await client.xPending('stream', 'workers')).pending).toBe(0);
await redis.readGroup('stream', 'audit', 'auditor', '>', { count: 1 });
await redis.acknowledgeAndDeleteStream('stream', 'audit', id);
expect(await client.xLen('stream')).toBe(0);
await expect(
redis.acknowledgeAndDeleteStream('stream', 'workers', id),
).resolves.toBeUndefined();
});
it('never trims unread or pending jobs even when MAXLEN is exceeded', async () => {
await redis.createConsumerGroup('stream', 'workers', '0');
const first = await redis.appendStream(
'stream',
{ job: 'pending' },
{ maxLen: 1 },
);
await redis.readGroup('stream', 'workers', 'worker', '>', { count: 1 });
for (let i = 0; i < 220; i++)
await redis.appendStream('stream', { job: String(i) }, { maxLen: 1 });
expect(await client.xLen('stream')).toBe(221);
expect((await client.xRange('stream', first, first))[0].message).toEqual({
job: 'pending',
});
});
it('does not release, renew, or recreate a lock belonging to another owner', async () => {
const locks = new DistributedLockService(redis);
const owner = await locks.acquireLock({
lockKey: 'lock',
lockOwnershipToken: 'old',
lockExpirationDurationMilliseconds: 1000,
});
expect(owner).not.toBeNull();
expect(await locks.extendLockIfOwned(owner!, 4000)).toBe(true);
expect(await client.pTTL('lock')).toBeGreaterThan(3000);
await client.set('lock', 'new', { PX: 5000 });
expect(await locks.releaseLockIfOwned(owner!)).toBe(false);
expect(await auctionRedis.releaseTokenLock('lock', 'old')).toBe(false);
expect(await locks.extendLockIfOwned(owner!, 9000)).toBe(false);
expect(await client.get('lock')).toBe('new');
expect(await auctionRedis.releaseTokenLock('lock', 'new')).toBe(true);
expect(await locks.extendLockIfOwned(owner!)).toBe(false);
expect(await client.exists('lock')).toBe(0);
});
it('does not clear the dedup token of a newer refresh job', async () => {
const job = {
jobId: 'old',
puuid: 'player',
queueGroup: 'summoners-rift',
} as const;
const key = 'riot:job:dedup:player:summoners-rift';
await redis.setJson(
'riot:job:status:old',
{ ...job, state: 'processing' },
86400,
);
await redis.setString(key, 'new', 300);
await queue.markCompleted(job as never);
expect(await redis.getString(key)).toBe('new');
await queue.markFailed(job as never, new Error('failure'));
expect(await redis.getString(key)).toBe('new');
await redis.setString(key, 'old', 300);
await queue.markCompleted(job as never);
expect(await redis.getString(key)).toBeNull();
});
it('increments failure counts without losing concurrent updates and refreshes the TTL', async () => {
const scheduler = Object.assign(
Object.create(BackgroundRefreshScheduler.prototype),
{
redis,
logger: { warn: jest.fn() },
},
) as {
bumpFailureCounter(puuid: string): Promise<void>;
clearFailureCounter(puuid: string): Promise<void>;
};
await client.set('riot:bg:fail:player', '1', { EX: 10 });
await Promise.all(
Array.from({ length: 10 }, () => scheduler.bumpFailureCounter('player')),
);
expect(await client.get('riot:bg:fail:player')).toBe('11');
expect(await client.ttl('riot:bg:fail:player')).toBeGreaterThan(86390);
await scheduler.clearFailureCounter('player');
expect(await client.exists('riot:bg:fail:player')).toBe(0);
});
});
beforeEach(async () => {
await client.flushDb();
});
afterAll(async () => {
if (client.isOpen) await client.quit();
});
function scheduledJob(priority = 'high', suffix = '1') {
return JSON.stringify({
kind: 'poll-tournament-result',
jobId: `job-${suffix}`,
matchId: 'match-1',
attempt: 1,
expandedToAllPuuids: false,
enqueuedAt: Date.now(),
priority,
});
}
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>((accept) => {
resolve = accept;
});
return { promise, resolve };
}
describe('Community Hash response cache', () => {
const cache = new CommunityStatsCacheService(redis);
it('does not share an old in-flight detail request after invalidation or overwrite the fresh response', async () => {
const service = new CommunityStatsService(
{} as never,
cache,
{} as never,
{} as never,
);
const communityId = new mongoose.Types.ObjectId();
const matchId = new mongoose.Types.ObjectId();
const dataset = (title: string) => ({
matches: [
{
matchId: matchId.toString(),
title,
scheduledAt: null,
status: 'finished',
gameCount: 0,
games: [],
teams: [],
seriesResults: [],
},
],
detailsByMatchId: new Map(),
});
const oldDataset = deferred<ReturnType<typeof dataset>>();
const oldStarted = deferred<void>();
const build = jest
.spyOn(
service as unknown as {
buildBaseDataset(): Promise<ReturnType<typeof dataset>>;
},
'buildBaseDataset',
)
.mockImplementationOnce(() => {
oldStarted.resolve();
return oldDataset.promise;
})
.mockResolvedValue(dataset('fresh'));
const oldRequest = service.getMatchDetail({ communityId, matchId });
await oldStarted.promise;
await cache.invalidateCommunity(communityId.toString());
try {
const fresh = await service.getMatchDetail({ communityId, matchId });
expect(fresh.match.title).toBe('fresh');
expect(build).toHaveBeenCalledTimes(2);
} finally {
oldDataset.resolve(dataset('old'));
await oldRequest;
}
expect((await oldRequest).match.title).toBe('old');
expect(
(await service.getMatchDetail({ communityId, matchId })).match.title,
).toBe('fresh');
expect(build).toHaveBeenCalledTimes(2);
});
it('preserves independent normal, detail and incomplete-detail TTLs', async () => {
const normal = new CommunityStatsCacheKey('community', 'matches', 'page-1');
const detail = new CommunityStatsCacheKey(
'community',
'matchDetail',
'match-1',
);
const incomplete = new CommunityStatsCacheKey(
'community',
'matchDetail',
'match-2',
);
for (const key of [normal, detail, incomplete])
expect(await cache.get(key)).toBeNull();
await cache.set(normal, { matches: [] });
await cache.setDetail(detail, { games: [1] });
await cache.set(incomplete, { games: [] }, 60);
for (const [key, ttl] of [
[normal, 300],
[detail, 86400],
[incomplete, 60],
] as const) {
const [remaining] = await client.sendCommand<number[]>([
'HTTL',
key.hashKey,
'FIELDS',
'1',
key.field,
]);
expect(remaining).toBeGreaterThan(ttl - 3);
expect(remaining).toBeLessThanOrEqual(ttl);
}
expect(await cache.get(detail)).toEqual({ games: [1] });
expect(await client.ttl(detail.hashKey)).toBe(-1);
await client.sendCommand([
'HPEXPIRE',
incomplete.hashKey,
'20',
'FIELDS',
'1',
incomplete.field,
]);
await delay(50);
expect(await cache.get(incomplete)).toBeNull();
expect(await cache.get(detail)).toEqual({ games: [1] });
});
it('invalidates all scopes for one community while preserving other communities', async () => {
const entries = (
[
'matches',
'leaderboard',
'teamInsights',
'matchDetail',
'userInternal',
] as const
).map((scope) => new CommunityStatsCacheKey('one', scope, 'field'));
const other = new CommunityStatsCacheKey('two', 'matches', 'field');
for (const key of [...entries, other]) {
await cache.get(key);
await cache.set(key, { fresh: true });
}
expect(await cache.invalidateCommunity('one')).toBe(5);
for (const key of entries) expect(await cache.get(key)).toBeNull();
expect(await cache.get(other)).toEqual({ fresh: true });
expect(await cache.invalidateCommunity('one')).toBe(0);
});
it('rejects stale fills and separates in-flight identities across invalidation', async () => {
const old = new CommunityStatsCacheKey('one', 'matches', 'page');
await cache.get(old);
const oldIdentity = old.identity;
await cache.invalidateCommunity('one');
const fresh = new CommunityStatsCacheKey('one', 'matches', 'page');
await cache.get(fresh);
expect(fresh.identity).not.toBe(oldIdentity);
await cache.set(fresh, { version: 2 });
await cache.set(old, { version: 1 });
expect(await cache.get(fresh)).toEqual({ version: 2 });
await cache.invalidateCommunity('one');
await cache.set(old, { version: 0 });
expect(await client.exists(old.hashKey)).toBe(0);
});
it('treats malformed data as a miss and refreshes only the written field TTL', async () => {
const key = new CommunityStatsCacheKey('one', 'matches', 'page');
await cache.get(key);
await client.hSet(key.hashKey, key.field, 'broken-json');
expect(await cache.get(key)).toBeNull();
await cache.set(key, { ok: true }, 1);
await cache.set(key, { ok: true }, 300);
const [ttl] = await client.sendCommand<number[]>([
'HTTL',
key.hashKey,
'FIELDS',
'1',
key.field,
]);
expect(ttl).toBeGreaterThan(297);
expect(await cache.get(key)).toEqual({ ok: true });
});
});
describe('Redis correctness regressions', () => {
it('rechecks the due timestamp after selection so rescheduled work is not promoted early', async () => {
const job = scheduledJob();
const selectedAt = Date.now();
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
score: selectedAt + 60000,
value: job,
});
expect(
await client.sendCommand([
'EVAL',
PROMOTE_SCHEDULED_RIOT_JOB_SCRIPT,
'2',
RIOT_JOB_SCHEDULED_ZSET,
RIOT_JOB_HIGH_STREAM,
job,
String(selectedAt),
]),
).toBe(0);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('discards malformed scheduled jobs and keeps the requested promotion batch limit', async () => {
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, [
{ score: 0, value: 'invalid-json' },
{ score: 0, value: JSON.stringify({ kind: 'unsupported' }) },
...Array.from({ length: 3 }, (_, i) => ({
score: 1,
value: scheduledJob('high', String(i)),
})),
]);
expect(await queue.promoteDueScheduledJobs(2)).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(3);
expect(await queue.promoteDueScheduledJobs(2)).toBe(2);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(2);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('keeps a scheduled job when destination append fails', async () => {
const job = scheduledJob();
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
score: Date.now() - 1,
value: job,
});
await client.set(RIOT_JOB_HIGH_STREAM, 'wrong-type');
await expect(queue.promoteDueScheduledJobs()).rejects.toThrow('WRONGTYPE');
expect(await client.zScore(RIOT_JOB_SCHEDULED_ZSET, job)).not.toBeNull();
await client.del(RIOT_JOB_HIGH_STREAM);
expect(await queue.promoteDueScheduledJobs()).toBe(1);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(1);
});
it('promotes each due job once under concurrent workers and leaves future jobs scheduled', async () => {
const jobs = Array.from({ length: 12 }, (_, i) =>
scheduledJob(i % 2 ? 'low' : 'high', String(i)),
);
await client.zAdd(
RIOT_JOB_SCHEDULED_ZSET,
jobs.map((value) => ({ score: Date.now() - 1, value })),
);
const future = scheduledJob('high', 'future');
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
score: Date.now() + 60_000,
value: future,
});
const counts = await Promise.all(
Array.from({ length: 4 }, () => queue.promoteDueScheduledJobs(5)),
);
await queue.promoteDueScheduledJobs();
expect(counts.reduce((sum, value) => sum + value, 0)).toBeLessThanOrEqual(
12,
);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(6);
expect(await client.xLen(RIOT_JOB_LOW_STREAM)).toBe(6);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('cleans only the requested lifecycle Hash payloads and queue references', async () => {
const keys = buildAuctionRedisKeys('match-1');
const oldKey = keys.durabilityPayloadKey('old');
const newKey = keys.durabilityPayloadKey('new');
for (const [key, lifecycle] of [
[oldKey, 'old-life'],
[newKey, 'new-life'],
]) {
await client.hSet(key, {
auctionLifecycleId: lifecycle,
payloadJson: JSON.stringify({ auctionLifecycleId: lifecycle }),
});
await client.zAdd(keys.durabilityRetryQueueKey, { score: 1, value: key });
await client.zAdd(keys.durabilityDeadLetterQueueKey, {
score: 1,
value: key,
});
}
const service = Object.assign(Object.create(AuctionService.prototype), {
auctionRedisService: auctionRedis,
}) as {
clearRedisDurabilityPayloadsForLifecycle(
keys: ReturnType<typeof buildAuctionRedisKeys>,
lifecycle: string,
): Promise<void>;
};
await service.clearRedisDurabilityPayloadsForLifecycle(keys, 'old-life');
expect(await client.exists(oldKey)).toBe(0);
expect(await client.exists(newKey)).toBe(1);
expect(await client.zRange(keys.durabilityRetryQueueKey, 0, -1)).toEqual([
newKey,
]);
expect(
await client.zRange(keys.durabilityDeadLetterQueueKey, 0, -1),
).toEqual([newKey]);
});
it('supports legacy JSON cleanup and preserves malformed or foreign payloads until an unconditional cleanup', async () => {
const keys = buildAuctionRedisKeys('legacy');
const legacy = keys.durabilityPayloadKey('old');
const foreign = keys.durabilityPayloadKey('foreign');
const malformed = keys.durabilityPayloadKey('malformed');
await client.set(
legacy,
JSON.stringify({ auctionLifecycleId: 'old-life' }),
);
await client.set(
foreign,
JSON.stringify({ auctionLifecycleId: 'new-life' }),
);
await client.set(malformed, 'broken-json');
await client.zAdd(
keys.durabilityRetryQueueKey,
[legacy, foreign, malformed].map((value) => ({ score: 1, value })),
);
const service = Object.assign(Object.create(AuctionService.prototype), {
auctionRedisService: auctionRedis,
}) as {
clearRedisDurabilityPayloadsForLifecycle(
keys: ReturnType<typeof buildAuctionRedisKeys>,
lifecycle?: string,
): Promise<void>;
};
await service.clearRedisDurabilityPayloadsForLifecycle(keys, 'old-life');
expect(await client.exists(legacy)).toBe(0);
expect(await client.exists([foreign, malformed])).toBe(2);
await service.clearRedisDurabilityPayloadsForLifecycle(keys);
expect(await client.exists([foreign, malformed])).toBe(0);
expect(await client.zCard(keys.durabilityRetryQueueKey)).toBe(0);
});
it('preserves the 500-request cap, replay window, and fingerprint conflict checks', async () => {
const keys = buildAuctionRedisKeys('retention');
const reserve = async (id: string, revision = 1) => {
const invocation = buildAuctionReserveActionRequestScriptInvocation(
keys,
{
auctionActionRequestId: id,
auctionActionType: 'next',
auctionStateTransitionId: `transition-${id}`,
clientObservedAuctionLifecycleId: 'life',
clientObservedPhase: 'sold',
clientObservedRevision: revision,
leaseDurationMS: 15000,
maximumStoredAuctionActionRequestResultCount: 500,
minimumAuctionActionRequestResultReplayWindowMS: 300000,
processingOwnerToken: 'owner',
},
);
return auctionRedis.evalScript<string[]>(
invocation.script,
invocation.keys,
invocation.values,
);
};
for (let i = 0; i < 500; i++)
expect((await reserve(String(i)))[0]).toBe('RESERVED');
expect((await reserve('0', 2)).slice(0, 2)).toEqual([
'REJECTED',
'DUPLICATE_ACTION_REQUEST_CONFLICT',
]);
expect((await reserve('501')).slice(0, 2)).toEqual([
'REJECTED',
'ACTION_REQUEST_RESULT_LIMIT',
]);
expect(await client.lLen(keys.actionRequestResultOrderKey)).toBe(500);
expect((await reserve('0'))[0]).toBe('RESERVED');
expect(await client.lLen(keys.actionRequestResultOrderKey)).toBe(500);
await client.hSet(
keys.actionRequestResultsKey,
'life:0:createdAtTimestampMS',
String(Date.now() - 300001),
);
expect((await reserve('501'))[0]).toBe('RESERVED');
expect(await client.lLen(keys.actionRequestResultOrderKey)).toBe(500);
expect(
await client.hExists(keys.actionRequestResultsKey, 'life:0:fingerprint'),
).toBe(0);
});
it('counts one order entry per reserved and applied action and still replays it', async () => {
const keys = buildAuctionRedisKeys('match-1');
const commands = new AuctionRedisCommandService(auctionRedis);
await commands.hydrateState('match-1', {
matchId: 'match-1',
auctionHistorySequence: 0,
auctionLifecycleId: 'life',
phase: 'sold',
revision: 0,
maximumBidRoundDurationSeconds: 30,
structureStateJson: '{}',
});
const reservation = buildAuctionReserveActionRequestScriptInvocation(keys, {
auctionActionRequestId: 'request',
auctionActionType: 'next',
auctionStateTransitionId: 'transition',
clientObservedAuctionLifecycleId: 'life',
clientObservedPhase: 'sold',
clientObservedRevision: 1,
leaseDurationMS: 15000,
maximumStoredAuctionActionRequestResultCount: 500,
minimumAuctionActionRequestResultReplayWindowMS: 300000,
processingOwnerToken: 'owner',
});
await auctionRedis.evalScript(
reservation.script,
reservation.keys,
reservation.values,
);
const apply = buildAuctionApplyTransitionScriptInvocation(keys, {
actionRequest: {
scopeKey: 'life:request',
processingOwnerToken: 'owner',
observedProcessingLeaseVersion: 1,
},
afterAuctionStructureStateJson: '{}',
auctionHistorySequence: 1,
auctionLifecycleId: 'life',
auctionStateTransitionId: 'transition',
broadcastStateRevision: 2,
deadline: { mode: 'none' },
durabilityPayloadJson: '{}',
expectedPhase: 'sold',
expectedRevision: 1,
liveStatePatch: { phase: 'sold' },
matchId: 'match-1',
shouldBroadcastState: false,
});
const result = await auctionRedis.evalScript<string[]>(
apply.script,
apply.keys,
apply.values,
);
expect(result[0]).toBe('OK');
expect(
await client.lRange(keys.actionRequestResultOrderKey, 0, -1),
).toEqual(['life:request']);
const replay = await auctionRedis.evalScript<string[]>(
reservation.script,
reservation.keys,
reservation.values,
);
expect(replay.slice(0, 2)).toEqual(['REPLAY', 'applied']);
});
});
+120
View File
@@ -0,0 +1,120 @@
import { spawn } from 'node:child_process';
import { createRequire } from 'node:module';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, relative, resolve } from 'node:path';
import { createServer } from 'node:net';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { createClient } from 'redis';
const require = createRequire(import.meta.url);
const listener = createServer();
await new Promise((accept, reject) => {
listener.once('error', reject);
listener.listen(0, '127.0.0.1', accept);
});
const port = listener.address().port;
await new Promise((accept) => listener.close(accept));
const directory = await mkdtemp(join(tmpdir(), 'lolympus-redis-test-'));
const password = randomUUID();
const redis = spawn(
process.env.REDIS_TEST_SERVER || 'redis-server',
[
'--bind',
'127.0.0.1',
'--port',
String(port),
'--protected-mode',
'yes',
'--requirepass',
password,
'--save',
'',
'--appendonly',
'no',
'--dir',
directory.replaceAll('\\', '/'),
],
{ windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] },
);
let startupError;
const redisClosed = new Promise((accept) => redis.once('close', accept));
let output = '';
redis.on('error', (error) => {
startupError = error;
});
redis.stdout.on('data', (data) => {
output += data;
});
redis.stderr.on('data', (data) => {
output += data;
});
const url = `redis://:${password}@127.0.0.1:${port}`;
const client = createClient({
url,
socket: { reconnectStrategy: false, connectTimeout: 500 },
});
client.on('error', () => {});
try {
let connected = false;
for (let attempt = 0; attempt < 100; attempt++) {
if (startupError) throw startupError;
if (redis.exitCode !== null) throw new Error(`Redis exited: ${output}`);
try {
await client.connect();
connected = true;
break;
} catch {
await delay(100);
}
}
if (!connected) throw new Error(`Redis did not start: ${output}`);
const version = (await client.info('server')).match(
/redis_version:([^\r\n]+)/,
)?.[1];
if (!version?.startsWith('8.10.'))
throw new Error(`Expected Redis 8.10.x, got ${version}`);
console.log(
`Integration tests using isolated Redis ${version} on loopback port ${port}`,
);
const jest = spawn(
process.execPath,
[
require.resolve('jest/bin/jest'),
'--config',
'test/jest-redis.json',
'--runInBand',
...process.argv.slice(2),
],
{
cwd: resolve(import.meta.dirname, '..'),
windowsHide: true,
stdio: 'inherit',
env: {
...process.env,
NODE_ENV: 'test',
REDIS_TEST_URL: url,
REDIS_TEST_ISOLATED: '1',
},
},
);
process.exitCode = await new Promise((accept, reject) => {
jest.once('error', reject);
jest.once('exit', (code) => accept(code ?? 1));
});
} finally {
if (client.isOpen) {
await client.sendCommand(['SHUTDOWN', 'NOSAVE']).catch(() => {});
if (client.isOpen) client.destroy();
}
if (redis.pid && redis.exitCode === null) redis.kill();
await redisClosed;
const cleanupTarget = relative(resolve(tmpdir()), resolve(directory));
if (!/^lolympus-redis-test-[^\\/]+$/.test(cleanupTarget)) {
throw new Error(
'Refusing to clean up a directory outside the isolated test workspace.',
);
}
await rm(directory, { recursive: true, force: true });
}
@@ -0,0 +1,284 @@
import { createClient } from 'redis';
import { RedisService } from '../src/modules/global/redis/redis.service';
import { RiotJobQueueService } from '../src/modules/riot-job-queue/riot-job-queue.service';
import {
RIOT_JOB_GROUP,
RIOT_JOB_HIGH_STREAM,
RIOT_JOB_LOW_STREAM,
RIOT_JOB_SCHEDULED_ZSET,
} from '../src/modules/riot-job-queue/riot-job-queue.types';
import type { PollTournamentResultJob } from '../src/modules/tournament-result/tournament-result.types';
const client = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const redis = new RedisService(client);
const queue = new RiotJobQueueService({} as never, redis);
beforeAll(async () => {
const url = process.env.REDIS_TEST_URL;
if (
process.env.REDIS_TEST_ISOLATED !== '1' ||
!url ||
!['127.0.0.1', 'localhost', '[::1]'].includes(new URL(url).hostname)
) {
throw new Error('Run with the isolated loopback Redis integration runner.');
}
client.on('error', () => {});
await client.connect();
});
beforeEach(async () => {
await client.flushDb();
});
afterAll(async () => {
if (client.isOpen) await client.quit();
});
describe('Tournament result retry delivery on Redis', () => {
it('replaces a pending poll with one durable retry and ignores repeated transfer requests', async () => {
const job = pollJob();
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
const before = Date.now();
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
const after = Date.now();
const scheduled = await client.zRangeWithScores(
RIOT_JOB_SCHEDULED_ZSET,
0,
-1,
);
expect(scheduled).toHaveLength(1);
expect(JSON.parse(scheduled[0].value)).toEqual({ ...job, retryCount: 1 });
expect(scheduled[0].score).toBeGreaterThanOrEqual(before + 30_000);
expect(scheduled[0].score).toBeLessThanOrEqual(after + 30_000);
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(0);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(0);
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
expect(
await client.zRangeWithScores(RIOT_JOB_SCHEDULED_ZSET, 0, -1),
).toEqual(scheduled);
});
it.each([
[1, 60_000],
[2, 120_000],
[3, 240_000],
[4, 300_000],
[20, 300_000],
])(
'backs off retry count %i by %i ms without changing the poll attempt',
async (retryCount, delayMs) => {
const job = { ...pollJob(), retryCount };
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
const before = Date.now();
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
const after = Date.now();
const scheduled = await client.zRangeWithScores(
RIOT_JOB_SCHEDULED_ZSET,
0,
-1,
);
expect(scheduled).toHaveLength(1);
expect(JSON.parse(scheduled[0].value)).toEqual({
...job,
retryCount: retryCount + 1,
});
expect(scheduled[0].score).toBeGreaterThanOrEqual(before + delayMs);
expect(scheduled[0].score).toBeLessThanOrEqual(after + delayMs);
},
);
it('keeps the original pending when retry persistence fails and can recover afterward', async () => {
const job = pollJob();
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
await client.set(RIOT_JOB_SCHEDULED_ZSET, 'wrong-type');
await expect(
queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
}),
).rejects.toThrow();
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(1);
expect(await client.xRange(RIOT_JOB_HIGH_STREAM, entryId, entryId)).toEqual(
[{ id: entryId, message: { job: JSON.stringify(job) } }],
);
await client.del(RIOT_JOB_SCHEDULED_ZSET);
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it('retains the original for another consumer group until that group acknowledges it', async () => {
const job = pollJob();
const stream = RIOT_JOB_HIGH_STREAM;
await redis.createConsumerGroup(stream, 'audit', '0');
const entryId = await pendingPoll(stream, job);
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0);
expect(await client.xLen(stream)).toBe(1);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
const auditEntry = await redis.readGroup(stream, 'audit', 'auditor', '>', {
count: 1,
});
expect(auditEntry?.[0].messages[0].id).toBe(entryId);
await redis.acknowledgeAndDeleteStream(stream, 'audit', entryId);
expect(await client.xLen(stream)).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
it.each([
['high', RIOT_JOB_HIGH_STREAM],
['low', RIOT_JOB_LOW_STREAM],
] as const)(
'promotes a due %s retry to its original priority and allows successful consumption',
async (priority, stream) => {
const job = { ...pollJob(), priority };
const entryId = await pendingPoll(stream, job);
await queue.rescheduleTournamentResultPoll({ stream, entryId, job });
const [scheduled] = await client.zRangeWithScores(
RIOT_JOB_SCHEDULED_ZSET,
0,
-1,
);
await client.zAdd(RIOT_JOB_SCHEDULED_ZSET, {
value: scheduled.value,
score: Date.now() - 1,
});
expect(await queue.promoteDueScheduledJobs()).toBe(1);
const delivery = await redis.readGroup(
stream,
RIOT_JOB_GROUP,
'recovered-worker',
'>',
{ count: 1 },
);
const retriedEntry = delivery?.[0].messages[0];
expect(retriedEntry).toBeDefined();
expect(JSON.parse(retriedEntry!.message.job)).toEqual({
...job,
retryCount: 1,
});
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0);
const otherStream =
priority === 'high' ? RIOT_JOB_LOW_STREAM : RIOT_JOB_HIGH_STREAM;
expect(await client.xLen(otherStream)).toBe(0);
await redis.acknowledgeAndDeleteStream(
stream,
RIOT_JOB_GROUP,
retriedEntry!.id,
);
expect((await client.xPending(stream, RIOT_JOB_GROUP)).pending).toBe(0);
expect(await client.xLen(stream)).toBe(0);
},
);
it('retains pending work during a disconnected retry client and recovers with a healthy connection', async () => {
const job = pollJob();
const entryId = await pendingPoll(RIOT_JOB_HIGH_STREAM, job);
const offlineClient = createClient({
url: process.env.REDIS_TEST_URL,
socket: { reconnectStrategy: false },
});
const offlineQueue = new RiotJobQueueService(
{} as never,
new RedisService(offlineClient),
);
await expect(
offlineQueue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
}),
).rejects.toThrow();
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(1);
expect(await client.xLen(RIOT_JOB_HIGH_STREAM)).toBe(1);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(0);
await queue.rescheduleTournamentResultPoll({
stream: RIOT_JOB_HIGH_STREAM,
entryId,
job,
});
expect(
(await client.xPending(RIOT_JOB_HIGH_STREAM, RIOT_JOB_GROUP)).pending,
).toBe(0);
expect(await client.zCard(RIOT_JOB_SCHEDULED_ZSET)).toBe(1);
});
});
function pollJob(): PollTournamentResultJob {
return {
kind: 'poll-tournament-result',
jobId: 'original-result-job',
matchId: '6aeecacdffed1eb545b8c0f3',
attempt: 7,
expandedToAllPuuids: true,
enqueuedAt: 1_794_053_383_203,
priority: 'high',
};
}
async function pendingPoll(
stream: string,
job: PollTournamentResultJob,
): Promise<string> {
await redis.createConsumerGroup(stream, RIOT_JOB_GROUP, '0');
const entryId = await redis.appendStream(stream, {
job: JSON.stringify(job),
});
const delivery = await redis.readGroup(
stream,
RIOT_JOB_GROUP,
'original-worker',
'>',
{ count: 1 },
);
expect(delivery?.[0].messages[0].id).toBe(entryId);
return entryId;
}