Compare commits

...
2 Commits
Author SHA1 Message Date
dahyeon de8bc87940 fix: unify riot refresh rate gating 2026-06-07 11:11:12 -07:00
dahyeon 25f3ccad57 fix: delete acknowledged riot job stream entries 2026-06-07 10:13:29 -07:00
9 changed files with 443 additions and 142 deletions

No files matched your search

@@ -1,9 +1,4 @@
import { import { Inject, Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
Inject,
Injectable,
Logger,
OnModuleDestroy,
} from '@nestjs/common';
import type { createClient } from 'redis'; import type { createClient } from 'redis';
import { GARNET_CLIENT } from './garnet.constants'; import { GARNET_CLIENT } from './garnet.constants';
@@ -203,6 +198,13 @@ export class GarnetService implements OnModuleDestroy {
return this.client.xAck(key, group, ids); return this.client.xAck(key, group, ids);
} }
public async deleteStreamEntries(
key: string,
ids: string | string[],
): Promise<number> {
return this.client.xDel(key, ids);
}
public async streamLength(key: string): Promise<number> { public async streamLength(key: string): Promise<number> {
return this.client.xLen(key); return this.client.xLen(key);
} }
@@ -41,7 +41,7 @@ function makeScheduler() {
raw: jest.fn().mockReturnValue(rawClient), raw: jest.fn().mockReturnValue(rawClient),
} as unknown as jest.Mocked<GarnetService> & { raw: jest.Mock }; } as unknown as jest.Mocked<GarnetService> & { raw: jest.Mock };
const rateLimiter = { const rateLimiter = {
acquireBackground: jest.fn().mockResolvedValue(undefined), isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
} as unknown as jest.Mocked<RiotRateLimiterService>; } as unknown as jest.Mocked<RiotRateLimiterService>;
const enrichmentService = { const enrichmentService = {
markRefreshRequested: jest.fn().mockResolvedValue(undefined), markRefreshRequested: jest.fn().mockResolvedValue(undefined),
@@ -84,9 +84,29 @@ describe('BackgroundRefreshScheduler', () => {
scheduler.onModuleDestroy(); scheduler.onModuleDestroy();
}); });
it('does not query stale accounts outside the KST window', async () => { it('uses the dawn refresh gate ratio inside the KST background window', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { scheduler, rateLimiter } = makeScheduler();
await scheduler.tick();
expect(rateLimiter.isRefreshWindowOpen).toHaveBeenCalledWith(0.8);
});
it('uses the opportunistic refresh gate ratio outside the KST background window', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z')); jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z'));
const { scheduler, staleAccountRepository } = makeScheduler(); const { scheduler, rateLimiter, staleAccountRepository } = makeScheduler();
await scheduler.tick();
expect(rateLimiter.isRefreshWindowOpen).toHaveBeenCalledWith(0.5);
expect(staleAccountRepository.findStaleAccounts).toHaveBeenCalledTimes(1);
});
it('does not query stale accounts when the refresh gate is closed', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { scheduler, rateLimiter, staleAccountRepository } = makeScheduler();
rateLimiter.isRefreshWindowOpen.mockResolvedValue(false);
await scheduler.tick(); await scheduler.tick();
@@ -103,21 +123,15 @@ describe('BackgroundRefreshScheduler', () => {
expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled(); expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled();
}); });
it('marks and enriches each stale candidate with background rate tokens', async () => { it('marks and enriches each stale candidate without background precharge', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { const { scheduler, enrichmentService, staleAccountRepository } =
scheduler, makeScheduler();
rateLimiter,
enrichmentService,
staleAccountRepository,
} = makeScheduler();
const candidates = [candidate('puuid-1'), candidate('puuid-2')]; const candidates = [candidate('puuid-1'), candidate('puuid-2')];
staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates); staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates);
await scheduler.tick(); await scheduler.tick();
expect(rateLimiter.acquireBackground).toHaveBeenCalledTimes(2);
expect(rateLimiter.acquireBackground).toHaveBeenCalledWith(8);
expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2); expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2);
expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith( expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith(
1, 1,
@@ -131,6 +145,28 @@ describe('BackgroundRefreshScheduler', () => {
}); });
}); });
it('stops the candidate loop when the refresh gate closes mid-tick', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const {
scheduler,
rateLimiter,
enrichmentService,
staleAccountRepository,
} = makeScheduler();
staleAccountRepository.findStaleAccounts.mockResolvedValue([
candidate('puuid-1'),
candidate('puuid-2'),
]);
rateLimiter.isRefreshWindowOpen
.mockResolvedValueOnce(true)
.mockResolvedValueOnce(true)
.mockResolvedValueOnce(false);
await scheduler.tick();
expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(1);
});
it('continues after a candidate throws and records a failure counter', async () => { it('continues after a candidate throws and records a failure counter', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { scheduler, rawClient, enrichmentService, staleAccountRepository } = const { scheduler, rawClient, enrichmentService, staleAccountRepository } =
@@ -157,13 +193,8 @@ describe('BackgroundRefreshScheduler', () => {
it('skips candidates already in failure backoff', async () => { it('skips candidates already in failure backoff', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { const { scheduler, garnet, enrichmentService, staleAccountRepository } =
scheduler, makeScheduler();
garnet,
rateLimiter,
enrichmentService,
staleAccountRepository,
} = makeScheduler();
staleAccountRepository.findStaleAccounts.mockResolvedValue([ staleAccountRepository.findStaleAccounts.mockResolvedValue([
candidate('skip-me'), candidate('skip-me'),
]); ]);
@@ -174,7 +205,6 @@ describe('BackgroundRefreshScheduler', () => {
await scheduler.tick(); await scheduler.tick();
expect(rateLimiter.acquireBackground).not.toHaveBeenCalled();
expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled(); expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled();
expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled(); expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled();
}); });
@@ -19,11 +19,17 @@ const LEADER_TTL_SECONDS = 90;
const LEADER_RENEW_INTERVAL_MS = 30_000; const LEADER_RENEW_INTERVAL_MS = 30_000;
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000; const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
const PER_TICK_CANDIDATE_LIMIT = 20; const PER_TICK_CANDIDATE_LIMIT = 20;
const ENRICH_COST_TOKENS = 8; const DAWN_USAGE_RATIO = 0.8;
const OPPORTUNISTIC_USAGE_RATIO = 0.5;
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail'; const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
const FAILURE_BACKOFF_THRESHOLD = 3; const FAILURE_BACKOFF_THRESHOLD = 3;
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60; const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
interface RefreshGatePolicy {
mode: 'dawn' | 'opportunistic';
usageRatio: number;
}
@Injectable() @Injectable()
export class BackgroundRefreshScheduler export class BackgroundRefreshScheduler
implements OnModuleInit, OnModuleDestroy implements OnModuleInit, OnModuleDestroy
@@ -75,7 +81,11 @@ export class BackgroundRefreshScheduler
} }
private async tick(): Promise<void> { private async tick(): Promise<void> {
if (!isInKstBackgroundWindow()) return; const initialGate = this.currentRefreshGatePolicy();
if (!(await this.rateLimiter.isRefreshWindowOpen(initialGate.usageRatio))) {
return;
}
const isLeader = await this.tryBecomeLeader(); const isLeader = await this.tryBecomeLeader();
if (!isLeader) return; if (!isLeader) return;
@@ -89,15 +99,20 @@ export class BackgroundRefreshScheduler
let processed = 0; let processed = 0;
let failed = 0; let failed = 0;
let skipped = 0; let skipped = 0;
let currentGate = initialGate;
for (const candidate of candidates) { for (const candidate of candidates) {
if (!isInKstBackgroundWindow()) break; currentGate = this.currentRefreshGatePolicy();
if (
!(await this.rateLimiter.isRefreshWindowOpen(currentGate.usageRatio))
) {
break;
}
if (await this.isInFailureBackoff(candidate.puuid)) { if (await this.isInFailureBackoff(candidate.puuid)) {
skipped += 1; skipped += 1;
continue; continue;
} }
await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS);
await this.enrichmentService.markRefreshRequested( await this.enrichmentService.markRefreshRequested(
candidate.userId, candidate.userId,
candidate.puuid, candidate.puuid,
@@ -130,10 +145,18 @@ export class BackgroundRefreshScheduler
} }
this.logger.debug( this.logger.debug(
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`, `Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length} mode=${currentGate.mode} ratio=${currentGate.usageRatio}`,
); );
} }
private currentRefreshGatePolicy(): RefreshGatePolicy {
const isDawn = isInKstBackgroundWindow();
return {
mode: isDawn ? 'dawn' : 'opportunistic',
usageRatio: isDawn ? DAWN_USAGE_RATIO : OPPORTUNISTIC_USAGE_RATIO,
};
}
private async tryBecomeLeader(): Promise<boolean> { private async tryBecomeLeader(): Promise<boolean> {
const current = await this.garnet.getString(LEADER_KEY); const current = await this.garnet.getString(LEADER_KEY);
if (current === this.consumer) { if (current === this.consumer) {
@@ -17,6 +17,7 @@ function execResult<T>(value: T) {
function makeWorker() { function makeWorker() {
const garnetService = { const garnetService = {
acknowledgeStream: jest.fn().mockResolvedValue(undefined), acknowledgeStream: jest.fn().mockResolvedValue(undefined),
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<GarnetService>; } as unknown as jest.Mocked<GarnetService>;
const jobQueueService = { const jobQueueService = {
markProcessing: jest.fn().mockResolvedValue(undefined), markProcessing: jest.fn().mockResolvedValue(undefined),
@@ -74,6 +75,32 @@ function makeWorker() {
} }
describe('RiotJobWorkerService', () => { describe('RiotJobWorkerService', () => {
it('acknowledges and deletes invalid stream entries', async () => {
const { worker, garnetService } = makeWorker();
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: 'bad-1',
message: { job: 'not-json' },
});
expect(garnetService.acknowledgeStream).toHaveBeenCalledWith(
'riot:queue:high',
RIOT_JOB_GROUP,
'bad-1',
);
expect(garnetService.deleteStreamEntries).toHaveBeenCalledWith(
'riot:queue:high',
'bad-1',
);
});
it('refreshes league entries while processing user history refresh jobs', async () => { it('refreshes league entries while processing user history refresh jobs', async () => {
const { worker, enrichmentService, recentMatchesService, garnetService } = const { worker, enrichmentService, recentMatchesService, garnetService } =
makeWorker(); makeWorker();
@@ -91,12 +118,14 @@ describe('RiotJobWorkerService', () => {
priority: 'high', priority: 'high',
}; };
await (worker as unknown as { await (
processEntry: ( worker as unknown as {
stream: string, processEntry: (
entry: { id: string; message: Record<string, string> }, stream: string,
) => Promise<void>; entry: { id: string; message: Record<string, string> },
}).processEntry('riot:queue:high', { ) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: '1-0', id: '1-0',
message: { job: JSON.stringify(job) }, message: { job: JSON.stringify(job) },
}); });
@@ -112,5 +141,14 @@ describe('RiotJobWorkerService', () => {
RIOT_JOB_GROUP, RIOT_JOB_GROUP,
'1-0', '1-0',
); );
expect(garnetService.deleteStreamEntries).toHaveBeenCalledWith(
'riot:queue:high',
'1-0',
);
expect(
garnetService.acknowledgeStream.mock.invocationCallOrder[0],
).toBeLessThan(
garnetService.deleteStreamEntries.mock.invocationCallOrder[0],
);
}); });
}); });
@@ -79,10 +79,13 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
} }
} }
private async processEntry(stream: string, entry: StreamEntry): Promise<void> { private async processEntry(
stream: string,
entry: StreamEntry,
): Promise<void> {
const job = this.parseJob(entry); const job = this.parseJob(entry);
if (!job) { if (!job) {
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, entry.id); await this.acknowledgeAndDelete(stream, entry.id);
return; return;
} }
@@ -134,7 +137,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
}`, }`,
); );
} finally { } finally {
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, entry.id); await this.acknowledgeAndDelete(stream, entry.id);
}
}
private async acknowledgeAndDelete(
stream: string,
id: string,
): Promise<void> {
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, id);
await this.deleteAcknowledgedEntry(stream, id);
}
private async deleteAcknowledgedEntry(
stream: string,
id: string,
): Promise<void> {
try {
await this.garnetService.deleteStreamEntries(stream, id);
} catch (error) {
this.logger.warn(
`Failed to delete acknowledged Riot job stream entry ${stream} ${id}: ${
error instanceof Error ? error.message : error
}`,
);
} }
} }
@@ -192,9 +218,7 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
return job.kind === 'poll-tournament-result'; return job.kind === 'poll-tournament-result';
} }
private isLegacyImportJob( private isLegacyImportJob(job: RiotJobPayload): job is LegacyImportMemberJob {
job: RiotJobPayload,
): job is LegacyImportMemberJob {
return job.kind === 'legacy-import-member'; return job.kind === 'legacy-import-member';
} }
@@ -219,7 +243,8 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
try { try {
await this.garnetService.createConsumerGroup(stream, RIOT_JOB_GROUP); await this.garnetService.createConsumerGroup(stream, RIOT_JOB_GROUP);
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : String(error); const message =
error instanceof Error ? error.message : String(error);
if (!message.includes('BUSYGROUP')) throw error; if (!message.includes('BUSYGROUP')) throw error;
} }
}), }),
@@ -3,63 +3,156 @@ import { RiotRateLimiterService } from './riot-rate-limiter.service';
function makeService(sendCommand = jest.fn().mockResolvedValue(0)) { function makeService(sendCommand = jest.fn().mockResolvedValue(0)) {
const garnet = { const garnet = {
getString: jest.fn().mockResolvedValue(null),
raw: jest.fn().mockReturnValue({ sendCommand }), raw: jest.fn().mockReturnValue({ sendCommand }),
} as unknown as GarnetService; setString: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<GarnetService>;
return { return {
service: new RiotRateLimiterService(garnet), service: new RiotRateLimiterService(garnet),
garnet,
sendCommand, sendCommand,
}; };
} }
describe('RiotRateLimiterService background limiter', () => { function observedState(params: {
count120: number;
limit120?: number;
observedAt: number;
windowStartAt: number;
}) {
return JSON.stringify({
count120: params.count120,
limit120: params.limit120 ?? 100,
observedAt: params.observedAt,
windowStartAt: params.windowStartAt,
});
}
describe('RiotRateLimiterService', () => {
afterEach(() => { afterEach(() => {
jest.restoreAllMocks(); jest.restoreAllMocks();
}); });
it('uses the background key namespace and passes cost to the Lua script', async () => { it('uses global app rate keys and passes cost to the Lua script', async () => {
jest.spyOn(Date, 'now').mockReturnValue(1_000); jest.spyOn(Date, 'now').mockReturnValue(1_000);
const { service, sendCommand } = makeService(); const { service, sendCommand } = makeService();
await service.acquireBackground(8); await service.acquire('asia', 3);
expect(sendCommand).toHaveBeenCalledTimes(1); expect(sendCommand).toHaveBeenCalledTimes(1);
const command = sendCommand.mock.calls[0][0] as string[]; const command = sendCommand.mock.calls[0][0] as string[];
expect(command[0]).toBe('EVAL'); expect(command[0]).toBe('EVAL');
expect(command[2]).toBe('2'); expect(command[2]).toBe('2');
expect(command[3]).toBe('riot:bg:rl:1000'); expect(command[3]).toBe('riot:rl:app:1000');
expect(command[4]).toBe('riot:bg:rl:120000'); expect(command[4]).toBe('riot:rl:app:120000');
expect(command[5]).toBe('1000'); expect(command[5]).toBe('1000');
expect(command[7]).toBe('8'); expect(command[7]).toBe('3');
}); });
it('reads cleaned background usage for both windows', async () => { it('records the 120 second app rate observation from Riot headers', async () => {
jest.spyOn(Date, 'now').mockReturnValue(200_000); jest.spyOn(Date, 'now').mockReturnValue(1_000_000);
const sendCommand = jest const { service, garnet } = makeService();
.fn()
.mockImplementation(async (command: string[]) => {
if (command[0] === 'ZREMRANGEBYSCORE') return 0;
if (command[1] === 'riot:bg:rl:1000') return 3;
if (command[1] === 'riot:bg:rl:120000') return 24;
return 0;
});
const { service } = makeService(sendCommand);
await expect(service.getBackgroundUsage()).resolves.toEqual({ await service.recordAppRateLimitObservation('47:120,3:1', '100:120,20:1');
used1s: 3,
used2m: 24, expect(garnet.setString).toHaveBeenCalledTimes(1);
const [key, raw, ttl] = garnet.setString.mock.calls[0];
expect(key).toBe('riot:app:rl:observed');
expect(ttl).toBe(300);
expect(JSON.parse(raw)).toEqual({
count120: 47,
limit120: 100,
observedAt: 1_000_000,
windowStartAt: 1_000_000,
});
});
it('resets the inferred window when Riot count decreases or observation is stale', async () => {
const { service, garnet } = makeService();
jest.spyOn(Date, 'now').mockReturnValue(1_060_000);
garnet.getString.mockResolvedValueOnce(
observedState({
count120: 80,
observedAt: 1_050_000,
windowStartAt: 1_000_000,
}),
);
await service.recordAppRateLimitObservation('2:120', null);
expect(JSON.parse(garnet.setString.mock.calls[0][1])).toMatchObject({
count120: 2,
windowStartAt: 1_060_000,
}); });
expect(sendCommand).toHaveBeenCalledWith([ jest.spyOn(Date, 'now').mockReturnValue(1_240_000);
'ZREMRANGEBYSCORE', garnet.getString.mockResolvedValueOnce(
'riot:bg:rl:1000', observedState({
'-inf', count120: 20,
'199000', observedAt: 1_120_000,
]); windowStartAt: 1_000_000,
expect(sendCommand).toHaveBeenCalledWith([ }),
'ZREMRANGEBYSCORE', );
'riot:bg:rl:120000',
'-inf', await service.recordAppRateLimitObservation('25:120', null);
'80000',
]); expect(JSON.parse(garnet.setString.mock.calls[1][1])).toMatchObject({
count120: 25,
windowStartAt: 1_240_000,
});
});
it('opens the refresh gate when there is no active observation', async () => {
const { service, garnet } = makeService();
await expect(service.isRefreshWindowOpen(0.5)).resolves.toBe(true);
garnet.getString.mockResolvedValueOnce(
observedState({
count120: 90,
observedAt: 1_000_000,
windowStartAt: 900_000,
}),
);
jest.spyOn(Date, 'now').mockReturnValue(1_120_000);
await expect(service.isRefreshWindowOpen(0.5)).resolves.toBe(true);
});
it.each([
{ ratio: 0.5, count120: 49, expected: true },
{ ratio: 0.5, count120: 50, expected: false },
{ ratio: 0.8, count120: 79, expected: true },
{ ratio: 0.8, count120: 80, expected: false },
])(
'checks tail headroom for refresh gate ratio=$ratio count=$count120',
async ({ ratio, count120, expected }) => {
jest.spyOn(Date, 'now').mockReturnValue(1_070_000);
const { service, garnet } = makeService();
garnet.getString.mockResolvedValue(
observedState({
count120,
observedAt: 1_069_000,
windowStartAt: 1_000_000,
}),
);
await expect(service.isRefreshWindowOpen(ratio)).resolves.toBe(expected);
},
);
it('keeps the refresh gate closed before the 120 second window tail', async () => {
jest.spyOn(Date, 'now').mockReturnValue(1_050_000);
const { service, garnet } = makeService();
garnet.getString.mockResolvedValue(
observedState({
count120: 10,
observedAt: 1_049_000,
windowStartAt: 1_000_000,
}),
);
await expect(service.isRefreshWindowOpen(0.8)).resolves.toBe(false);
}); });
}); });
@@ -7,11 +7,19 @@ const APP_LIMITS = [
{ windowMs: 120_000, max: 90 }, { windowMs: 120_000, max: 90 },
] as const; ] as const;
const BACKGROUND_LIMITS = [ const APP_KEY_PREFIX = 'riot:rl:app';
{ windowMs: 1_000, max: 6 }, const APP_WINDOW_MS = 120_000;
{ windowMs: 120_000, max: 36 }, const TAIL_MS = 60_000;
] as const; const APP_OBSERVED_KEY = 'riot:app:rl:observed';
const BACKGROUND_KEY_PREFIX = 'riot:bg:rl'; const APP_OBSERVED_TTL_SECONDS = 300;
const DEFAULT_APP_LIMIT_120 = 100;
interface AppRateObservation {
count120: number;
limit120: number;
observedAt: number;
windowStartAt: number;
}
const ACQUIRE_SCRIPT = ` const ACQUIRE_SCRIPT = `
local now = tonumber(ARGV[1]) local now = tonumber(ARGV[1])
@@ -81,85 +89,78 @@ export class RiotRateLimiterService {
public async acquire( public async acquire(
route: RiotRegionalRoute | RiotPlatformRoute, route: RiotRegionalRoute | RiotPlatformRoute,
cost = 1,
): Promise<void> { ): Promise<void> {
for (;;) {
const now = Date.now();
const waitMs = await this.checkAndAdd(route, now, 1);
if (waitMs <= 0) return;
if (waitMs >= 100) {
this.logger.warn(
`Riot API throttle waiting ${waitMs}ms for route=${route}.`,
);
}
await sleep(waitMs);
}
}
public async acquireBackground(cost = 1): Promise<void> {
const normalizedCost = normalizeCost(cost); const normalizedCost = normalizeCost(cost);
for (;;) { for (;;) {
const now = Date.now(); const now = Date.now();
const waitMs = await this.checkAndAddBackground(now, normalizedCost); const waitMs = await this.checkAndAdd(now, normalizedCost);
if (waitMs <= 0) return; if (waitMs <= 0) return;
if (waitMs >= 100) { if (waitMs >= 100) {
this.logger.warn( this.logger.warn(
`Background Riot throttle waiting ${waitMs}ms (cost=${normalizedCost}).`, `Riot API throttle waiting ${waitMs}ms for route=${route} cost=${normalizedCost}.`,
); );
} }
await sleep(waitMs); await sleep(waitMs);
} }
} }
public async getBackgroundUsage(): Promise<{ public async recordAppRateLimitObservation(
used1s: number; countHeader: string | null,
used2m: number; limitHeader?: string | null,
}> { ): Promise<void> {
const client = this.garnet.raw(); const count120 = parseRateLimitHeader(countHeader, 120);
const now = Date.now(); if (count120 === null) return;
const usages = await Promise.all(
BACKGROUND_LIMITS.map(async (limit) => {
const key = `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`;
await client.sendCommand([
'ZREMRANGEBYSCORE',
key,
'-inf',
String(now - limit.windowMs),
]);
const response = await client.sendCommand(['ZCARD', key]);
const count = Number(response ?? 0);
return Number.isFinite(count) && count > 0 ? count : 0;
}),
);
return { const limit120 =
used1s: usages[0] ?? 0, parseRateLimitHeader(limitHeader ?? null, 120) ?? DEFAULT_APP_LIMIT_120;
used2m: usages[1] ?? 0, const now = Date.now();
const previous = await this.getAppRateObservation();
const windowStartAt =
!previous ||
now - previous.observedAt >= APP_WINDOW_MS ||
count120 < previous.count120
? now
: previous.windowStartAt;
const observation: AppRateObservation = {
count120,
limit120,
observedAt: now,
windowStartAt,
}; };
// Concurrent responses can race here. This observation is advisory, so
// last-writer-wins is acceptable and self-corrects on the next Riot header.
await this.garnet.setString(
APP_OBSERVED_KEY,
JSON.stringify(observation),
APP_OBSERVED_TTL_SECONDS,
);
} }
private async checkAndAdd( public async isRefreshWindowOpen(
route: RiotRegionalRoute | RiotPlatformRoute, usageRatio: number,
now: number, now = Date.now(),
cost: number, ): Promise<boolean> {
): Promise<number> { const observation = await this.getAppRateObservation();
if (!observation) return true;
if (now - observation.observedAt >= APP_WINDOW_MS) return true;
const ratio = Number.isFinite(usageRatio) ? usageRatio : 0;
const headroomOk = observation.count120 < observation.limit120 * ratio;
const inTail = now >= observation.windowStartAt + APP_WINDOW_MS - TAIL_MS;
return headroomOk && inTail;
}
private async checkAndAdd(now: number, cost: number): Promise<number> {
const keys = APP_LIMITS.map( const keys = APP_LIMITS.map(
(limit) => `riot:rl:${route}:${limit.windowMs}`, (limit) => `${APP_KEY_PREFIX}:${limit.windowMs}`,
); );
return this.checkAndAddWithLimits(keys, APP_LIMITS, now, cost); return this.checkAndAddWithLimits(keys, APP_LIMITS, now, cost);
} }
private async checkAndAddBackground(
now: number,
cost: number,
): Promise<number> {
const keys = BACKGROUND_LIMITS.map(
(limit) => `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`,
);
return this.checkAndAddWithLimits(keys, BACKGROUND_LIMITS, now, cost);
}
private async checkAndAddWithLimits( private async checkAndAddWithLimits(
keys: string[], keys: string[],
limits: ReadonlyArray<{ windowMs: number; max: number }>, limits: ReadonlyArray<{ windowMs: number; max: number }>,
@@ -187,6 +188,32 @@ export class RiotRateLimiterService {
if (!Number.isFinite(waitMs) || waitMs < 0) return 0; if (!Number.isFinite(waitMs) || waitMs < 0) return 0;
return waitMs; return waitMs;
} }
private async getAppRateObservation(): Promise<AppRateObservation | null> {
const raw = await this.garnet.getString(APP_OBSERVED_KEY);
if (!raw) return null;
try {
const parsed = JSON.parse(raw) as Partial<AppRateObservation>;
if (
typeof parsed.count120 !== 'number' ||
typeof parsed.limit120 !== 'number' ||
typeof parsed.observedAt !== 'number' ||
typeof parsed.windowStartAt !== 'number'
) {
return null;
}
return {
count120: parsed.count120,
limit120: parsed.limit120,
observedAt: parsed.observedAt,
windowStartAt: parsed.windowStartAt,
};
} catch {
return null;
}
}
} }
function normalizeCost(cost: number): number { function normalizeCost(cost: number): number {
@@ -194,6 +221,24 @@ function normalizeCost(cost: number): number {
return Math.max(1, Math.floor(cost)); return Math.max(1, Math.floor(cost));
} }
function parseRateLimitHeader(
header: string | null,
windowSeconds: number,
): number | null {
if (!header) return null;
for (const rawEntry of header.split(',')) {
const [rawCount, rawWindow] = rawEntry.trim().split(':');
const count = Number(rawCount);
const window = Number(rawWindow);
if (window === windowSeconds && Number.isFinite(count) && count >= 0) {
return count;
}
}
return null;
}
function sleep(ms: number): Promise<void> { function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
@@ -0,0 +1,43 @@
import { SecretService } from '../global/secret/secret.service';
import { RiotClientService } from './riot.client.service';
import { RiotRateLimiterService } from './riot-rate-limiter.service';
describe('RiotClientService', () => {
const originalFetch = global.fetch;
afterEach(() => {
global.fetch = originalFetch;
jest.restoreAllMocks();
});
it('records Riot app rate limit headers after a response', async () => {
const secretService = {
get: jest.fn().mockResolvedValue('riot-key'),
} as unknown as jest.Mocked<SecretService>;
const rateLimiter = {
acquire: jest.fn().mockResolvedValue(undefined),
recordAppRateLimitObservation: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RiotRateLimiterService>;
global.fetch = jest.fn().mockResolvedValue({
headers: {
get: jest.fn((name: string) => {
if (name === 'X-App-Rate-Limit-Count') return '47:120,3:1';
if (name === 'X-App-Rate-Limit') return '100:120,20:1';
return null;
}),
},
json: jest.fn().mockResolvedValue({ ok: true }),
ok: true,
status: 200,
} as unknown as Response);
const service = new RiotClientService(secretService, rateLimiter);
await service.get('/lol/status/v4/platform-data', { platform: 'kr' });
expect(rateLimiter.acquire).toHaveBeenCalledWith('kr');
expect(rateLimiter.recordAppRateLimitObservation).toHaveBeenCalledWith(
'47:120,3:1',
'100:120,20:1',
);
});
});
@@ -140,16 +140,18 @@ export class RiotClientService {
}); });
const appRateLimitCount = response.headers.get('X-App-Rate-Limit-Count'); const appRateLimitCount = response.headers.get('X-App-Rate-Limit-Count');
const appRateLimit = response.headers.get('X-App-Rate-Limit');
void this.rateLimiter.recordAppRateLimitObservation(
appRateLimitCount,
appRateLimit,
);
if (appRateLimitCount && Math.random() < 0.05) { if (appRateLimitCount && Math.random() < 0.05) {
this.logger.debug(`Riot rate usage ${routeKey}: ${appRateLimitCount}`); this.logger.debug(`Riot rate usage ${routeKey}: ${appRateLimitCount}`);
} }
if (!response.ok) { if (!response.ok) {
const error = await this.createApiError(response, url); const error = await this.createApiError(response, url);
if ( if (error.status === 429 && attempt === 0) {
error.status === 429 &&
attempt === 0
) {
await sleep((error.retryAfterSeconds ?? 1) * 1000); await sleep((error.retryAfterSeconds ?? 1) * 1000);
return this.fetchWithRetry( return this.fetchWithRetry(
url, url,