diff --git a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts index ad54fcd5..fa0d8196 100644 --- a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts +++ b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts @@ -41,7 +41,7 @@ function makeScheduler() { raw: jest.fn().mockReturnValue(rawClient), } as unknown as jest.Mocked & { raw: jest.Mock }; const rateLimiter = { - acquireBackground: jest.fn().mockResolvedValue(undefined), + isRefreshWindowOpen: jest.fn().mockResolvedValue(true), } as unknown as jest.Mocked; const enrichmentService = { markRefreshRequested: jest.fn().mockResolvedValue(undefined), @@ -84,9 +84,29 @@ describe('BackgroundRefreshScheduler', () => { scheduler.onModuleDestroy(); }); - it('does not query stale accounts outside the KST window', async () => { + it('uses the dawn refresh gate ratio inside the KST background window', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); + const { scheduler, rateLimiter } = makeScheduler(); + + await scheduler.tick(); + + expect(rateLimiter.isRefreshWindowOpen).toHaveBeenCalledWith(0.8); + }); + + it('uses the opportunistic refresh gate ratio outside the KST background window', async () => { jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z')); - const { scheduler, staleAccountRepository } = makeScheduler(); + const { scheduler, rateLimiter, staleAccountRepository } = makeScheduler(); + + await scheduler.tick(); + + expect(rateLimiter.isRefreshWindowOpen).toHaveBeenCalledWith(0.5); + expect(staleAccountRepository.findStaleAccounts).toHaveBeenCalledTimes(1); + }); + + it('does not query stale accounts when the refresh gate is closed', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); + const { scheduler, rateLimiter, staleAccountRepository } = makeScheduler(); + rateLimiter.isRefreshWindowOpen.mockResolvedValue(false); await scheduler.tick(); @@ -103,21 +123,15 @@ describe('BackgroundRefreshScheduler', () => { expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled(); }); - it('marks and enriches each stale candidate with background rate tokens', async () => { + it('marks and enriches each stale candidate without background precharge', async () => { jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); - const { - scheduler, - rateLimiter, - enrichmentService, - staleAccountRepository, - } = makeScheduler(); + const { scheduler, enrichmentService, staleAccountRepository } = + makeScheduler(); const candidates = [candidate('puuid-1'), candidate('puuid-2')]; staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates); await scheduler.tick(); - expect(rateLimiter.acquireBackground).toHaveBeenCalledTimes(2); - expect(rateLimiter.acquireBackground).toHaveBeenCalledWith(8); expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2); expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith( 1, @@ -131,6 +145,28 @@ describe('BackgroundRefreshScheduler', () => { }); }); + it('stops the candidate loop when the refresh gate closes mid-tick', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); + const { + scheduler, + rateLimiter, + enrichmentService, + staleAccountRepository, + } = makeScheduler(); + staleAccountRepository.findStaleAccounts.mockResolvedValue([ + candidate('puuid-1'), + candidate('puuid-2'), + ]); + rateLimiter.isRefreshWindowOpen + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + + await scheduler.tick(); + + expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(1); + }); + it('continues after a candidate throws and records a failure counter', async () => { jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); const { scheduler, rawClient, enrichmentService, staleAccountRepository } = @@ -157,13 +193,8 @@ describe('BackgroundRefreshScheduler', () => { it('skips candidates already in failure backoff', async () => { jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); - const { - scheduler, - garnet, - rateLimiter, - enrichmentService, - staleAccountRepository, - } = makeScheduler(); + const { scheduler, garnet, enrichmentService, staleAccountRepository } = + makeScheduler(); staleAccountRepository.findStaleAccounts.mockResolvedValue([ candidate('skip-me'), ]); @@ -174,7 +205,6 @@ describe('BackgroundRefreshScheduler', () => { await scheduler.tick(); - expect(rateLimiter.acquireBackground).not.toHaveBeenCalled(); expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled(); expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled(); }); diff --git a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts index a95448fb..82dc3936 100644 --- a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts +++ b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts @@ -19,11 +19,17 @@ const LEADER_TTL_SECONDS = 90; const LEADER_RENEW_INTERVAL_MS = 30_000; const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000; const PER_TICK_CANDIDATE_LIMIT = 20; -const ENRICH_COST_TOKENS = 8; +const DAWN_USAGE_RATIO = 0.8; +const OPPORTUNISTIC_USAGE_RATIO = 0.5; const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail'; const FAILURE_BACKOFF_THRESHOLD = 3; const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60; +interface RefreshGatePolicy { + mode: 'dawn' | 'opportunistic'; + usageRatio: number; +} + @Injectable() export class BackgroundRefreshScheduler implements OnModuleInit, OnModuleDestroy @@ -75,7 +81,11 @@ export class BackgroundRefreshScheduler } private async tick(): Promise { - if (!isInKstBackgroundWindow()) return; + const initialGate = this.currentRefreshGatePolicy(); + if (!(await this.rateLimiter.isRefreshWindowOpen(initialGate.usageRatio))) { + return; + } + const isLeader = await this.tryBecomeLeader(); if (!isLeader) return; @@ -89,15 +99,20 @@ export class BackgroundRefreshScheduler let processed = 0; let failed = 0; let skipped = 0; + let currentGate = initialGate; for (const candidate of candidates) { - if (!isInKstBackgroundWindow()) break; + currentGate = this.currentRefreshGatePolicy(); + if ( + !(await this.rateLimiter.isRefreshWindowOpen(currentGate.usageRatio)) + ) { + break; + } if (await this.isInFailureBackoff(candidate.puuid)) { skipped += 1; continue; } - await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS); await this.enrichmentService.markRefreshRequested( candidate.userId, candidate.puuid, @@ -130,10 +145,18 @@ export class BackgroundRefreshScheduler } this.logger.debug( - `Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`, + `Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length} mode=${currentGate.mode} ratio=${currentGate.usageRatio}`, ); } + private currentRefreshGatePolicy(): RefreshGatePolicy { + const isDawn = isInKstBackgroundWindow(); + return { + mode: isDawn ? 'dawn' : 'opportunistic', + usageRatio: isDawn ? DAWN_USAGE_RATIO : OPPORTUNISTIC_USAGE_RATIO, + }; + } + private async tryBecomeLeader(): Promise { const current = await this.garnet.getString(LEADER_KEY); if (current === this.consumer) { diff --git a/apps/api/src/modules/riot/riot-rate-limiter.service.spec.ts b/apps/api/src/modules/riot/riot-rate-limiter.service.spec.ts index af91d290..67023ba6 100644 --- a/apps/api/src/modules/riot/riot-rate-limiter.service.spec.ts +++ b/apps/api/src/modules/riot/riot-rate-limiter.service.spec.ts @@ -3,63 +3,156 @@ import { RiotRateLimiterService } from './riot-rate-limiter.service'; function makeService(sendCommand = jest.fn().mockResolvedValue(0)) { const garnet = { + getString: jest.fn().mockResolvedValue(null), raw: jest.fn().mockReturnValue({ sendCommand }), - } as unknown as GarnetService; + setString: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; return { service: new RiotRateLimiterService(garnet), + garnet, sendCommand, }; } -describe('RiotRateLimiterService background limiter', () => { +function observedState(params: { + count120: number; + limit120?: number; + observedAt: number; + windowStartAt: number; +}) { + return JSON.stringify({ + count120: params.count120, + limit120: params.limit120 ?? 100, + observedAt: params.observedAt, + windowStartAt: params.windowStartAt, + }); +} + +describe('RiotRateLimiterService', () => { afterEach(() => { jest.restoreAllMocks(); }); - it('uses the background key namespace and passes cost to the Lua script', async () => { + it('uses global app rate keys and passes cost to the Lua script', async () => { jest.spyOn(Date, 'now').mockReturnValue(1_000); const { service, sendCommand } = makeService(); - await service.acquireBackground(8); + await service.acquire('asia', 3); expect(sendCommand).toHaveBeenCalledTimes(1); const command = sendCommand.mock.calls[0][0] as string[]; expect(command[0]).toBe('EVAL'); expect(command[2]).toBe('2'); - expect(command[3]).toBe('riot:bg:rl:1000'); - expect(command[4]).toBe('riot:bg:rl:120000'); + expect(command[3]).toBe('riot:rl:app:1000'); + expect(command[4]).toBe('riot:rl:app:120000'); expect(command[5]).toBe('1000'); - expect(command[7]).toBe('8'); + expect(command[7]).toBe('3'); }); - it('reads cleaned background usage for both windows', async () => { - jest.spyOn(Date, 'now').mockReturnValue(200_000); - const sendCommand = jest - .fn() - .mockImplementation(async (command: string[]) => { - if (command[0] === 'ZREMRANGEBYSCORE') return 0; - if (command[1] === 'riot:bg:rl:1000') return 3; - if (command[1] === 'riot:bg:rl:120000') return 24; - return 0; - }); - const { service } = makeService(sendCommand); + it('records the 120 second app rate observation from Riot headers', async () => { + jest.spyOn(Date, 'now').mockReturnValue(1_000_000); + const { service, garnet } = makeService(); - await expect(service.getBackgroundUsage()).resolves.toEqual({ - used1s: 3, - used2m: 24, + await service.recordAppRateLimitObservation('47:120,3:1', '100:120,20:1'); + + expect(garnet.setString).toHaveBeenCalledTimes(1); + const [key, raw, ttl] = garnet.setString.mock.calls[0]; + expect(key).toBe('riot:app:rl:observed'); + expect(ttl).toBe(300); + expect(JSON.parse(raw)).toEqual({ + count120: 47, + limit120: 100, + observedAt: 1_000_000, + windowStartAt: 1_000_000, + }); + }); + + it('resets the inferred window when Riot count decreases or observation is stale', async () => { + const { service, garnet } = makeService(); + + jest.spyOn(Date, 'now').mockReturnValue(1_060_000); + garnet.getString.mockResolvedValueOnce( + observedState({ + count120: 80, + observedAt: 1_050_000, + windowStartAt: 1_000_000, + }), + ); + + await service.recordAppRateLimitObservation('2:120', null); + + expect(JSON.parse(garnet.setString.mock.calls[0][1])).toMatchObject({ + count120: 2, + windowStartAt: 1_060_000, }); - expect(sendCommand).toHaveBeenCalledWith([ - 'ZREMRANGEBYSCORE', - 'riot:bg:rl:1000', - '-inf', - '199000', - ]); - expect(sendCommand).toHaveBeenCalledWith([ - 'ZREMRANGEBYSCORE', - 'riot:bg:rl:120000', - '-inf', - '80000', - ]); + jest.spyOn(Date, 'now').mockReturnValue(1_240_000); + garnet.getString.mockResolvedValueOnce( + observedState({ + count120: 20, + observedAt: 1_120_000, + windowStartAt: 1_000_000, + }), + ); + + await service.recordAppRateLimitObservation('25:120', null); + + expect(JSON.parse(garnet.setString.mock.calls[1][1])).toMatchObject({ + count120: 25, + windowStartAt: 1_240_000, + }); + }); + + it('opens the refresh gate when there is no active observation', async () => { + const { service, garnet } = makeService(); + + await expect(service.isRefreshWindowOpen(0.5)).resolves.toBe(true); + + garnet.getString.mockResolvedValueOnce( + observedState({ + count120: 90, + observedAt: 1_000_000, + windowStartAt: 900_000, + }), + ); + jest.spyOn(Date, 'now').mockReturnValue(1_120_000); + + await expect(service.isRefreshWindowOpen(0.5)).resolves.toBe(true); + }); + + it.each([ + { ratio: 0.5, count120: 49, expected: true }, + { ratio: 0.5, count120: 50, expected: false }, + { ratio: 0.8, count120: 79, expected: true }, + { ratio: 0.8, count120: 80, expected: false }, + ])( + 'checks tail headroom for refresh gate ratio=$ratio count=$count120', + async ({ ratio, count120, expected }) => { + jest.spyOn(Date, 'now').mockReturnValue(1_070_000); + const { service, garnet } = makeService(); + garnet.getString.mockResolvedValue( + observedState({ + count120, + observedAt: 1_069_000, + windowStartAt: 1_000_000, + }), + ); + + await expect(service.isRefreshWindowOpen(ratio)).resolves.toBe(expected); + }, + ); + + it('keeps the refresh gate closed before the 120 second window tail', async () => { + jest.spyOn(Date, 'now').mockReturnValue(1_050_000); + const { service, garnet } = makeService(); + garnet.getString.mockResolvedValue( + observedState({ + count120: 10, + observedAt: 1_049_000, + windowStartAt: 1_000_000, + }), + ); + + await expect(service.isRefreshWindowOpen(0.8)).resolves.toBe(false); }); }); diff --git a/apps/api/src/modules/riot/riot-rate-limiter.service.ts b/apps/api/src/modules/riot/riot-rate-limiter.service.ts index a26dde07..a7d39e99 100644 --- a/apps/api/src/modules/riot/riot-rate-limiter.service.ts +++ b/apps/api/src/modules/riot/riot-rate-limiter.service.ts @@ -7,11 +7,19 @@ const APP_LIMITS = [ { windowMs: 120_000, max: 90 }, ] as const; -const BACKGROUND_LIMITS = [ - { windowMs: 1_000, max: 6 }, - { windowMs: 120_000, max: 36 }, -] as const; -const BACKGROUND_KEY_PREFIX = 'riot:bg:rl'; +const APP_KEY_PREFIX = 'riot:rl:app'; +const APP_WINDOW_MS = 120_000; +const TAIL_MS = 60_000; +const APP_OBSERVED_KEY = 'riot:app:rl:observed'; +const APP_OBSERVED_TTL_SECONDS = 300; +const DEFAULT_APP_LIMIT_120 = 100; + +interface AppRateObservation { + count120: number; + limit120: number; + observedAt: number; + windowStartAt: number; +} const ACQUIRE_SCRIPT = ` local now = tonumber(ARGV[1]) @@ -81,85 +89,78 @@ export class RiotRateLimiterService { public async acquire( route: RiotRegionalRoute | RiotPlatformRoute, + cost = 1, ): Promise { - for (;;) { - const now = Date.now(); - const waitMs = await this.checkAndAdd(route, now, 1); - if (waitMs <= 0) return; - - if (waitMs >= 100) { - this.logger.warn( - `Riot API throttle waiting ${waitMs}ms for route=${route}.`, - ); - } - await sleep(waitMs); - } - } - - public async acquireBackground(cost = 1): Promise { const normalizedCost = normalizeCost(cost); for (;;) { const now = Date.now(); - const waitMs = await this.checkAndAddBackground(now, normalizedCost); + const waitMs = await this.checkAndAdd(now, normalizedCost); if (waitMs <= 0) return; if (waitMs >= 100) { this.logger.warn( - `Background Riot throttle waiting ${waitMs}ms (cost=${normalizedCost}).`, + `Riot API throttle waiting ${waitMs}ms for route=${route} cost=${normalizedCost}.`, ); } await sleep(waitMs); } } - public async getBackgroundUsage(): Promise<{ - used1s: number; - used2m: number; - }> { - const client = this.garnet.raw(); - const now = Date.now(); - const usages = await Promise.all( - BACKGROUND_LIMITS.map(async (limit) => { - const key = `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`; - await client.sendCommand([ - 'ZREMRANGEBYSCORE', - key, - '-inf', - String(now - limit.windowMs), - ]); - const response = await client.sendCommand(['ZCARD', key]); - const count = Number(response ?? 0); - return Number.isFinite(count) && count > 0 ? count : 0; - }), - ); + public async recordAppRateLimitObservation( + countHeader: string | null, + limitHeader?: string | null, + ): Promise { + const count120 = parseRateLimitHeader(countHeader, 120); + if (count120 === null) return; - return { - used1s: usages[0] ?? 0, - used2m: usages[1] ?? 0, + const limit120 = + parseRateLimitHeader(limitHeader ?? null, 120) ?? DEFAULT_APP_LIMIT_120; + const now = Date.now(); + const previous = await this.getAppRateObservation(); + const windowStartAt = + !previous || + now - previous.observedAt >= APP_WINDOW_MS || + count120 < previous.count120 + ? now + : previous.windowStartAt; + + const observation: AppRateObservation = { + count120, + limit120, + observedAt: now, + windowStartAt, }; + + // Concurrent responses can race here. This observation is advisory, so + // last-writer-wins is acceptable and self-corrects on the next Riot header. + await this.garnet.setString( + APP_OBSERVED_KEY, + JSON.stringify(observation), + APP_OBSERVED_TTL_SECONDS, + ); } - private async checkAndAdd( - route: RiotRegionalRoute | RiotPlatformRoute, - now: number, - cost: number, - ): Promise { + public async isRefreshWindowOpen( + usageRatio: number, + now = Date.now(), + ): Promise { + const observation = await this.getAppRateObservation(); + if (!observation) return true; + if (now - observation.observedAt >= APP_WINDOW_MS) return true; + + const ratio = Number.isFinite(usageRatio) ? usageRatio : 0; + const headroomOk = observation.count120 < observation.limit120 * ratio; + const inTail = now >= observation.windowStartAt + APP_WINDOW_MS - TAIL_MS; + return headroomOk && inTail; + } + + private async checkAndAdd(now: number, cost: number): Promise { const keys = APP_LIMITS.map( - (limit) => `riot:rl:${route}:${limit.windowMs}`, + (limit) => `${APP_KEY_PREFIX}:${limit.windowMs}`, ); return this.checkAndAddWithLimits(keys, APP_LIMITS, now, cost); } - private async checkAndAddBackground( - now: number, - cost: number, - ): Promise { - const keys = BACKGROUND_LIMITS.map( - (limit) => `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`, - ); - return this.checkAndAddWithLimits(keys, BACKGROUND_LIMITS, now, cost); - } - private async checkAndAddWithLimits( keys: string[], limits: ReadonlyArray<{ windowMs: number; max: number }>, @@ -187,6 +188,32 @@ export class RiotRateLimiterService { if (!Number.isFinite(waitMs) || waitMs < 0) return 0; return waitMs; } + + private async getAppRateObservation(): Promise { + const raw = await this.garnet.getString(APP_OBSERVED_KEY); + if (!raw) return null; + + try { + const parsed = JSON.parse(raw) as Partial; + if ( + typeof parsed.count120 !== 'number' || + typeof parsed.limit120 !== 'number' || + typeof parsed.observedAt !== 'number' || + typeof parsed.windowStartAt !== 'number' + ) { + return null; + } + + return { + count120: parsed.count120, + limit120: parsed.limit120, + observedAt: parsed.observedAt, + windowStartAt: parsed.windowStartAt, + }; + } catch { + return null; + } + } } function normalizeCost(cost: number): number { @@ -194,6 +221,24 @@ function normalizeCost(cost: number): number { return Math.max(1, Math.floor(cost)); } +function parseRateLimitHeader( + header: string | null, + windowSeconds: number, +): number | null { + if (!header) return null; + + for (const rawEntry of header.split(',')) { + const [rawCount, rawWindow] = rawEntry.trim().split(':'); + const count = Number(rawCount); + const window = Number(rawWindow); + if (window === windowSeconds && Number.isFinite(count) && count >= 0) { + return count; + } + } + + return null; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/apps/api/src/modules/riot/riot.client.service.spec.ts b/apps/api/src/modules/riot/riot.client.service.spec.ts new file mode 100644 index 00000000..479a6b52 --- /dev/null +++ b/apps/api/src/modules/riot/riot.client.service.spec.ts @@ -0,0 +1,43 @@ +import { SecretService } from '../global/secret/secret.service'; +import { RiotClientService } from './riot.client.service'; +import { RiotRateLimiterService } from './riot-rate-limiter.service'; + +describe('RiotClientService', () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + jest.restoreAllMocks(); + }); + + it('records Riot app rate limit headers after a response', async () => { + const secretService = { + get: jest.fn().mockResolvedValue('riot-key'), + } as unknown as jest.Mocked; + const rateLimiter = { + acquire: jest.fn().mockResolvedValue(undefined), + recordAppRateLimitObservation: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + global.fetch = jest.fn().mockResolvedValue({ + headers: { + get: jest.fn((name: string) => { + if (name === 'X-App-Rate-Limit-Count') return '47:120,3:1'; + if (name === 'X-App-Rate-Limit') return '100:120,20:1'; + return null; + }), + }, + json: jest.fn().mockResolvedValue({ ok: true }), + ok: true, + status: 200, + } as unknown as Response); + const service = new RiotClientService(secretService, rateLimiter); + + await service.get('/lol/status/v4/platform-data', { platform: 'kr' }); + + expect(rateLimiter.acquire).toHaveBeenCalledWith('kr'); + expect(rateLimiter.recordAppRateLimitObservation).toHaveBeenCalledWith( + '47:120,3:1', + '100:120,20:1', + ); + }); +}); diff --git a/apps/api/src/modules/riot/riot.client.service.ts b/apps/api/src/modules/riot/riot.client.service.ts index 9c8e3225..c8c0ecd9 100644 --- a/apps/api/src/modules/riot/riot.client.service.ts +++ b/apps/api/src/modules/riot/riot.client.service.ts @@ -140,16 +140,18 @@ export class RiotClientService { }); const appRateLimitCount = response.headers.get('X-App-Rate-Limit-Count'); + const appRateLimit = response.headers.get('X-App-Rate-Limit'); + void this.rateLimiter.recordAppRateLimitObservation( + appRateLimitCount, + appRateLimit, + ); if (appRateLimitCount && Math.random() < 0.05) { this.logger.debug(`Riot rate usage ${routeKey}: ${appRateLimitCount}`); } if (!response.ok) { const error = await this.createApiError(response, url); - if ( - error.status === 429 && - attempt === 0 - ) { + if (error.status === 429 && attempt === 0) { await sleep((error.retryAfterSeconds ?? 1) * 1000); return this.fetchWithRetry( url,