Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de8bc87940 | ||
|
|
25f3ccad57 |
No files matched your search
@@ -1,9 +1,4 @@
|
|||||||
import {
|
import { Inject, Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||||
Inject,
|
|
||||||
Injectable,
|
|
||||||
Logger,
|
|
||||||
OnModuleDestroy,
|
|
||||||
} from '@nestjs/common';
|
|
||||||
import type { createClient } from 'redis';
|
import type { createClient } from 'redis';
|
||||||
import { GARNET_CLIENT } from './garnet.constants';
|
import { GARNET_CLIENT } from './garnet.constants';
|
||||||
|
|
||||||
@@ -203,6 +198,13 @@ export class GarnetService implements OnModuleDestroy {
|
|||||||
return this.client.xAck(key, group, ids);
|
return this.client.xAck(key, group, ids);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async deleteStreamEntries(
|
||||||
|
key: string,
|
||||||
|
ids: string | string[],
|
||||||
|
): Promise<number> {
|
||||||
|
return this.client.xDel(key, ids);
|
||||||
|
}
|
||||||
|
|
||||||
public async streamLength(key: string): Promise<number> {
|
public async streamLength(key: string): Promise<number> {
|
||||||
return this.client.xLen(key);
|
return this.client.xLen(key);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ function makeScheduler() {
|
|||||||
raw: jest.fn().mockReturnValue(rawClient),
|
raw: jest.fn().mockReturnValue(rawClient),
|
||||||
} as unknown as jest.Mocked<GarnetService> & { raw: jest.Mock };
|
} as unknown as jest.Mocked<GarnetService> & { raw: jest.Mock };
|
||||||
const rateLimiter = {
|
const rateLimiter = {
|
||||||
acquireBackground: jest.fn().mockResolvedValue(undefined),
|
isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
|
||||||
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
||||||
const enrichmentService = {
|
const enrichmentService = {
|
||||||
markRefreshRequested: jest.fn().mockResolvedValue(undefined),
|
markRefreshRequested: jest.fn().mockResolvedValue(undefined),
|
||||||
@@ -84,9 +84,29 @@ describe('BackgroundRefreshScheduler', () => {
|
|||||||
scheduler.onModuleDestroy();
|
scheduler.onModuleDestroy();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not query stale accounts outside the KST window', async () => {
|
it('uses the dawn refresh gate ratio inside the KST background window', async () => {
|
||||||
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||||
|
const { scheduler, rateLimiter } = makeScheduler();
|
||||||
|
|
||||||
|
await scheduler.tick();
|
||||||
|
|
||||||
|
expect(rateLimiter.isRefreshWindowOpen).toHaveBeenCalledWith(0.8);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the opportunistic refresh gate ratio outside the KST background window', async () => {
|
||||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z'));
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z'));
|
||||||
const { scheduler, staleAccountRepository } = makeScheduler();
|
const { scheduler, rateLimiter, staleAccountRepository } = makeScheduler();
|
||||||
|
|
||||||
|
await scheduler.tick();
|
||||||
|
|
||||||
|
expect(rateLimiter.isRefreshWindowOpen).toHaveBeenCalledWith(0.5);
|
||||||
|
expect(staleAccountRepository.findStaleAccounts).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not query stale accounts when the refresh gate is closed', async () => {
|
||||||
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||||
|
const { scheduler, rateLimiter, staleAccountRepository } = makeScheduler();
|
||||||
|
rateLimiter.isRefreshWindowOpen.mockResolvedValue(false);
|
||||||
|
|
||||||
await scheduler.tick();
|
await scheduler.tick();
|
||||||
|
|
||||||
@@ -103,21 +123,15 @@ describe('BackgroundRefreshScheduler', () => {
|
|||||||
expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled();
|
expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('marks and enriches each stale candidate with background rate tokens', async () => {
|
it('marks and enriches each stale candidate without background precharge', async () => {
|
||||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||||
const {
|
const { scheduler, enrichmentService, staleAccountRepository } =
|
||||||
scheduler,
|
makeScheduler();
|
||||||
rateLimiter,
|
|
||||||
enrichmentService,
|
|
||||||
staleAccountRepository,
|
|
||||||
} = makeScheduler();
|
|
||||||
const candidates = [candidate('puuid-1'), candidate('puuid-2')];
|
const candidates = [candidate('puuid-1'), candidate('puuid-2')];
|
||||||
staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates);
|
staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates);
|
||||||
|
|
||||||
await scheduler.tick();
|
await scheduler.tick();
|
||||||
|
|
||||||
expect(rateLimiter.acquireBackground).toHaveBeenCalledTimes(2);
|
|
||||||
expect(rateLimiter.acquireBackground).toHaveBeenCalledWith(8);
|
|
||||||
expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2);
|
expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2);
|
||||||
expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith(
|
expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith(
|
||||||
1,
|
1,
|
||||||
@@ -131,6 +145,28 @@ describe('BackgroundRefreshScheduler', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('stops the candidate loop when the refresh gate closes mid-tick', async () => {
|
||||||
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||||
|
const {
|
||||||
|
scheduler,
|
||||||
|
rateLimiter,
|
||||||
|
enrichmentService,
|
||||||
|
staleAccountRepository,
|
||||||
|
} = makeScheduler();
|
||||||
|
staleAccountRepository.findStaleAccounts.mockResolvedValue([
|
||||||
|
candidate('puuid-1'),
|
||||||
|
candidate('puuid-2'),
|
||||||
|
]);
|
||||||
|
rateLimiter.isRefreshWindowOpen
|
||||||
|
.mockResolvedValueOnce(true)
|
||||||
|
.mockResolvedValueOnce(true)
|
||||||
|
.mockResolvedValueOnce(false);
|
||||||
|
|
||||||
|
await scheduler.tick();
|
||||||
|
|
||||||
|
expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
it('continues after a candidate throws and records a failure counter', async () => {
|
it('continues after a candidate throws and records a failure counter', async () => {
|
||||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||||
const { scheduler, rawClient, enrichmentService, staleAccountRepository } =
|
const { scheduler, rawClient, enrichmentService, staleAccountRepository } =
|
||||||
@@ -157,13 +193,8 @@ describe('BackgroundRefreshScheduler', () => {
|
|||||||
|
|
||||||
it('skips candidates already in failure backoff', async () => {
|
it('skips candidates already in failure backoff', async () => {
|
||||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||||
const {
|
const { scheduler, garnet, enrichmentService, staleAccountRepository } =
|
||||||
scheduler,
|
makeScheduler();
|
||||||
garnet,
|
|
||||||
rateLimiter,
|
|
||||||
enrichmentService,
|
|
||||||
staleAccountRepository,
|
|
||||||
} = makeScheduler();
|
|
||||||
staleAccountRepository.findStaleAccounts.mockResolvedValue([
|
staleAccountRepository.findStaleAccounts.mockResolvedValue([
|
||||||
candidate('skip-me'),
|
candidate('skip-me'),
|
||||||
]);
|
]);
|
||||||
@@ -174,7 +205,6 @@ describe('BackgroundRefreshScheduler', () => {
|
|||||||
|
|
||||||
await scheduler.tick();
|
await scheduler.tick();
|
||||||
|
|
||||||
expect(rateLimiter.acquireBackground).not.toHaveBeenCalled();
|
|
||||||
expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled();
|
expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled();
|
||||||
expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled();
|
expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -19,11 +19,17 @@ const LEADER_TTL_SECONDS = 90;
|
|||||||
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
||||||
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
||||||
const PER_TICK_CANDIDATE_LIMIT = 20;
|
const PER_TICK_CANDIDATE_LIMIT = 20;
|
||||||
const ENRICH_COST_TOKENS = 8;
|
const DAWN_USAGE_RATIO = 0.8;
|
||||||
|
const OPPORTUNISTIC_USAGE_RATIO = 0.5;
|
||||||
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
|
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
|
||||||
const FAILURE_BACKOFF_THRESHOLD = 3;
|
const FAILURE_BACKOFF_THRESHOLD = 3;
|
||||||
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
|
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
|
||||||
|
|
||||||
|
interface RefreshGatePolicy {
|
||||||
|
mode: 'dawn' | 'opportunistic';
|
||||||
|
usageRatio: number;
|
||||||
|
}
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class BackgroundRefreshScheduler
|
export class BackgroundRefreshScheduler
|
||||||
implements OnModuleInit, OnModuleDestroy
|
implements OnModuleInit, OnModuleDestroy
|
||||||
@@ -75,7 +81,11 @@ export class BackgroundRefreshScheduler
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async tick(): Promise<void> {
|
private async tick(): Promise<void> {
|
||||||
if (!isInKstBackgroundWindow()) return;
|
const initialGate = this.currentRefreshGatePolicy();
|
||||||
|
if (!(await this.rateLimiter.isRefreshWindowOpen(initialGate.usageRatio))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const isLeader = await this.tryBecomeLeader();
|
const isLeader = await this.tryBecomeLeader();
|
||||||
if (!isLeader) return;
|
if (!isLeader) return;
|
||||||
|
|
||||||
@@ -89,15 +99,20 @@ export class BackgroundRefreshScheduler
|
|||||||
let processed = 0;
|
let processed = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
let skipped = 0;
|
let skipped = 0;
|
||||||
|
let currentGate = initialGate;
|
||||||
|
|
||||||
for (const candidate of candidates) {
|
for (const candidate of candidates) {
|
||||||
if (!isInKstBackgroundWindow()) break;
|
currentGate = this.currentRefreshGatePolicy();
|
||||||
|
if (
|
||||||
|
!(await this.rateLimiter.isRefreshWindowOpen(currentGate.usageRatio))
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
if (await this.isInFailureBackoff(candidate.puuid)) {
|
if (await this.isInFailureBackoff(candidate.puuid)) {
|
||||||
skipped += 1;
|
skipped += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS);
|
|
||||||
await this.enrichmentService.markRefreshRequested(
|
await this.enrichmentService.markRefreshRequested(
|
||||||
candidate.userId,
|
candidate.userId,
|
||||||
candidate.puuid,
|
candidate.puuid,
|
||||||
@@ -130,10 +145,18 @@ export class BackgroundRefreshScheduler
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`,
|
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length} mode=${currentGate.mode} ratio=${currentGate.usageRatio}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private currentRefreshGatePolicy(): RefreshGatePolicy {
|
||||||
|
const isDawn = isInKstBackgroundWindow();
|
||||||
|
return {
|
||||||
|
mode: isDawn ? 'dawn' : 'opportunistic',
|
||||||
|
usageRatio: isDawn ? DAWN_USAGE_RATIO : OPPORTUNISTIC_USAGE_RATIO,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private async tryBecomeLeader(): Promise<boolean> {
|
private async tryBecomeLeader(): Promise<boolean> {
|
||||||
const current = await this.garnet.getString(LEADER_KEY);
|
const current = await this.garnet.getString(LEADER_KEY);
|
||||||
if (current === this.consumer) {
|
if (current === this.consumer) {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ function execResult<T>(value: T) {
|
|||||||
function makeWorker() {
|
function makeWorker() {
|
||||||
const garnetService = {
|
const garnetService = {
|
||||||
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
||||||
|
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
|
||||||
} as unknown as jest.Mocked<GarnetService>;
|
} as unknown as jest.Mocked<GarnetService>;
|
||||||
const jobQueueService = {
|
const jobQueueService = {
|
||||||
markProcessing: jest.fn().mockResolvedValue(undefined),
|
markProcessing: jest.fn().mockResolvedValue(undefined),
|
||||||
@@ -74,6 +75,32 @@ function makeWorker() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe('RiotJobWorkerService', () => {
|
describe('RiotJobWorkerService', () => {
|
||||||
|
it('acknowledges and deletes invalid stream entries', async () => {
|
||||||
|
const { worker, garnetService } = makeWorker();
|
||||||
|
|
||||||
|
await (
|
||||||
|
worker as unknown as {
|
||||||
|
processEntry: (
|
||||||
|
stream: string,
|
||||||
|
entry: { id: string; message: Record<string, string> },
|
||||||
|
) => Promise<void>;
|
||||||
|
}
|
||||||
|
).processEntry('riot:queue:high', {
|
||||||
|
id: 'bad-1',
|
||||||
|
message: { job: 'not-json' },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(garnetService.acknowledgeStream).toHaveBeenCalledWith(
|
||||||
|
'riot:queue:high',
|
||||||
|
RIOT_JOB_GROUP,
|
||||||
|
'bad-1',
|
||||||
|
);
|
||||||
|
expect(garnetService.deleteStreamEntries).toHaveBeenCalledWith(
|
||||||
|
'riot:queue:high',
|
||||||
|
'bad-1',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('refreshes league entries while processing user history refresh jobs', async () => {
|
it('refreshes league entries while processing user history refresh jobs', async () => {
|
||||||
const { worker, enrichmentService, recentMatchesService, garnetService } =
|
const { worker, enrichmentService, recentMatchesService, garnetService } =
|
||||||
makeWorker();
|
makeWorker();
|
||||||
@@ -91,12 +118,14 @@ describe('RiotJobWorkerService', () => {
|
|||||||
priority: 'high',
|
priority: 'high',
|
||||||
};
|
};
|
||||||
|
|
||||||
await (worker as unknown as {
|
await (
|
||||||
|
worker as unknown as {
|
||||||
processEntry: (
|
processEntry: (
|
||||||
stream: string,
|
stream: string,
|
||||||
entry: { id: string; message: Record<string, string> },
|
entry: { id: string; message: Record<string, string> },
|
||||||
) => Promise<void>;
|
) => Promise<void>;
|
||||||
}).processEntry('riot:queue:high', {
|
}
|
||||||
|
).processEntry('riot:queue:high', {
|
||||||
id: '1-0',
|
id: '1-0',
|
||||||
message: { job: JSON.stringify(job) },
|
message: { job: JSON.stringify(job) },
|
||||||
});
|
});
|
||||||
@@ -112,5 +141,14 @@ describe('RiotJobWorkerService', () => {
|
|||||||
RIOT_JOB_GROUP,
|
RIOT_JOB_GROUP,
|
||||||
'1-0',
|
'1-0',
|
||||||
);
|
);
|
||||||
|
expect(garnetService.deleteStreamEntries).toHaveBeenCalledWith(
|
||||||
|
'riot:queue:high',
|
||||||
|
'1-0',
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
garnetService.acknowledgeStream.mock.invocationCallOrder[0],
|
||||||
|
).toBeLessThan(
|
||||||
|
garnetService.deleteStreamEntries.mock.invocationCallOrder[0],
|
||||||
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -79,10 +79,13 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processEntry(stream: string, entry: StreamEntry): Promise<void> {
|
private async processEntry(
|
||||||
|
stream: string,
|
||||||
|
entry: StreamEntry,
|
||||||
|
): Promise<void> {
|
||||||
const job = this.parseJob(entry);
|
const job = this.parseJob(entry);
|
||||||
if (!job) {
|
if (!job) {
|
||||||
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, entry.id);
|
await this.acknowledgeAndDelete(stream, entry.id);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -134,7 +137,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
}`,
|
}`,
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, entry.id);
|
await this.acknowledgeAndDelete(stream, entry.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async acknowledgeAndDelete(
|
||||||
|
stream: string,
|
||||||
|
id: string,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, id);
|
||||||
|
await this.deleteAcknowledgedEntry(stream, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async deleteAcknowledgedEntry(
|
||||||
|
stream: string,
|
||||||
|
id: string,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.garnetService.deleteStreamEntries(stream, id);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Failed to delete acknowledged Riot job stream entry ${stream} ${id}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,9 +218,7 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
return job.kind === 'poll-tournament-result';
|
return job.kind === 'poll-tournament-result';
|
||||||
}
|
}
|
||||||
|
|
||||||
private isLegacyImportJob(
|
private isLegacyImportJob(job: RiotJobPayload): job is LegacyImportMemberJob {
|
||||||
job: RiotJobPayload,
|
|
||||||
): job is LegacyImportMemberJob {
|
|
||||||
return job.kind === 'legacy-import-member';
|
return job.kind === 'legacy-import-member';
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -219,7 +243,8 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
try {
|
try {
|
||||||
await this.garnetService.createConsumerGroup(stream, RIOT_JOB_GROUP);
|
await this.garnetService.createConsumerGroup(stream, RIOT_JOB_GROUP);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : String(error);
|
const message =
|
||||||
|
error instanceof Error ? error.message : String(error);
|
||||||
if (!message.includes('BUSYGROUP')) throw error;
|
if (!message.includes('BUSYGROUP')) throw error;
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -3,63 +3,156 @@ import { RiotRateLimiterService } from './riot-rate-limiter.service';
|
|||||||
|
|
||||||
function makeService(sendCommand = jest.fn().mockResolvedValue(0)) {
|
function makeService(sendCommand = jest.fn().mockResolvedValue(0)) {
|
||||||
const garnet = {
|
const garnet = {
|
||||||
|
getString: jest.fn().mockResolvedValue(null),
|
||||||
raw: jest.fn().mockReturnValue({ sendCommand }),
|
raw: jest.fn().mockReturnValue({ sendCommand }),
|
||||||
} as unknown as GarnetService;
|
setString: jest.fn().mockResolvedValue(undefined),
|
||||||
|
} as unknown as jest.Mocked<GarnetService>;
|
||||||
return {
|
return {
|
||||||
service: new RiotRateLimiterService(garnet),
|
service: new RiotRateLimiterService(garnet),
|
||||||
|
garnet,
|
||||||
sendCommand,
|
sendCommand,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('RiotRateLimiterService background limiter', () => {
|
function observedState(params: {
|
||||||
|
count120: number;
|
||||||
|
limit120?: number;
|
||||||
|
observedAt: number;
|
||||||
|
windowStartAt: number;
|
||||||
|
}) {
|
||||||
|
return JSON.stringify({
|
||||||
|
count120: params.count120,
|
||||||
|
limit120: params.limit120 ?? 100,
|
||||||
|
observedAt: params.observedAt,
|
||||||
|
windowStartAt: params.windowStartAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('RiotRateLimiterService', () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
jest.restoreAllMocks();
|
jest.restoreAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('uses the background key namespace and passes cost to the Lua script', async () => {
|
it('uses global app rate keys and passes cost to the Lua script', async () => {
|
||||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||||
const { service, sendCommand } = makeService();
|
const { service, sendCommand } = makeService();
|
||||||
|
|
||||||
await service.acquireBackground(8);
|
await service.acquire('asia', 3);
|
||||||
|
|
||||||
expect(sendCommand).toHaveBeenCalledTimes(1);
|
expect(sendCommand).toHaveBeenCalledTimes(1);
|
||||||
const command = sendCommand.mock.calls[0][0] as string[];
|
const command = sendCommand.mock.calls[0][0] as string[];
|
||||||
expect(command[0]).toBe('EVAL');
|
expect(command[0]).toBe('EVAL');
|
||||||
expect(command[2]).toBe('2');
|
expect(command[2]).toBe('2');
|
||||||
expect(command[3]).toBe('riot:bg:rl:1000');
|
expect(command[3]).toBe('riot:rl:app:1000');
|
||||||
expect(command[4]).toBe('riot:bg:rl:120000');
|
expect(command[4]).toBe('riot:rl:app:120000');
|
||||||
expect(command[5]).toBe('1000');
|
expect(command[5]).toBe('1000');
|
||||||
expect(command[7]).toBe('8');
|
expect(command[7]).toBe('3');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('reads cleaned background usage for both windows', async () => {
|
it('records the 120 second app rate observation from Riot headers', async () => {
|
||||||
jest.spyOn(Date, 'now').mockReturnValue(200_000);
|
jest.spyOn(Date, 'now').mockReturnValue(1_000_000);
|
||||||
const sendCommand = jest
|
const { service, garnet } = makeService();
|
||||||
.fn()
|
|
||||||
.mockImplementation(async (command: string[]) => {
|
|
||||||
if (command[0] === 'ZREMRANGEBYSCORE') return 0;
|
|
||||||
if (command[1] === 'riot:bg:rl:1000') return 3;
|
|
||||||
if (command[1] === 'riot:bg:rl:120000') return 24;
|
|
||||||
return 0;
|
|
||||||
});
|
|
||||||
const { service } = makeService(sendCommand);
|
|
||||||
|
|
||||||
await expect(service.getBackgroundUsage()).resolves.toEqual({
|
await service.recordAppRateLimitObservation('47:120,3:1', '100:120,20:1');
|
||||||
used1s: 3,
|
|
||||||
used2m: 24,
|
expect(garnet.setString).toHaveBeenCalledTimes(1);
|
||||||
|
const [key, raw, ttl] = garnet.setString.mock.calls[0];
|
||||||
|
expect(key).toBe('riot:app:rl:observed');
|
||||||
|
expect(ttl).toBe(300);
|
||||||
|
expect(JSON.parse(raw)).toEqual({
|
||||||
|
count120: 47,
|
||||||
|
limit120: 100,
|
||||||
|
observedAt: 1_000_000,
|
||||||
|
windowStartAt: 1_000_000,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(sendCommand).toHaveBeenCalledWith([
|
it('resets the inferred window when Riot count decreases or observation is stale', async () => {
|
||||||
'ZREMRANGEBYSCORE',
|
const { service, garnet } = makeService();
|
||||||
'riot:bg:rl:1000',
|
|
||||||
'-inf',
|
jest.spyOn(Date, 'now').mockReturnValue(1_060_000);
|
||||||
'199000',
|
garnet.getString.mockResolvedValueOnce(
|
||||||
]);
|
observedState({
|
||||||
expect(sendCommand).toHaveBeenCalledWith([
|
count120: 80,
|
||||||
'ZREMRANGEBYSCORE',
|
observedAt: 1_050_000,
|
||||||
'riot:bg:rl:120000',
|
windowStartAt: 1_000_000,
|
||||||
'-inf',
|
}),
|
||||||
'80000',
|
);
|
||||||
]);
|
|
||||||
|
await service.recordAppRateLimitObservation('2:120', null);
|
||||||
|
|
||||||
|
expect(JSON.parse(garnet.setString.mock.calls[0][1])).toMatchObject({
|
||||||
|
count120: 2,
|
||||||
|
windowStartAt: 1_060_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
jest.spyOn(Date, 'now').mockReturnValue(1_240_000);
|
||||||
|
garnet.getString.mockResolvedValueOnce(
|
||||||
|
observedState({
|
||||||
|
count120: 20,
|
||||||
|
observedAt: 1_120_000,
|
||||||
|
windowStartAt: 1_000_000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await service.recordAppRateLimitObservation('25:120', null);
|
||||||
|
|
||||||
|
expect(JSON.parse(garnet.setString.mock.calls[1][1])).toMatchObject({
|
||||||
|
count120: 25,
|
||||||
|
windowStartAt: 1_240_000,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('opens the refresh gate when there is no active observation', async () => {
|
||||||
|
const { service, garnet } = makeService();
|
||||||
|
|
||||||
|
await expect(service.isRefreshWindowOpen(0.5)).resolves.toBe(true);
|
||||||
|
|
||||||
|
garnet.getString.mockResolvedValueOnce(
|
||||||
|
observedState({
|
||||||
|
count120: 90,
|
||||||
|
observedAt: 1_000_000,
|
||||||
|
windowStartAt: 900_000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
jest.spyOn(Date, 'now').mockReturnValue(1_120_000);
|
||||||
|
|
||||||
|
await expect(service.isRefreshWindowOpen(0.5)).resolves.toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
{ ratio: 0.5, count120: 49, expected: true },
|
||||||
|
{ ratio: 0.5, count120: 50, expected: false },
|
||||||
|
{ ratio: 0.8, count120: 79, expected: true },
|
||||||
|
{ ratio: 0.8, count120: 80, expected: false },
|
||||||
|
])(
|
||||||
|
'checks tail headroom for refresh gate ratio=$ratio count=$count120',
|
||||||
|
async ({ ratio, count120, expected }) => {
|
||||||
|
jest.spyOn(Date, 'now').mockReturnValue(1_070_000);
|
||||||
|
const { service, garnet } = makeService();
|
||||||
|
garnet.getString.mockResolvedValue(
|
||||||
|
observedState({
|
||||||
|
count120,
|
||||||
|
observedAt: 1_069_000,
|
||||||
|
windowStartAt: 1_000_000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(service.isRefreshWindowOpen(ratio)).resolves.toBe(expected);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it('keeps the refresh gate closed before the 120 second window tail', async () => {
|
||||||
|
jest.spyOn(Date, 'now').mockReturnValue(1_050_000);
|
||||||
|
const { service, garnet } = makeService();
|
||||||
|
garnet.getString.mockResolvedValue(
|
||||||
|
observedState({
|
||||||
|
count120: 10,
|
||||||
|
observedAt: 1_049_000,
|
||||||
|
windowStartAt: 1_000_000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(service.isRefreshWindowOpen(0.8)).resolves.toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -7,11 +7,19 @@ const APP_LIMITS = [
|
|||||||
{ windowMs: 120_000, max: 90 },
|
{ windowMs: 120_000, max: 90 },
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
const BACKGROUND_LIMITS = [
|
const APP_KEY_PREFIX = 'riot:rl:app';
|
||||||
{ windowMs: 1_000, max: 6 },
|
const APP_WINDOW_MS = 120_000;
|
||||||
{ windowMs: 120_000, max: 36 },
|
const TAIL_MS = 60_000;
|
||||||
] as const;
|
const APP_OBSERVED_KEY = 'riot:app:rl:observed';
|
||||||
const BACKGROUND_KEY_PREFIX = 'riot:bg:rl';
|
const APP_OBSERVED_TTL_SECONDS = 300;
|
||||||
|
const DEFAULT_APP_LIMIT_120 = 100;
|
||||||
|
|
||||||
|
interface AppRateObservation {
|
||||||
|
count120: number;
|
||||||
|
limit120: number;
|
||||||
|
observedAt: number;
|
||||||
|
windowStartAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
const ACQUIRE_SCRIPT = `
|
const ACQUIRE_SCRIPT = `
|
||||||
local now = tonumber(ARGV[1])
|
local now = tonumber(ARGV[1])
|
||||||
@@ -81,85 +89,78 @@ export class RiotRateLimiterService {
|
|||||||
|
|
||||||
public async acquire(
|
public async acquire(
|
||||||
route: RiotRegionalRoute | RiotPlatformRoute,
|
route: RiotRegionalRoute | RiotPlatformRoute,
|
||||||
|
cost = 1,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
for (;;) {
|
|
||||||
const now = Date.now();
|
|
||||||
const waitMs = await this.checkAndAdd(route, now, 1);
|
|
||||||
if (waitMs <= 0) return;
|
|
||||||
|
|
||||||
if (waitMs >= 100) {
|
|
||||||
this.logger.warn(
|
|
||||||
`Riot API throttle waiting ${waitMs}ms for route=${route}.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
await sleep(waitMs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public async acquireBackground(cost = 1): Promise<void> {
|
|
||||||
const normalizedCost = normalizeCost(cost);
|
const normalizedCost = normalizeCost(cost);
|
||||||
for (;;) {
|
for (;;) {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const waitMs = await this.checkAndAddBackground(now, normalizedCost);
|
const waitMs = await this.checkAndAdd(now, normalizedCost);
|
||||||
if (waitMs <= 0) return;
|
if (waitMs <= 0) return;
|
||||||
|
|
||||||
if (waitMs >= 100) {
|
if (waitMs >= 100) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`Background Riot throttle waiting ${waitMs}ms (cost=${normalizedCost}).`,
|
`Riot API throttle waiting ${waitMs}ms for route=${route} cost=${normalizedCost}.`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
await sleep(waitMs);
|
await sleep(waitMs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getBackgroundUsage(): Promise<{
|
public async recordAppRateLimitObservation(
|
||||||
used1s: number;
|
countHeader: string | null,
|
||||||
used2m: number;
|
limitHeader?: string | null,
|
||||||
}> {
|
): Promise<void> {
|
||||||
const client = this.garnet.raw();
|
const count120 = parseRateLimitHeader(countHeader, 120);
|
||||||
const now = Date.now();
|
if (count120 === null) return;
|
||||||
const usages = await Promise.all(
|
|
||||||
BACKGROUND_LIMITS.map(async (limit) => {
|
|
||||||
const key = `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`;
|
|
||||||
await client.sendCommand([
|
|
||||||
'ZREMRANGEBYSCORE',
|
|
||||||
key,
|
|
||||||
'-inf',
|
|
||||||
String(now - limit.windowMs),
|
|
||||||
]);
|
|
||||||
const response = await client.sendCommand(['ZCARD', key]);
|
|
||||||
const count = Number(response ?? 0);
|
|
||||||
return Number.isFinite(count) && count > 0 ? count : 0;
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
return {
|
const limit120 =
|
||||||
used1s: usages[0] ?? 0,
|
parseRateLimitHeader(limitHeader ?? null, 120) ?? DEFAULT_APP_LIMIT_120;
|
||||||
used2m: usages[1] ?? 0,
|
const now = Date.now();
|
||||||
|
const previous = await this.getAppRateObservation();
|
||||||
|
const windowStartAt =
|
||||||
|
!previous ||
|
||||||
|
now - previous.observedAt >= APP_WINDOW_MS ||
|
||||||
|
count120 < previous.count120
|
||||||
|
? now
|
||||||
|
: previous.windowStartAt;
|
||||||
|
|
||||||
|
const observation: AppRateObservation = {
|
||||||
|
count120,
|
||||||
|
limit120,
|
||||||
|
observedAt: now,
|
||||||
|
windowStartAt,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Concurrent responses can race here. This observation is advisory, so
|
||||||
|
// last-writer-wins is acceptable and self-corrects on the next Riot header.
|
||||||
|
await this.garnet.setString(
|
||||||
|
APP_OBSERVED_KEY,
|
||||||
|
JSON.stringify(observation),
|
||||||
|
APP_OBSERVED_TTL_SECONDS,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async checkAndAdd(
|
public async isRefreshWindowOpen(
|
||||||
route: RiotRegionalRoute | RiotPlatformRoute,
|
usageRatio: number,
|
||||||
now: number,
|
now = Date.now(),
|
||||||
cost: number,
|
): Promise<boolean> {
|
||||||
): Promise<number> {
|
const observation = await this.getAppRateObservation();
|
||||||
|
if (!observation) return true;
|
||||||
|
if (now - observation.observedAt >= APP_WINDOW_MS) return true;
|
||||||
|
|
||||||
|
const ratio = Number.isFinite(usageRatio) ? usageRatio : 0;
|
||||||
|
const headroomOk = observation.count120 < observation.limit120 * ratio;
|
||||||
|
const inTail = now >= observation.windowStartAt + APP_WINDOW_MS - TAIL_MS;
|
||||||
|
return headroomOk && inTail;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async checkAndAdd(now: number, cost: number): Promise<number> {
|
||||||
const keys = APP_LIMITS.map(
|
const keys = APP_LIMITS.map(
|
||||||
(limit) => `riot:rl:${route}:${limit.windowMs}`,
|
(limit) => `${APP_KEY_PREFIX}:${limit.windowMs}`,
|
||||||
);
|
);
|
||||||
return this.checkAndAddWithLimits(keys, APP_LIMITS, now, cost);
|
return this.checkAndAddWithLimits(keys, APP_LIMITS, now, cost);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async checkAndAddBackground(
|
|
||||||
now: number,
|
|
||||||
cost: number,
|
|
||||||
): Promise<number> {
|
|
||||||
const keys = BACKGROUND_LIMITS.map(
|
|
||||||
(limit) => `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`,
|
|
||||||
);
|
|
||||||
return this.checkAndAddWithLimits(keys, BACKGROUND_LIMITS, now, cost);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async checkAndAddWithLimits(
|
private async checkAndAddWithLimits(
|
||||||
keys: string[],
|
keys: string[],
|
||||||
limits: ReadonlyArray<{ windowMs: number; max: number }>,
|
limits: ReadonlyArray<{ windowMs: number; max: number }>,
|
||||||
@@ -187,6 +188,32 @@ export class RiotRateLimiterService {
|
|||||||
if (!Number.isFinite(waitMs) || waitMs < 0) return 0;
|
if (!Number.isFinite(waitMs) || waitMs < 0) return 0;
|
||||||
return waitMs;
|
return waitMs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async getAppRateObservation(): Promise<AppRateObservation | null> {
|
||||||
|
const raw = await this.garnet.getString(APP_OBSERVED_KEY);
|
||||||
|
if (!raw) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(raw) as Partial<AppRateObservation>;
|
||||||
|
if (
|
||||||
|
typeof parsed.count120 !== 'number' ||
|
||||||
|
typeof parsed.limit120 !== 'number' ||
|
||||||
|
typeof parsed.observedAt !== 'number' ||
|
||||||
|
typeof parsed.windowStartAt !== 'number'
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
count120: parsed.count120,
|
||||||
|
limit120: parsed.limit120,
|
||||||
|
observedAt: parsed.observedAt,
|
||||||
|
windowStartAt: parsed.windowStartAt,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeCost(cost: number): number {
|
function normalizeCost(cost: number): number {
|
||||||
@@ -194,6 +221,24 @@ function normalizeCost(cost: number): number {
|
|||||||
return Math.max(1, Math.floor(cost));
|
return Math.max(1, Math.floor(cost));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseRateLimitHeader(
|
||||||
|
header: string | null,
|
||||||
|
windowSeconds: number,
|
||||||
|
): number | null {
|
||||||
|
if (!header) return null;
|
||||||
|
|
||||||
|
for (const rawEntry of header.split(',')) {
|
||||||
|
const [rawCount, rawWindow] = rawEntry.trim().split(':');
|
||||||
|
const count = Number(rawCount);
|
||||||
|
const window = Number(rawWindow);
|
||||||
|
if (window === windowSeconds && Number.isFinite(count) && count >= 0) {
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
function sleep(ms: number): Promise<void> {
|
function sleep(ms: number): Promise<void> {
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import { SecretService } from '../global/secret/secret.service';
|
||||||
|
import { RiotClientService } from './riot.client.service';
|
||||||
|
import { RiotRateLimiterService } from './riot-rate-limiter.service';
|
||||||
|
|
||||||
|
describe('RiotClientService', () => {
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
jest.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('records Riot app rate limit headers after a response', async () => {
|
||||||
|
const secretService = {
|
||||||
|
get: jest.fn().mockResolvedValue('riot-key'),
|
||||||
|
} as unknown as jest.Mocked<SecretService>;
|
||||||
|
const rateLimiter = {
|
||||||
|
acquire: jest.fn().mockResolvedValue(undefined),
|
||||||
|
recordAppRateLimitObservation: jest.fn().mockResolvedValue(undefined),
|
||||||
|
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
||||||
|
global.fetch = jest.fn().mockResolvedValue({
|
||||||
|
headers: {
|
||||||
|
get: jest.fn((name: string) => {
|
||||||
|
if (name === 'X-App-Rate-Limit-Count') return '47:120,3:1';
|
||||||
|
if (name === 'X-App-Rate-Limit') return '100:120,20:1';
|
||||||
|
return null;
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
json: jest.fn().mockResolvedValue({ ok: true }),
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
} as unknown as Response);
|
||||||
|
const service = new RiotClientService(secretService, rateLimiter);
|
||||||
|
|
||||||
|
await service.get('/lol/status/v4/platform-data', { platform: 'kr' });
|
||||||
|
|
||||||
|
expect(rateLimiter.acquire).toHaveBeenCalledWith('kr');
|
||||||
|
expect(rateLimiter.recordAppRateLimitObservation).toHaveBeenCalledWith(
|
||||||
|
'47:120,3:1',
|
||||||
|
'100:120,20:1',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -140,16 +140,18 @@ export class RiotClientService {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const appRateLimitCount = response.headers.get('X-App-Rate-Limit-Count');
|
const appRateLimitCount = response.headers.get('X-App-Rate-Limit-Count');
|
||||||
|
const appRateLimit = response.headers.get('X-App-Rate-Limit');
|
||||||
|
void this.rateLimiter.recordAppRateLimitObservation(
|
||||||
|
appRateLimitCount,
|
||||||
|
appRateLimit,
|
||||||
|
);
|
||||||
if (appRateLimitCount && Math.random() < 0.05) {
|
if (appRateLimitCount && Math.random() < 0.05) {
|
||||||
this.logger.debug(`Riot rate usage ${routeKey}: ${appRateLimitCount}`);
|
this.logger.debug(`Riot rate usage ${routeKey}: ${appRateLimitCount}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const error = await this.createApiError(response, url);
|
const error = await this.createApiError(response, url);
|
||||||
if (
|
if (error.status === 429 && attempt === 0) {
|
||||||
error.status === 429 &&
|
|
||||||
attempt === 0
|
|
||||||
) {
|
|
||||||
await sleep((error.retryAfterSeconds ?? 1) * 1000);
|
await sleep((error.retryAfterSeconds ?? 1) * 1000);
|
||||||
return this.fetchWithRetry(
|
return this.fetchWithRetry(
|
||||||
url,
|
url,
|
||||||
|
|||||||
Reference in new issue
Block a user