.
This commit is contained in:
1 parent
5df3ea67e7
commit
3b3c564075
13 files changed
+821
-27
No files matched your search
@@ -0,0 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { RiotModule } from '../riot/riot.module';
|
||||
import { BackgroundRefreshScheduler } from './background-refresh.scheduler';
|
||||
import { StaleAccountRepository } from './stale-account.repository';
|
||||
|
||||
@Module({
|
||||
imports: [ConfigModule, RiotModule],
|
||||
providers: [BackgroundRefreshScheduler, StaleAccountRepository],
|
||||
})
|
||||
export class BackgroundRefreshModule {}
|
||||
@@ -0,0 +1,186 @@
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import mongoose from 'mongoose';
|
||||
import { GarnetService } from '../global/garnet/garnet.service';
|
||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
import { BackgroundRefreshScheduler } from './background-refresh.scheduler';
|
||||
import {
|
||||
StaleAccountCandidate,
|
||||
StaleAccountRepository,
|
||||
} from './stale-account.repository';
|
||||
|
||||
type SchedulerInternals = BackgroundRefreshScheduler & {
|
||||
tick(): Promise<void>;
|
||||
};
|
||||
|
||||
function successResult() {
|
||||
return {
|
||||
summoner: 'fresh',
|
||||
league: 'fresh',
|
||||
masteries: 'fresh',
|
||||
recentMatches: 'fresh',
|
||||
account: {},
|
||||
};
|
||||
}
|
||||
|
||||
function candidate(puuid: string): StaleAccountCandidate {
|
||||
return {
|
||||
userId: new mongoose.Types.ObjectId(),
|
||||
puuid,
|
||||
};
|
||||
}
|
||||
|
||||
function makeScheduler(enabled = true) {
|
||||
const rawClient = {
|
||||
sendCommand: jest.fn().mockResolvedValue(1),
|
||||
};
|
||||
const garnet = {
|
||||
getString: jest.fn().mockResolvedValue(null),
|
||||
setString: jest.fn().mockResolvedValue(undefined),
|
||||
setStringIfAbsent: jest.fn().mockResolvedValue(true),
|
||||
delete: jest.fn().mockResolvedValue(1),
|
||||
raw: jest.fn().mockReturnValue(rawClient),
|
||||
} as unknown as jest.Mocked<GarnetService> & { raw: jest.Mock };
|
||||
const rateLimiter = {
|
||||
acquireBackground: jest.fn().mockResolvedValue(undefined),
|
||||
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
||||
const enrichmentService = {
|
||||
markRefreshRequested: jest.fn().mockResolvedValue(undefined),
|
||||
enrichRiotAccount: jest.fn().mockResolvedValue(successResult()),
|
||||
} as unknown as jest.Mocked<RiotAccountEnrichmentService>;
|
||||
const staleAccountRepository = {
|
||||
findStaleAccounts: jest.fn().mockResolvedValue([]),
|
||||
} as unknown as jest.Mocked<StaleAccountRepository>;
|
||||
const config = {
|
||||
get: jest.fn().mockReturnValue(enabled ? 'true' : 'false'),
|
||||
} as unknown as ConfigService;
|
||||
|
||||
const scheduler = new BackgroundRefreshScheduler(
|
||||
garnet,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
staleAccountRepository,
|
||||
config,
|
||||
) as SchedulerInternals;
|
||||
|
||||
return {
|
||||
scheduler,
|
||||
garnet,
|
||||
rawClient,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
staleAccountRepository,
|
||||
};
|
||||
}
|
||||
|
||||
describe('BackgroundRefreshScheduler', () => {
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('does not schedule ticks when disabled', () => {
|
||||
jest.useFakeTimers();
|
||||
const setIntervalSpy = jest.spyOn(global, 'setInterval');
|
||||
const { scheduler } = makeScheduler(false);
|
||||
|
||||
scheduler.onModuleInit();
|
||||
|
||||
expect(setIntervalSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not query stale accounts outside the KST window', async () => {
|
||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z'));
|
||||
const { scheduler, staleAccountRepository } = makeScheduler();
|
||||
|
||||
await scheduler.tick();
|
||||
|
||||
expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not query stale accounts when another leader holds the lock', async () => {
|
||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||
const { scheduler, garnet, staleAccountRepository } = makeScheduler();
|
||||
garnet.getString.mockResolvedValue('another-consumer');
|
||||
|
||||
await scheduler.tick();
|
||||
|
||||
expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('marks and enriches each stale candidate with background rate tokens', async () => {
|
||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||
const {
|
||||
scheduler,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
staleAccountRepository,
|
||||
} = makeScheduler();
|
||||
const candidates = [candidate('puuid-1'), candidate('puuid-2')];
|
||||
staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates);
|
||||
|
||||
await scheduler.tick();
|
||||
|
||||
expect(rateLimiter.acquireBackground).toHaveBeenCalledTimes(2);
|
||||
expect(rateLimiter.acquireBackground).toHaveBeenCalledWith(8);
|
||||
expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2);
|
||||
expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
candidates[0].userId,
|
||||
'puuid-1',
|
||||
);
|
||||
expect(enrichmentService.enrichRiotAccount).toHaveBeenNthCalledWith(2, {
|
||||
userId: candidates[1].userId,
|
||||
puuid: 'puuid-2',
|
||||
options: { force: false },
|
||||
});
|
||||
});
|
||||
|
||||
it('continues after a candidate throws and records a failure counter', async () => {
|
||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||
const { scheduler, rawClient, enrichmentService, staleAccountRepository } =
|
||||
makeScheduler();
|
||||
const candidates = [candidate('puuid-1'), candidate('puuid-2')];
|
||||
staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates);
|
||||
enrichmentService.enrichRiotAccount
|
||||
.mockRejectedValueOnce(new Error('boom'))
|
||||
.mockResolvedValueOnce(successResult() as never);
|
||||
|
||||
await scheduler.tick();
|
||||
|
||||
expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(2);
|
||||
expect(rawClient.sendCommand).toHaveBeenCalledWith([
|
||||
'INCR',
|
||||
'riot:bg:fail:puuid-1',
|
||||
]);
|
||||
expect(rawClient.sendCommand).toHaveBeenCalledWith([
|
||||
'EXPIRE',
|
||||
'riot:bg:fail:puuid-1',
|
||||
'86400',
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips candidates already in failure backoff', async () => {
|
||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||
const {
|
||||
scheduler,
|
||||
garnet,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
staleAccountRepository,
|
||||
} = makeScheduler();
|
||||
staleAccountRepository.findStaleAccounts.mockResolvedValue([
|
||||
candidate('skip-me'),
|
||||
]);
|
||||
garnet.getString.mockImplementation(async (key: string) => {
|
||||
if (key.includes('riot:bg:fail:skip-me')) return '3';
|
||||
return null;
|
||||
});
|
||||
|
||||
await scheduler.tick();
|
||||
|
||||
expect(rateLimiter.acquireBackground).not.toHaveBeenCalled();
|
||||
expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled();
|
||||
expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,233 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import {
|
||||
type EnrichResult,
|
||||
RiotAccountEnrichmentService,
|
||||
} from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
import { GarnetService } from '../global/garnet/garnet.service';
|
||||
import { isInKstBackgroundWindow } from './kst-window.util';
|
||||
import { StaleAccountRepository } from './stale-account.repository';
|
||||
|
||||
const TICK_INTERVAL_MS = 60_000;
|
||||
const LEADER_KEY = 'riot:bg-refresh:leader';
|
||||
const LEADER_TTL_SECONDS = 90;
|
||||
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
||||
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
||||
const PER_TICK_CANDIDATE_LIMIT = 20;
|
||||
const ENRICH_COST_TOKENS = 8;
|
||||
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
|
||||
const FAILURE_BACKOFF_THRESHOLD = 3;
|
||||
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
|
||||
|
||||
@Injectable()
|
||||
export class BackgroundRefreshScheduler
|
||||
implements OnModuleInit, OnModuleDestroy
|
||||
{
|
||||
private readonly logger = new Logger(BackgroundRefreshScheduler.name);
|
||||
private tickTimer?: NodeJS.Timeout;
|
||||
private leaderRenewTimer?: NodeJS.Timeout;
|
||||
private tickInProgress = false;
|
||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(
|
||||
private readonly garnet: GarnetService,
|
||||
private readonly rateLimiter: RiotRateLimiterService,
|
||||
private readonly enrichmentService: RiotAccountEnrichmentService,
|
||||
private readonly staleAccountRepository: StaleAccountRepository,
|
||||
config: ConfigService,
|
||||
) {
|
||||
this.enabled = config.get<string>('RIOT_BG_REFRESH_ENABLED') === 'true';
|
||||
}
|
||||
|
||||
public onModuleInit(): void {
|
||||
if (!this.enabled) {
|
||||
this.logger.log(
|
||||
'Background Riot refresh is disabled (RIOT_BG_REFRESH_ENABLED).',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS);
|
||||
this.leaderRenewTimer = setInterval(
|
||||
() => void this.renewLeaderIfHeld(),
|
||||
LEADER_RENEW_INTERVAL_MS,
|
||||
);
|
||||
this.tickTimer.unref?.();
|
||||
this.leaderRenewTimer.unref?.();
|
||||
|
||||
void this.safeTick();
|
||||
}
|
||||
|
||||
public onModuleDestroy(): void {
|
||||
if (this.tickTimer) clearInterval(this.tickTimer);
|
||||
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
||||
}
|
||||
|
||||
private async safeTick(): Promise<void> {
|
||||
if (this.tickInProgress) return;
|
||||
this.tickInProgress = true;
|
||||
try {
|
||||
await this.tick();
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Background refresh tick failed: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
} finally {
|
||||
this.tickInProgress = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async tick(): Promise<void> {
|
||||
if (!isInKstBackgroundWindow()) return;
|
||||
const isLeader = await this.tryBecomeLeader();
|
||||
if (!isLeader) return;
|
||||
|
||||
const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS);
|
||||
const candidates = await this.staleAccountRepository.findStaleAccounts({
|
||||
olderThan,
|
||||
limit: PER_TICK_CANDIDATE_LIMIT,
|
||||
});
|
||||
if (candidates.length === 0) return;
|
||||
|
||||
let processed = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (!isInKstBackgroundWindow()) break;
|
||||
if (await this.isInFailureBackoff(candidate.puuid)) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS);
|
||||
await this.enrichmentService.markRefreshRequested(
|
||||
candidate.userId,
|
||||
candidate.puuid,
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await this.enrichmentService.enrichRiotAccount({
|
||||
userId: candidate.userId,
|
||||
puuid: candidate.puuid,
|
||||
options: { force: false },
|
||||
});
|
||||
|
||||
if (this.isFailedEnrichment(result)) {
|
||||
failed += 1;
|
||||
await this.bumpFailureCounter(candidate.puuid);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.clearFailureCounter(candidate.puuid);
|
||||
processed += 1;
|
||||
} catch (err) {
|
||||
failed += 1;
|
||||
await this.bumpFailureCounter(candidate.puuid);
|
||||
this.logger.warn(
|
||||
`Background enrichRiotAccount failed for ${candidate.puuid}: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`,
|
||||
);
|
||||
}
|
||||
|
||||
private async tryBecomeLeader(): Promise<boolean> {
|
||||
const current = await this.garnet.getString(LEADER_KEY);
|
||||
if (current === this.consumer) {
|
||||
await this.garnet.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if (current) return false;
|
||||
|
||||
return this.garnet.setStringIfAbsent(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
}
|
||||
|
||||
private async renewLeaderIfHeld(): Promise<void> {
|
||||
try {
|
||||
const current = await this.garnet.getString(LEADER_KEY);
|
||||
if (current !== this.consumer) return;
|
||||
await this.garnet.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Background refresh leader renew failed: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async isInFailureBackoff(puuid: string): Promise<boolean> {
|
||||
const count = await this.getFailureCount(puuid);
|
||||
return count >= FAILURE_BACKOFF_THRESHOLD;
|
||||
}
|
||||
|
||||
private async getFailureCount(puuid: string): Promise<number> {
|
||||
const raw = await this.garnet.getString(this.failureBackoffKey(puuid));
|
||||
const count = Number(raw ?? 0);
|
||||
if (!Number.isFinite(count) || count < 0) return 0;
|
||||
return count;
|
||||
}
|
||||
|
||||
private async bumpFailureCounter(puuid: string): Promise<void> {
|
||||
const key = this.failureBackoffKey(puuid);
|
||||
const client = this.garnet.raw();
|
||||
const response = await client.sendCommand(['INCR', key]);
|
||||
await client.sendCommand([
|
||||
'EXPIRE',
|
||||
key,
|
||||
String(FAILURE_BACKOFF_TTL_SECONDS),
|
||||
]);
|
||||
|
||||
const count = Number(response ?? 0);
|
||||
if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) {
|
||||
this.logger.warn(
|
||||
`Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async clearFailureCounter(puuid: string): Promise<void> {
|
||||
await this.garnet.delete(this.failureBackoffKey(puuid));
|
||||
}
|
||||
|
||||
private isFailedEnrichment(result: EnrichResult): boolean {
|
||||
if (!result.account) return true;
|
||||
return [
|
||||
result.summoner,
|
||||
result.league,
|
||||
result.masteries,
|
||||
result.recentMatches,
|
||||
].every((status) => status === 'failed');
|
||||
}
|
||||
|
||||
private failureBackoffKey(puuid: string): string {
|
||||
return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { isInKstBackgroundWindow } from './kst-window.util';
|
||||
|
||||
describe('isInKstBackgroundWindow', () => {
|
||||
it('includes KST 03:00', () => {
|
||||
expect(isInKstBackgroundWindow(new Date('2026-05-02T18:00:00.000Z'))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('includes KST 09:59', () => {
|
||||
expect(isInKstBackgroundWindow(new Date('2026-05-03T00:59:59.000Z'))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('excludes KST 10:00', () => {
|
||||
expect(isInKstBackgroundWindow(new Date('2026-05-03T01:00:00.000Z'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('excludes KST 02:59', () => {
|
||||
expect(isInKstBackgroundWindow(new Date('2026-05-02T17:59:59.000Z'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
const BG_WINDOW_START_HOUR = 3;
|
||||
const BG_WINDOW_END_HOUR = 10;
|
||||
const KST_HOUR_FORMAT = new Intl.DateTimeFormat('en-US', {
|
||||
timeZone: 'Asia/Seoul',
|
||||
hour: '2-digit',
|
||||
hour12: false,
|
||||
});
|
||||
|
||||
export function isInKstBackgroundWindow(now: Date = new Date()): boolean {
|
||||
const hourString = KST_HOUR_FORMAT.format(now);
|
||||
const hour = Number.parseInt(hourString, 10);
|
||||
if (Number.isNaN(hour)) return false;
|
||||
|
||||
const normalizedHour = hour % 24;
|
||||
return (
|
||||
normalizedHour >= BG_WINDOW_START_HOUR &&
|
||||
normalizedHour < BG_WINDOW_END_HOUR
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import {
|
||||
buildStaleAccountsPipeline,
|
||||
StaleAccountRepository,
|
||||
} from './stale-account.repository';
|
||||
|
||||
describe('buildStaleAccountsPipeline', () => {
|
||||
const olderThan = new Date('2026-05-03T00:00:00.000Z');
|
||||
|
||||
it('filters to active users before unwinding Riot accounts', () => {
|
||||
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 });
|
||||
|
||||
expect(pipeline[0]).toEqual({ $match: { isActivated: true } });
|
||||
expect(pipeline[1]).toEqual({ $unwind: '$riotAccounts' });
|
||||
});
|
||||
|
||||
it('matches missing or older lastRefreshRequestedAt values', () => {
|
||||
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 });
|
||||
|
||||
expect(pipeline[2]).toEqual({
|
||||
$match: {
|
||||
$or: [
|
||||
{ 'riotAccounts.lastRefreshRequestedAt': { $exists: false } },
|
||||
{ 'riotAccounts.lastRefreshRequestedAt': { $lt: olderThan } },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('sorts missing refresh timestamps first', () => {
|
||||
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 });
|
||||
|
||||
expect(pipeline[3]).toEqual({
|
||||
$addFields: {
|
||||
'riotAccounts.lastRefreshRequestedAtSort': {
|
||||
$ifNull: ['$riotAccounts.lastRefreshRequestedAt', new Date(0)],
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(pipeline[4]).toEqual({
|
||||
$sort: { 'riotAccounts.lastRefreshRequestedAtSort': 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it('applies the provided limit', () => {
|
||||
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 7 });
|
||||
|
||||
expect(pipeline[5]).toEqual({ $limit: 7 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('StaleAccountRepository', () => {
|
||||
it('returns aggregate candidates from the user model', async () => {
|
||||
const userId = new mongoose.Types.ObjectId();
|
||||
const candidates = [{ userId, puuid: 'puuid-1' }];
|
||||
const exec = jest.fn().mockResolvedValue(candidates);
|
||||
const aggregate = jest.fn().mockReturnValue({ exec });
|
||||
const connectionService = {
|
||||
getModel: jest.fn().mockResolvedValue({ aggregate }),
|
||||
} as unknown as ConnectionService;
|
||||
|
||||
const repository = new StaleAccountRepository(connectionService);
|
||||
await expect(
|
||||
repository.findStaleAccounts({
|
||||
olderThan: new Date('2026-05-03T00:00:00.000Z'),
|
||||
limit: 1,
|
||||
}),
|
||||
).resolves.toBe(candidates);
|
||||
|
||||
expect(connectionService.getModel).toHaveBeenCalledWith('user');
|
||||
expect(aggregate).toHaveBeenCalledWith(
|
||||
buildStaleAccountsPipeline({
|
||||
olderThan: new Date('2026-05-03T00:00:00.000Z'),
|
||||
limit: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import mongoose, { type PipelineStage } from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
|
||||
export interface StaleAccountCandidate {
|
||||
userId: mongoose.Types.ObjectId;
|
||||
puuid: string;
|
||||
lastRefreshRequestedAt?: Date;
|
||||
}
|
||||
|
||||
export function buildStaleAccountsPipeline(opts: {
|
||||
olderThan: Date;
|
||||
limit: number;
|
||||
}): PipelineStage[] {
|
||||
return [
|
||||
{ $match: { isActivated: true } },
|
||||
{ $unwind: '$riotAccounts' },
|
||||
{
|
||||
$match: {
|
||||
$or: [
|
||||
{ 'riotAccounts.lastRefreshRequestedAt': { $exists: false } },
|
||||
{ 'riotAccounts.lastRefreshRequestedAt': { $lt: opts.olderThan } },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
$addFields: {
|
||||
'riotAccounts.lastRefreshRequestedAtSort': {
|
||||
$ifNull: ['$riotAccounts.lastRefreshRequestedAt', new Date(0)],
|
||||
},
|
||||
},
|
||||
},
|
||||
{ $sort: { 'riotAccounts.lastRefreshRequestedAtSort': 1 } },
|
||||
{ $limit: opts.limit },
|
||||
{
|
||||
$project: {
|
||||
_id: 0,
|
||||
userId: '$_id',
|
||||
puuid: '$riotAccounts.puuid',
|
||||
lastRefreshRequestedAt: '$riotAccounts.lastRefreshRequestedAt',
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class StaleAccountRepository {
|
||||
constructor(private readonly connectionService: ConnectionService) {}
|
||||
|
||||
public async findStaleAccounts(opts: {
|
||||
olderThan: Date;
|
||||
limit: number;
|
||||
}): Promise<StaleAccountCandidate[]> {
|
||||
const UserModel = await this.connectionService.getModel('user');
|
||||
return UserModel.aggregate<StaleAccountCandidate>(
|
||||
buildStaleAccountsPipeline(opts),
|
||||
).exec();
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,18 @@ export class RiotAccountEnrichmentService {
|
||||
private readonly recentStatsService: RiotRecentStatsService,
|
||||
) {}
|
||||
|
||||
public async markRefreshRequested(
|
||||
userId: mongoose.Types.ObjectId,
|
||||
puuid: string,
|
||||
at: Date = new Date(),
|
||||
): Promise<void> {
|
||||
const UserModel = await this.connectionService.getModel('user');
|
||||
await UserModel.updateOne(
|
||||
{ _id: userId, 'riotAccounts.puuid': puuid },
|
||||
{ $set: { 'riotAccounts.$.lastRefreshRequestedAt': at } },
|
||||
).exec();
|
||||
}
|
||||
|
||||
public async enrichRiotAccount(params: {
|
||||
userId: mongoose.Types.ObjectId;
|
||||
puuid: string;
|
||||
@@ -117,7 +129,8 @@ export class RiotAccountEnrichmentService {
|
||||
.exec();
|
||||
|
||||
const opts = params.options ?? {};
|
||||
const platform = opts.platform ?? community?.riotPlatform ?? DEFAULT_PLATFORM;
|
||||
const platform =
|
||||
opts.platform ?? community?.riotPlatform ?? DEFAULT_PLATFORM;
|
||||
const region = opts.region ?? community?.riotRegion ?? DEFAULT_REGION;
|
||||
const now = Date.now();
|
||||
|
||||
@@ -161,19 +174,19 @@ export class RiotAccountEnrichmentService {
|
||||
|
||||
const [leagueOutcome, masteryOutcome, recentMatchesOutcome] =
|
||||
await Promise.all([
|
||||
leagueStale
|
||||
? this.fetchLeagueSafe(params.puuid, effectivePlatform)
|
||||
: Promise.resolve({ status: 'fresh' as const, dto: null }),
|
||||
masteryStale
|
||||
? this.fetchMasteriesSafe(params.puuid, effectivePlatform)
|
||||
: Promise.resolve({ status: 'fresh' as const, dto: null }),
|
||||
recentMatchesStale
|
||||
? this.fetchRecentMatchesSafe(params.puuid, region, seasonStartedAt)
|
||||
: Promise.resolve({
|
||||
status: 'fresh' as const,
|
||||
summary: null,
|
||||
matchIds: null,
|
||||
}),
|
||||
leagueStale
|
||||
? this.fetchLeagueSafe(params.puuid, effectivePlatform)
|
||||
: Promise.resolve({ status: 'fresh' as const, dto: null }),
|
||||
masteryStale
|
||||
? this.fetchMasteriesSafe(params.puuid, effectivePlatform)
|
||||
: Promise.resolve({ status: 'fresh' as const, dto: null }),
|
||||
recentMatchesStale
|
||||
? this.fetchRecentMatchesSafe(params.puuid, region, seasonStartedAt)
|
||||
: Promise.resolve({
|
||||
status: 'fresh' as const,
|
||||
summary: null,
|
||||
matchIds: null,
|
||||
}),
|
||||
]);
|
||||
|
||||
if (summonerOutcome.status === 'updated' && summonerOutcome.dto) {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { GarnetService } from '../global/garnet/garnet.service';
|
||||
import { RiotRateLimiterService } from './riot-rate-limiter.service';
|
||||
|
||||
function makeService(sendCommand = jest.fn().mockResolvedValue(0)) {
|
||||
const garnet = {
|
||||
raw: jest.fn().mockReturnValue({ sendCommand }),
|
||||
} as unknown as GarnetService;
|
||||
return {
|
||||
service: new RiotRateLimiterService(garnet),
|
||||
sendCommand,
|
||||
};
|
||||
}
|
||||
|
||||
describe('RiotRateLimiterService background limiter', () => {
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('uses the background key namespace and passes cost to the Lua script', async () => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const { service, sendCommand } = makeService();
|
||||
|
||||
await service.acquireBackground(8);
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledTimes(1);
|
||||
const command = sendCommand.mock.calls[0][0] as string[];
|
||||
expect(command[0]).toBe('EVAL');
|
||||
expect(command[2]).toBe('2');
|
||||
expect(command[3]).toBe('riot:bg:rl:1000');
|
||||
expect(command[4]).toBe('riot:bg:rl:120000');
|
||||
expect(command[5]).toBe('1000');
|
||||
expect(command[7]).toBe('8');
|
||||
});
|
||||
|
||||
it('reads cleaned background usage for both windows', async () => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(200_000);
|
||||
const sendCommand = jest
|
||||
.fn()
|
||||
.mockImplementation(async (command: string[]) => {
|
||||
if (command[0] === 'ZREMRANGEBYSCORE') return 0;
|
||||
if (command[1] === 'riot:bg:rl:1000') return 3;
|
||||
if (command[1] === 'riot:bg:rl:120000') return 24;
|
||||
return 0;
|
||||
});
|
||||
const { service } = makeService(sendCommand);
|
||||
|
||||
await expect(service.getBackgroundUsage()).resolves.toEqual({
|
||||
used1s: 3,
|
||||
used2m: 24,
|
||||
});
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'ZREMRANGEBYSCORE',
|
||||
'riot:bg:rl:1000',
|
||||
'-inf',
|
||||
'199000',
|
||||
]);
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'ZREMRANGEBYSCORE',
|
||||
'riot:bg:rl:120000',
|
||||
'-inf',
|
||||
'80000',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -7,21 +7,44 @@ const APP_LIMITS = [
|
||||
{ windowMs: 120_000, max: 90 },
|
||||
] as const;
|
||||
|
||||
const BACKGROUND_LIMITS = [
|
||||
{ windowMs: 1_000, max: 6 },
|
||||
{ windowMs: 120_000, max: 36 },
|
||||
] as const;
|
||||
const BACKGROUND_KEY_PREFIX = 'riot:bg:rl';
|
||||
|
||||
const ACQUIRE_SCRIPT = `
|
||||
local now = tonumber(ARGV[1])
|
||||
local member = ARGV[2]
|
||||
local memberBase = ARGV[2]
|
||||
local cost = tonumber(ARGV[3])
|
||||
if cost == nil or cost < 1 then
|
||||
cost = 1
|
||||
end
|
||||
local maxWait = 0
|
||||
local effectiveCosts = {}
|
||||
|
||||
for i = 1, #KEYS do
|
||||
local argIndex = 3 + ((i - 1) * 2)
|
||||
local argIndex = 4 + ((i - 1) * 2)
|
||||
local windowMs = tonumber(ARGV[argIndex])
|
||||
local limit = tonumber(ARGV[argIndex + 1])
|
||||
local key = KEYS[i]
|
||||
local effectiveCost = cost
|
||||
if effectiveCost > limit then
|
||||
effectiveCost = limit
|
||||
end
|
||||
effectiveCosts[i] = effectiveCost
|
||||
|
||||
redis.call('ZREMRANGEBYSCORE', key, '-inf', now - windowMs)
|
||||
local count = redis.call('ZCARD', key)
|
||||
if count >= limit then
|
||||
local oldest = redis.call('ZRANGE', key, 0, 0, 'WITHSCORES')
|
||||
local overflow = count + effectiveCost - limit
|
||||
if overflow > 0 then
|
||||
local oldest = redis.call(
|
||||
'ZRANGE',
|
||||
key,
|
||||
overflow - 1,
|
||||
overflow - 1,
|
||||
'WITHSCORES'
|
||||
)
|
||||
local waitMs = windowMs
|
||||
if oldest[2] then
|
||||
waitMs = windowMs - (now - tonumber(oldest[2])) + 1
|
||||
@@ -37,10 +60,13 @@ if maxWait > 0 then
|
||||
end
|
||||
|
||||
for i = 1, #KEYS do
|
||||
local argIndex = 3 + ((i - 1) * 2)
|
||||
local argIndex = 4 + ((i - 1) * 2)
|
||||
local windowMs = tonumber(ARGV[argIndex])
|
||||
local key = KEYS[i]
|
||||
redis.call('ZADD', key, now, member)
|
||||
local effectiveCost = effectiveCosts[i] or cost
|
||||
for c = 1, effectiveCost do
|
||||
redis.call('ZADD', key, now, memberBase .. ':' .. i .. ':' .. c)
|
||||
end
|
||||
redis.call('PEXPIRE', key, windowMs)
|
||||
end
|
||||
|
||||
@@ -58,7 +84,7 @@ export class RiotRateLimiterService {
|
||||
): Promise<void> {
|
||||
for (;;) {
|
||||
const now = Date.now();
|
||||
const waitMs = await this.checkAndAdd(route, now);
|
||||
const waitMs = await this.checkAndAdd(route, now, 1);
|
||||
if (waitMs <= 0) return;
|
||||
|
||||
if (waitMs >= 100) {
|
||||
@@ -70,22 +96,83 @@ export class RiotRateLimiterService {
|
||||
}
|
||||
}
|
||||
|
||||
public async acquireBackground(cost = 1): Promise<void> {
|
||||
const normalizedCost = normalizeCost(cost);
|
||||
for (;;) {
|
||||
const now = Date.now();
|
||||
const waitMs = await this.checkAndAddBackground(now, normalizedCost);
|
||||
if (waitMs <= 0) return;
|
||||
|
||||
if (waitMs >= 100) {
|
||||
this.logger.warn(
|
||||
`Background Riot throttle waiting ${waitMs}ms (cost=${normalizedCost}).`,
|
||||
);
|
||||
}
|
||||
await sleep(waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
public async getBackgroundUsage(): Promise<{
|
||||
used1s: number;
|
||||
used2m: number;
|
||||
}> {
|
||||
const client = this.garnet.raw();
|
||||
const now = Date.now();
|
||||
const usages = await Promise.all(
|
||||
BACKGROUND_LIMITS.map(async (limit) => {
|
||||
const key = `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`;
|
||||
await client.sendCommand([
|
||||
'ZREMRANGEBYSCORE',
|
||||
key,
|
||||
'-inf',
|
||||
String(now - limit.windowMs),
|
||||
]);
|
||||
const response = await client.sendCommand(['ZCARD', key]);
|
||||
const count = Number(response ?? 0);
|
||||
return Number.isFinite(count) && count > 0 ? count : 0;
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
used1s: usages[0] ?? 0,
|
||||
used2m: usages[1] ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
private async checkAndAdd(
|
||||
route: RiotRegionalRoute | RiotPlatformRoute,
|
||||
now: number,
|
||||
cost: number,
|
||||
): Promise<number> {
|
||||
const client = this.garnet.raw();
|
||||
const keys = APP_LIMITS.map(
|
||||
(limit) => `riot:rl:${route}:${limit.windowMs}`,
|
||||
);
|
||||
return this.checkAndAddWithLimits(keys, APP_LIMITS, now, cost);
|
||||
}
|
||||
|
||||
private async checkAndAddBackground(
|
||||
now: number,
|
||||
cost: number,
|
||||
): Promise<number> {
|
||||
const keys = BACKGROUND_LIMITS.map(
|
||||
(limit) => `${BACKGROUND_KEY_PREFIX}:${limit.windowMs}`,
|
||||
);
|
||||
return this.checkAndAddWithLimits(keys, BACKGROUND_LIMITS, now, cost);
|
||||
}
|
||||
|
||||
private async checkAndAddWithLimits(
|
||||
keys: string[],
|
||||
limits: ReadonlyArray<{ windowMs: number; max: number }>,
|
||||
now: number,
|
||||
cost: number,
|
||||
): Promise<number> {
|
||||
const client = this.garnet.raw();
|
||||
const member = `${process.pid}:${now}:${Math.random().toString(36).slice(2)}`;
|
||||
const argumentsList = [
|
||||
String(now),
|
||||
member,
|
||||
...APP_LIMITS.flatMap((limit) => [
|
||||
String(limit.windowMs),
|
||||
String(limit.max),
|
||||
]),
|
||||
String(normalizeCost(cost)),
|
||||
...limits.flatMap((limit) => [String(limit.windowMs), String(limit.max)]),
|
||||
];
|
||||
|
||||
const response = await client.sendCommand([
|
||||
@@ -102,6 +189,11 @@ export class RiotRateLimiterService {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeCost(cost: number): number {
|
||||
if (!Number.isFinite(cost)) return 1;
|
||||
return Math.max(1, Math.floor(cost));
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
Reference in new issue
Block a user