feat(api): add background refresh

This commit is contained in:
dahyeon committed 2026-04-19 01:29:00 +09:00
1 parent 4bc2df7bcc
commit 7e78b1f92d
7 files changed
+614

No files matched your search

@@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { RiotModule } from '../riot/riot.module';
import { BackgroundRefreshScheduler } from './background-refresh.scheduler';
import { StaleAccountRepository } from './stale-account.repository';
@Module({
imports: [ConfigModule, RiotModule],
providers: [BackgroundRefreshScheduler, StaleAccountRepository],
})
export class BackgroundRefreshModule {}
@@ -0,0 +1,186 @@
import { ConfigService } from '@nestjs/config';
import mongoose from 'mongoose';
import { GarnetService } from '../global/garnet/garnet.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
import { BackgroundRefreshScheduler } from './background-refresh.scheduler';
import {
StaleAccountCandidate,
StaleAccountRepository,
} from './stale-account.repository';
type SchedulerInternals = BackgroundRefreshScheduler & {
tick(): Promise<void>;
};
function successResult() {
return {
summoner: 'fresh',
league: 'fresh',
masteries: 'fresh',
recentMatches: 'fresh',
account: {},
};
}
function candidate(puuid: string): StaleAccountCandidate {
return {
userId: new mongoose.Types.ObjectId(),
puuid,
};
}
function makeScheduler(enabled = true) {
const rawClient = {
sendCommand: jest.fn().mockResolvedValue(1),
};
const garnet = {
getString: jest.fn().mockResolvedValue(null),
setString: jest.fn().mockResolvedValue(undefined),
setStringIfAbsent: jest.fn().mockResolvedValue(true),
delete: jest.fn().mockResolvedValue(1),
raw: jest.fn().mockReturnValue(rawClient),
} as unknown as jest.Mocked<GarnetService> & { raw: jest.Mock };
const rateLimiter = {
acquireBackground: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RiotRateLimiterService>;
const enrichmentService = {
markRefreshRequested: jest.fn().mockResolvedValue(undefined),
enrichRiotAccount: jest.fn().mockResolvedValue(successResult()),
} as unknown as jest.Mocked<RiotAccountEnrichmentService>;
const staleAccountRepository = {
findStaleAccounts: jest.fn().mockResolvedValue([]),
} as unknown as jest.Mocked<StaleAccountRepository>;
const config = {
get: jest.fn().mockReturnValue(enabled ? 'true' : 'false'),
} as unknown as ConfigService;
const scheduler = new BackgroundRefreshScheduler(
garnet,
rateLimiter,
enrichmentService,
staleAccountRepository,
config,
) as SchedulerInternals;
return {
scheduler,
garnet,
rawClient,
rateLimiter,
enrichmentService,
staleAccountRepository,
};
}
describe('BackgroundRefreshScheduler', () => {
afterEach(() => {
jest.useRealTimers();
jest.restoreAllMocks();
});
it('does not schedule ticks when disabled', () => {
jest.useFakeTimers();
const setIntervalSpy = jest.spyOn(global, 'setInterval');
const { scheduler } = makeScheduler(false);
scheduler.onModuleInit();
expect(setIntervalSpy).not.toHaveBeenCalled();
});
it('does not query stale accounts outside the KST window', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z'));
const { scheduler, staleAccountRepository } = makeScheduler();
await scheduler.tick();
expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled();
});
it('does not query stale accounts when another leader holds the lock', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { scheduler, garnet, staleAccountRepository } = makeScheduler();
garnet.getString.mockResolvedValue('another-consumer');
await scheduler.tick();
expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled();
});
it('marks and enriches each stale candidate with background rate tokens', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const {
scheduler,
rateLimiter,
enrichmentService,
staleAccountRepository,
} = makeScheduler();
const candidates = [candidate('puuid-1'), candidate('puuid-2')];
staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates);
await scheduler.tick();
expect(rateLimiter.acquireBackground).toHaveBeenCalledTimes(2);
expect(rateLimiter.acquireBackground).toHaveBeenCalledWith(8);
expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2);
expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith(
1,
candidates[0].userId,
'puuid-1',
);
expect(enrichmentService.enrichRiotAccount).toHaveBeenNthCalledWith(2, {
userId: candidates[1].userId,
puuid: 'puuid-2',
options: { force: false },
});
});
it('continues after a candidate throws and records a failure counter', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { scheduler, rawClient, enrichmentService, staleAccountRepository } =
makeScheduler();
const candidates = [candidate('puuid-1'), candidate('puuid-2')];
staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates);
enrichmentService.enrichRiotAccount
.mockRejectedValueOnce(new Error('boom'))
.mockResolvedValueOnce(successResult() as never);
await scheduler.tick();
expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(2);
expect(rawClient.sendCommand).toHaveBeenCalledWith([
'INCR',
'riot:bg:fail:puuid-1',
]);
expect(rawClient.sendCommand).toHaveBeenCalledWith([
'EXPIRE',
'riot:bg:fail:puuid-1',
'86400',
]);
});
it('skips candidates already in failure backoff', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const {
scheduler,
garnet,
rateLimiter,
enrichmentService,
staleAccountRepository,
} = makeScheduler();
staleAccountRepository.findStaleAccounts.mockResolvedValue([
candidate('skip-me'),
]);
garnet.getString.mockImplementation(async (key: string) => {
if (key.includes('riot:bg:fail:skip-me')) return '3';
return null;
});
await scheduler.tick();
expect(rateLimiter.acquireBackground).not.toHaveBeenCalled();
expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled();
expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,233 @@
import {
Injectable,
Logger,
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import {
type EnrichResult,
RiotAccountEnrichmentService,
} from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
import { GarnetService } from '../global/garnet/garnet.service';
import { isInKstBackgroundWindow } from './kst-window.util';
import { StaleAccountRepository } from './stale-account.repository';
const TICK_INTERVAL_MS = 60_000;
const LEADER_KEY = 'riot:bg-refresh:leader';
const LEADER_TTL_SECONDS = 90;
const LEADER_RENEW_INTERVAL_MS = 30_000;
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
const PER_TICK_CANDIDATE_LIMIT = 20;
const ENRICH_COST_TOKENS = 8;
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
const FAILURE_BACKOFF_THRESHOLD = 3;
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
@Injectable()
export class BackgroundRefreshScheduler
implements OnModuleInit, OnModuleDestroy
{
private readonly logger = new Logger(BackgroundRefreshScheduler.name);
private tickTimer?: NodeJS.Timeout;
private leaderRenewTimer?: NodeJS.Timeout;
private tickInProgress = false;
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private readonly enabled: boolean;
constructor(
private readonly garnet: GarnetService,
private readonly rateLimiter: RiotRateLimiterService,
private readonly enrichmentService: RiotAccountEnrichmentService,
private readonly staleAccountRepository: StaleAccountRepository,
config: ConfigService,
) {
this.enabled = config.get<string>('RIOT_BG_REFRESH_ENABLED') === 'true';
}
public onModuleInit(): void {
if (!this.enabled) {
this.logger.log(
'Background Riot refresh is disabled (RIOT_BG_REFRESH_ENABLED).',
);
return;
}
this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS);
this.leaderRenewTimer = setInterval(
() => void this.renewLeaderIfHeld(),
LEADER_RENEW_INTERVAL_MS,
);
this.tickTimer.unref?.();
this.leaderRenewTimer.unref?.();
void this.safeTick();
}
public onModuleDestroy(): void {
if (this.tickTimer) clearInterval(this.tickTimer);
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
}
private async safeTick(): Promise<void> {
if (this.tickInProgress) return;
this.tickInProgress = true;
try {
await this.tick();
} catch (err) {
this.logger.warn(
`Background refresh tick failed: ${
err instanceof Error ? err.message : err
}`,
);
} finally {
this.tickInProgress = false;
}
}
private async tick(): Promise<void> {
if (!isInKstBackgroundWindow()) return;
const isLeader = await this.tryBecomeLeader();
if (!isLeader) return;
const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS);
const candidates = await this.staleAccountRepository.findStaleAccounts({
olderThan,
limit: PER_TICK_CANDIDATE_LIMIT,
});
if (candidates.length === 0) return;
let processed = 0;
let failed = 0;
let skipped = 0;
for (const candidate of candidates) {
if (!isInKstBackgroundWindow()) break;
if (await this.isInFailureBackoff(candidate.puuid)) {
skipped += 1;
continue;
}
await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS);
await this.enrichmentService.markRefreshRequested(
candidate.userId,
candidate.puuid,
);
try {
const result = await this.enrichmentService.enrichRiotAccount({
userId: candidate.userId,
puuid: candidate.puuid,
options: { force: false },
});
if (this.isFailedEnrichment(result)) {
failed += 1;
await this.bumpFailureCounter(candidate.puuid);
continue;
}
await this.clearFailureCounter(candidate.puuid);
processed += 1;
} catch (err) {
failed += 1;
await this.bumpFailureCounter(candidate.puuid);
this.logger.warn(
`Background enrichRiotAccount failed for ${candidate.puuid}: ${
err instanceof Error ? err.message : err
}`,
);
}
}
this.logger.debug(
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`,
);
}
private async tryBecomeLeader(): Promise<boolean> {
const current = await this.garnet.getString(LEADER_KEY);
if (current === this.consumer) {
await this.garnet.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
return true;
}
if (current) return false;
return this.garnet.setStringIfAbsent(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
}
private async renewLeaderIfHeld(): Promise<void> {
try {
const current = await this.garnet.getString(LEADER_KEY);
if (current !== this.consumer) return;
await this.garnet.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
} catch (err) {
this.logger.warn(
`Background refresh leader renew failed: ${
err instanceof Error ? err.message : err
}`,
);
}
}
private async isInFailureBackoff(puuid: string): Promise<boolean> {
const count = await this.getFailureCount(puuid);
return count >= FAILURE_BACKOFF_THRESHOLD;
}
private async getFailureCount(puuid: string): Promise<number> {
const raw = await this.garnet.getString(this.failureBackoffKey(puuid));
const count = Number(raw ?? 0);
if (!Number.isFinite(count) || count < 0) return 0;
return count;
}
private async bumpFailureCounter(puuid: string): Promise<void> {
const key = this.failureBackoffKey(puuid);
const client = this.garnet.raw();
const response = await client.sendCommand(['INCR', key]);
await client.sendCommand([
'EXPIRE',
key,
String(FAILURE_BACKOFF_TTL_SECONDS),
]);
const count = Number(response ?? 0);
if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) {
this.logger.warn(
`Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`,
);
}
}
private async clearFailureCounter(puuid: string): Promise<void> {
await this.garnet.delete(this.failureBackoffKey(puuid));
}
private isFailedEnrichment(result: EnrichResult): boolean {
if (!result.account) return true;
return [
result.summoner,
result.league,
result.masteries,
result.recentMatches,
].every((status) => status === 'failed');
}
private failureBackoffKey(puuid: string): string {
return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`;
}
}
@@ -0,0 +1,27 @@
import { isInKstBackgroundWindow } from './kst-window.util';
describe('isInKstBackgroundWindow', () => {
it('includes KST 03:00', () => {
expect(isInKstBackgroundWindow(new Date('2026-05-02T18:00:00.000Z'))).toBe(
true,
);
});
it('includes KST 09:59', () => {
expect(isInKstBackgroundWindow(new Date('2026-05-03T00:59:59.000Z'))).toBe(
true,
);
});
it('excludes KST 10:00', () => {
expect(isInKstBackgroundWindow(new Date('2026-05-03T01:00:00.000Z'))).toBe(
false,
);
});
it('excludes KST 02:59', () => {
expect(isInKstBackgroundWindow(new Date('2026-05-02T17:59:59.000Z'))).toBe(
false,
);
});
});
@@ -0,0 +1,19 @@
const BG_WINDOW_START_HOUR = 3;
const BG_WINDOW_END_HOUR = 10;
const KST_HOUR_FORMAT = new Intl.DateTimeFormat('en-US', {
timeZone: 'Asia/Seoul',
hour: '2-digit',
hour12: false,
});
export function isInKstBackgroundWindow(now: Date = new Date()): boolean {
const hourString = KST_HOUR_FORMAT.format(now);
const hour = Number.parseInt(hourString, 10);
if (Number.isNaN(hour)) return false;
const normalizedHour = hour % 24;
return (
normalizedHour >= BG_WINDOW_START_HOUR &&
normalizedHour < BG_WINDOW_END_HOUR
);
}
@@ -0,0 +1,79 @@
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import {
buildStaleAccountsPipeline,
StaleAccountRepository,
} from './stale-account.repository';
describe('buildStaleAccountsPipeline', () => {
const olderThan = new Date('2026-05-03T00:00:00.000Z');
it('filters to active users before unwinding Riot accounts', () => {
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 });
expect(pipeline[0]).toEqual({ $match: { isActivated: true } });
expect(pipeline[1]).toEqual({ $unwind: '$riotAccounts' });
});
it('matches missing or older lastRefreshRequestedAt values', () => {
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 });
expect(pipeline[2]).toEqual({
$match: {
$or: [
{ 'riotAccounts.lastRefreshRequestedAt': { $exists: false } },
{ 'riotAccounts.lastRefreshRequestedAt': { $lt: olderThan } },
],
},
});
});
it('sorts missing refresh timestamps first', () => {
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 });
expect(pipeline[3]).toEqual({
$addFields: {
'riotAccounts.lastRefreshRequestedAtSort': {
$ifNull: ['$riotAccounts.lastRefreshRequestedAt', new Date(0)],
},
},
});
expect(pipeline[4]).toEqual({
$sort: { 'riotAccounts.lastRefreshRequestedAtSort': 1 },
});
});
it('applies the provided limit', () => {
const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 7 });
expect(pipeline[5]).toEqual({ $limit: 7 });
});
});
describe('StaleAccountRepository', () => {
it('returns aggregate candidates from the user model', async () => {
const userId = new mongoose.Types.ObjectId();
const candidates = [{ userId, puuid: 'puuid-1' }];
const exec = jest.fn().mockResolvedValue(candidates);
const aggregate = jest.fn().mockReturnValue({ exec });
const connectionService = {
getModel: jest.fn().mockResolvedValue({ aggregate }),
} as unknown as ConnectionService;
const repository = new StaleAccountRepository(connectionService);
await expect(
repository.findStaleAccounts({
olderThan: new Date('2026-05-03T00:00:00.000Z'),
limit: 1,
}),
).resolves.toBe(candidates);
expect(connectionService.getModel).toHaveBeenCalledWith('user');
expect(aggregate).toHaveBeenCalledWith(
buildStaleAccountsPipeline({
olderThan: new Date('2026-05-03T00:00:00.000Z'),
limit: 1,
}),
);
});
});
@@ -0,0 +1,59 @@
import { Injectable } from '@nestjs/common';
import mongoose, { type PipelineStage } from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
export interface StaleAccountCandidate {
userId: mongoose.Types.ObjectId;
puuid: string;
lastRefreshRequestedAt?: Date;
}
export function buildStaleAccountsPipeline(opts: {
olderThan: Date;
limit: number;
}): PipelineStage[] {
return [
{ $match: { isActivated: true } },
{ $unwind: '$riotAccounts' },
{
$match: {
$or: [
{ 'riotAccounts.lastRefreshRequestedAt': { $exists: false } },
{ 'riotAccounts.lastRefreshRequestedAt': { $lt: opts.olderThan } },
],
},
},
{
$addFields: {
'riotAccounts.lastRefreshRequestedAtSort': {
$ifNull: ['$riotAccounts.lastRefreshRequestedAt', new Date(0)],
},
},
},
{ $sort: { 'riotAccounts.lastRefreshRequestedAtSort': 1 } },
{ $limit: opts.limit },
{
$project: {
_id: 0,
userId: '$_id',
puuid: '$riotAccounts.puuid',
lastRefreshRequestedAt: '$riotAccounts.lastRefreshRequestedAt',
},
},
];
}
@Injectable()
export class StaleAccountRepository {
constructor(private readonly connectionService: ConnectionService) {}
public async findStaleAccounts(opts: {
olderThan: Date;
limit: number;
}): Promise<StaleAccountCandidate[]> {
const UserModel = await this.connectionService.getModel('user');
return UserModel.aggregate<StaleAccountCandidate>(
buildStaleAccountsPipeline(opts),
).exec();
}
}