From 7e78b1f92d65fd069950a213567fa32c0473cc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=95=EB=8B=A4=ED=98=84=20=28Dahyeon=20Park=29?= Date: Sun, 19 Apr 2026 01:29:00 +0900 Subject: [PATCH] feat(api): add background refresh --- .../background-refresh.module.ts | 11 + .../background-refresh.scheduler.spec.ts | 186 ++++++++++++++ .../background-refresh.scheduler.ts | 233 ++++++++++++++++++ .../kst-window.util.spec.ts | 27 ++ .../kst-window.util.ts | 19 ++ .../stale-account.repository.spec.ts | 79 ++++++ .../stale-account.repository.ts | 59 +++++ 7 files changed, 614 insertions(+) create mode 100644 apps/api/src/modules/riot-background-refresh/background-refresh.module.ts create mode 100644 apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts create mode 100644 apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts create mode 100644 apps/api/src/modules/riot-background-refresh/kst-window.util.spec.ts create mode 100644 apps/api/src/modules/riot-background-refresh/kst-window.util.ts create mode 100644 apps/api/src/modules/riot-background-refresh/stale-account.repository.spec.ts create mode 100644 apps/api/src/modules/riot-background-refresh/stale-account.repository.ts diff --git a/apps/api/src/modules/riot-background-refresh/background-refresh.module.ts b/apps/api/src/modules/riot-background-refresh/background-refresh.module.ts new file mode 100644 index 00000000..2b212a95 --- /dev/null +++ b/apps/api/src/modules/riot-background-refresh/background-refresh.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { RiotModule } from '../riot/riot.module'; +import { BackgroundRefreshScheduler } from './background-refresh.scheduler'; +import { StaleAccountRepository } from './stale-account.repository'; + +@Module({ + imports: [ConfigModule, RiotModule], + providers: [BackgroundRefreshScheduler, StaleAccountRepository], +}) +export class BackgroundRefreshModule {} diff --git a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts new file mode 100644 index 00000000..01b88226 --- /dev/null +++ b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts @@ -0,0 +1,186 @@ +import { ConfigService } from '@nestjs/config'; +import mongoose from 'mongoose'; +import { GarnetService } from '../global/garnet/garnet.service'; +import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service'; +import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service'; +import { BackgroundRefreshScheduler } from './background-refresh.scheduler'; +import { + StaleAccountCandidate, + StaleAccountRepository, +} from './stale-account.repository'; + +type SchedulerInternals = BackgroundRefreshScheduler & { + tick(): Promise; +}; + +function successResult() { + return { + summoner: 'fresh', + league: 'fresh', + masteries: 'fresh', + recentMatches: 'fresh', + account: {}, + }; +} + +function candidate(puuid: string): StaleAccountCandidate { + return { + userId: new mongoose.Types.ObjectId(), + puuid, + }; +} + +function makeScheduler(enabled = true) { + const rawClient = { + sendCommand: jest.fn().mockResolvedValue(1), + }; + const garnet = { + getString: jest.fn().mockResolvedValue(null), + setString: jest.fn().mockResolvedValue(undefined), + setStringIfAbsent: jest.fn().mockResolvedValue(true), + delete: jest.fn().mockResolvedValue(1), + raw: jest.fn().mockReturnValue(rawClient), + } as unknown as jest.Mocked & { raw: jest.Mock }; + const rateLimiter = { + acquireBackground: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + const enrichmentService = { + markRefreshRequested: jest.fn().mockResolvedValue(undefined), + enrichRiotAccount: jest.fn().mockResolvedValue(successResult()), + } as unknown as jest.Mocked; + const staleAccountRepository = { + findStaleAccounts: jest.fn().mockResolvedValue([]), + } as unknown as jest.Mocked; + const config = { + get: jest.fn().mockReturnValue(enabled ? 'true' : 'false'), + } as unknown as ConfigService; + + const scheduler = new BackgroundRefreshScheduler( + garnet, + rateLimiter, + enrichmentService, + staleAccountRepository, + config, + ) as SchedulerInternals; + + return { + scheduler, + garnet, + rawClient, + rateLimiter, + enrichmentService, + staleAccountRepository, + }; +} + +describe('BackgroundRefreshScheduler', () => { + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('does not schedule ticks when disabled', () => { + jest.useFakeTimers(); + const setIntervalSpy = jest.spyOn(global, 'setInterval'); + const { scheduler } = makeScheduler(false); + + scheduler.onModuleInit(); + + expect(setIntervalSpy).not.toHaveBeenCalled(); + }); + + it('does not query stale accounts outside the KST window', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T17:59:59.000Z')); + const { scheduler, staleAccountRepository } = makeScheduler(); + + await scheduler.tick(); + + expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled(); + }); + + it('does not query stale accounts when another leader holds the lock', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); + const { scheduler, garnet, staleAccountRepository } = makeScheduler(); + garnet.getString.mockResolvedValue('another-consumer'); + + await scheduler.tick(); + + expect(staleAccountRepository.findStaleAccounts).not.toHaveBeenCalled(); + }); + + it('marks and enriches each stale candidate with background rate tokens', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); + const { + scheduler, + rateLimiter, + enrichmentService, + staleAccountRepository, + } = makeScheduler(); + const candidates = [candidate('puuid-1'), candidate('puuid-2')]; + staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates); + + await scheduler.tick(); + + expect(rateLimiter.acquireBackground).toHaveBeenCalledTimes(2); + expect(rateLimiter.acquireBackground).toHaveBeenCalledWith(8); + expect(enrichmentService.markRefreshRequested).toHaveBeenCalledTimes(2); + expect(enrichmentService.markRefreshRequested).toHaveBeenNthCalledWith( + 1, + candidates[0].userId, + 'puuid-1', + ); + expect(enrichmentService.enrichRiotAccount).toHaveBeenNthCalledWith(2, { + userId: candidates[1].userId, + puuid: 'puuid-2', + options: { force: false }, + }); + }); + + it('continues after a candidate throws and records a failure counter', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); + const { scheduler, rawClient, enrichmentService, staleAccountRepository } = + makeScheduler(); + const candidates = [candidate('puuid-1'), candidate('puuid-2')]; + staleAccountRepository.findStaleAccounts.mockResolvedValue(candidates); + enrichmentService.enrichRiotAccount + .mockRejectedValueOnce(new Error('boom')) + .mockResolvedValueOnce(successResult() as never); + + await scheduler.tick(); + + expect(enrichmentService.enrichRiotAccount).toHaveBeenCalledTimes(2); + expect(rawClient.sendCommand).toHaveBeenCalledWith([ + 'INCR', + 'riot:bg:fail:puuid-1', + ]); + expect(rawClient.sendCommand).toHaveBeenCalledWith([ + 'EXPIRE', + 'riot:bg:fail:puuid-1', + '86400', + ]); + }); + + it('skips candidates already in failure backoff', async () => { + jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); + const { + scheduler, + garnet, + rateLimiter, + enrichmentService, + staleAccountRepository, + } = makeScheduler(); + staleAccountRepository.findStaleAccounts.mockResolvedValue([ + candidate('skip-me'), + ]); + garnet.getString.mockImplementation(async (key: string) => { + if (key.includes('riot:bg:fail:skip-me')) return '3'; + return null; + }); + + await scheduler.tick(); + + expect(rateLimiter.acquireBackground).not.toHaveBeenCalled(); + expect(enrichmentService.markRefreshRequested).not.toHaveBeenCalled(); + expect(enrichmentService.enrichRiotAccount).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts new file mode 100644 index 00000000..730f3e1d --- /dev/null +++ b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts @@ -0,0 +1,233 @@ +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { + type EnrichResult, + RiotAccountEnrichmentService, +} from '../riot/riot-account-enrichment.service'; +import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service'; +import { GarnetService } from '../global/garnet/garnet.service'; +import { isInKstBackgroundWindow } from './kst-window.util'; +import { StaleAccountRepository } from './stale-account.repository'; + +const TICK_INTERVAL_MS = 60_000; +const LEADER_KEY = 'riot:bg-refresh:leader'; +const LEADER_TTL_SECONDS = 90; +const LEADER_RENEW_INTERVAL_MS = 30_000; +const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000; +const PER_TICK_CANDIDATE_LIMIT = 20; +const ENRICH_COST_TOKENS = 8; +const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail'; +const FAILURE_BACKOFF_THRESHOLD = 3; +const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60; + +@Injectable() +export class BackgroundRefreshScheduler + implements OnModuleInit, OnModuleDestroy +{ + private readonly logger = new Logger(BackgroundRefreshScheduler.name); + private tickTimer?: NodeJS.Timeout; + private leaderRenewTimer?: NodeJS.Timeout; + private tickInProgress = false; + private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`; + private readonly enabled: boolean; + + constructor( + private readonly garnet: GarnetService, + private readonly rateLimiter: RiotRateLimiterService, + private readonly enrichmentService: RiotAccountEnrichmentService, + private readonly staleAccountRepository: StaleAccountRepository, + config: ConfigService, + ) { + this.enabled = config.get('RIOT_BG_REFRESH_ENABLED') === 'true'; + } + + public onModuleInit(): void { + if (!this.enabled) { + this.logger.log( + 'Background Riot refresh is disabled (RIOT_BG_REFRESH_ENABLED).', + ); + return; + } + + this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS); + this.leaderRenewTimer = setInterval( + () => void this.renewLeaderIfHeld(), + LEADER_RENEW_INTERVAL_MS, + ); + this.tickTimer.unref?.(); + this.leaderRenewTimer.unref?.(); + + void this.safeTick(); + } + + public onModuleDestroy(): void { + if (this.tickTimer) clearInterval(this.tickTimer); + if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer); + } + + private async safeTick(): Promise { + if (this.tickInProgress) return; + this.tickInProgress = true; + try { + await this.tick(); + } catch (err) { + this.logger.warn( + `Background refresh tick failed: ${ + err instanceof Error ? err.message : err + }`, + ); + } finally { + this.tickInProgress = false; + } + } + + private async tick(): Promise { + if (!isInKstBackgroundWindow()) return; + const isLeader = await this.tryBecomeLeader(); + if (!isLeader) return; + + const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS); + const candidates = await this.staleAccountRepository.findStaleAccounts({ + olderThan, + limit: PER_TICK_CANDIDATE_LIMIT, + }); + if (candidates.length === 0) return; + + let processed = 0; + let failed = 0; + let skipped = 0; + + for (const candidate of candidates) { + if (!isInKstBackgroundWindow()) break; + if (await this.isInFailureBackoff(candidate.puuid)) { + skipped += 1; + continue; + } + + await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS); + await this.enrichmentService.markRefreshRequested( + candidate.userId, + candidate.puuid, + ); + + try { + const result = await this.enrichmentService.enrichRiotAccount({ + userId: candidate.userId, + puuid: candidate.puuid, + options: { force: false }, + }); + + if (this.isFailedEnrichment(result)) { + failed += 1; + await this.bumpFailureCounter(candidate.puuid); + continue; + } + + await this.clearFailureCounter(candidate.puuid); + processed += 1; + } catch (err) { + failed += 1; + await this.bumpFailureCounter(candidate.puuid); + this.logger.warn( + `Background enrichRiotAccount failed for ${candidate.puuid}: ${ + err instanceof Error ? err.message : err + }`, + ); + } + } + + this.logger.debug( + `Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`, + ); + } + + private async tryBecomeLeader(): Promise { + const current = await this.garnet.getString(LEADER_KEY); + if (current === this.consumer) { + await this.garnet.setString( + LEADER_KEY, + this.consumer, + LEADER_TTL_SECONDS, + ); + return true; + } + if (current) return false; + + return this.garnet.setStringIfAbsent( + LEADER_KEY, + this.consumer, + LEADER_TTL_SECONDS, + ); + } + + private async renewLeaderIfHeld(): Promise { + try { + const current = await this.garnet.getString(LEADER_KEY); + if (current !== this.consumer) return; + await this.garnet.setString( + LEADER_KEY, + this.consumer, + LEADER_TTL_SECONDS, + ); + } catch (err) { + this.logger.warn( + `Background refresh leader renew failed: ${ + err instanceof Error ? err.message : err + }`, + ); + } + } + + private async isInFailureBackoff(puuid: string): Promise { + const count = await this.getFailureCount(puuid); + return count >= FAILURE_BACKOFF_THRESHOLD; + } + + private async getFailureCount(puuid: string): Promise { + const raw = await this.garnet.getString(this.failureBackoffKey(puuid)); + const count = Number(raw ?? 0); + if (!Number.isFinite(count) || count < 0) return 0; + return count; + } + + private async bumpFailureCounter(puuid: string): Promise { + const key = this.failureBackoffKey(puuid); + const client = this.garnet.raw(); + const response = await client.sendCommand(['INCR', key]); + await client.sendCommand([ + 'EXPIRE', + key, + String(FAILURE_BACKOFF_TTL_SECONDS), + ]); + + const count = Number(response ?? 0); + if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) { + this.logger.warn( + `Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`, + ); + } + } + + private async clearFailureCounter(puuid: string): Promise { + await this.garnet.delete(this.failureBackoffKey(puuid)); + } + + private isFailedEnrichment(result: EnrichResult): boolean { + if (!result.account) return true; + return [ + result.summoner, + result.league, + result.masteries, + result.recentMatches, + ].every((status) => status === 'failed'); + } + + private failureBackoffKey(puuid: string): string { + return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`; + } +} diff --git a/apps/api/src/modules/riot-background-refresh/kst-window.util.spec.ts b/apps/api/src/modules/riot-background-refresh/kst-window.util.spec.ts new file mode 100644 index 00000000..b2c421fa --- /dev/null +++ b/apps/api/src/modules/riot-background-refresh/kst-window.util.spec.ts @@ -0,0 +1,27 @@ +import { isInKstBackgroundWindow } from './kst-window.util'; + +describe('isInKstBackgroundWindow', () => { + it('includes KST 03:00', () => { + expect(isInKstBackgroundWindow(new Date('2026-05-02T18:00:00.000Z'))).toBe( + true, + ); + }); + + it('includes KST 09:59', () => { + expect(isInKstBackgroundWindow(new Date('2026-05-03T00:59:59.000Z'))).toBe( + true, + ); + }); + + it('excludes KST 10:00', () => { + expect(isInKstBackgroundWindow(new Date('2026-05-03T01:00:00.000Z'))).toBe( + false, + ); + }); + + it('excludes KST 02:59', () => { + expect(isInKstBackgroundWindow(new Date('2026-05-02T17:59:59.000Z'))).toBe( + false, + ); + }); +}); diff --git a/apps/api/src/modules/riot-background-refresh/kst-window.util.ts b/apps/api/src/modules/riot-background-refresh/kst-window.util.ts new file mode 100644 index 00000000..527fc6ad --- /dev/null +++ b/apps/api/src/modules/riot-background-refresh/kst-window.util.ts @@ -0,0 +1,19 @@ +const BG_WINDOW_START_HOUR = 3; +const BG_WINDOW_END_HOUR = 10; +const KST_HOUR_FORMAT = new Intl.DateTimeFormat('en-US', { + timeZone: 'Asia/Seoul', + hour: '2-digit', + hour12: false, +}); + +export function isInKstBackgroundWindow(now: Date = new Date()): boolean { + const hourString = KST_HOUR_FORMAT.format(now); + const hour = Number.parseInt(hourString, 10); + if (Number.isNaN(hour)) return false; + + const normalizedHour = hour % 24; + return ( + normalizedHour >= BG_WINDOW_START_HOUR && + normalizedHour < BG_WINDOW_END_HOUR + ); +} diff --git a/apps/api/src/modules/riot-background-refresh/stale-account.repository.spec.ts b/apps/api/src/modules/riot-background-refresh/stale-account.repository.spec.ts new file mode 100644 index 00000000..3629bf20 --- /dev/null +++ b/apps/api/src/modules/riot-background-refresh/stale-account.repository.spec.ts @@ -0,0 +1,79 @@ +import mongoose from 'mongoose'; +import { ConnectionService } from '../global/database/connection.service'; +import { + buildStaleAccountsPipeline, + StaleAccountRepository, +} from './stale-account.repository'; + +describe('buildStaleAccountsPipeline', () => { + const olderThan = new Date('2026-05-03T00:00:00.000Z'); + + it('filters to active users before unwinding Riot accounts', () => { + const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 }); + + expect(pipeline[0]).toEqual({ $match: { isActivated: true } }); + expect(pipeline[1]).toEqual({ $unwind: '$riotAccounts' }); + }); + + it('matches missing or older lastRefreshRequestedAt values', () => { + const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 }); + + expect(pipeline[2]).toEqual({ + $match: { + $or: [ + { 'riotAccounts.lastRefreshRequestedAt': { $exists: false } }, + { 'riotAccounts.lastRefreshRequestedAt': { $lt: olderThan } }, + ], + }, + }); + }); + + it('sorts missing refresh timestamps first', () => { + const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 20 }); + + expect(pipeline[3]).toEqual({ + $addFields: { + 'riotAccounts.lastRefreshRequestedAtSort': { + $ifNull: ['$riotAccounts.lastRefreshRequestedAt', new Date(0)], + }, + }, + }); + expect(pipeline[4]).toEqual({ + $sort: { 'riotAccounts.lastRefreshRequestedAtSort': 1 }, + }); + }); + + it('applies the provided limit', () => { + const pipeline = buildStaleAccountsPipeline({ olderThan, limit: 7 }); + + expect(pipeline[5]).toEqual({ $limit: 7 }); + }); +}); + +describe('StaleAccountRepository', () => { + it('returns aggregate candidates from the user model', async () => { + const userId = new mongoose.Types.ObjectId(); + const candidates = [{ userId, puuid: 'puuid-1' }]; + const exec = jest.fn().mockResolvedValue(candidates); + const aggregate = jest.fn().mockReturnValue({ exec }); + const connectionService = { + getModel: jest.fn().mockResolvedValue({ aggregate }), + } as unknown as ConnectionService; + + const repository = new StaleAccountRepository(connectionService); + await expect( + repository.findStaleAccounts({ + olderThan: new Date('2026-05-03T00:00:00.000Z'), + limit: 1, + }), + ).resolves.toBe(candidates); + + expect(connectionService.getModel).toHaveBeenCalledWith('user'); + expect(aggregate).toHaveBeenCalledWith( + buildStaleAccountsPipeline({ + olderThan: new Date('2026-05-03T00:00:00.000Z'), + limit: 1, + }), + ); + }); +}); diff --git a/apps/api/src/modules/riot-background-refresh/stale-account.repository.ts b/apps/api/src/modules/riot-background-refresh/stale-account.repository.ts new file mode 100644 index 00000000..a16d54b0 --- /dev/null +++ b/apps/api/src/modules/riot-background-refresh/stale-account.repository.ts @@ -0,0 +1,59 @@ +import { Injectable } from '@nestjs/common'; +import mongoose, { type PipelineStage } from 'mongoose'; +import { ConnectionService } from '../global/database/connection.service'; + +export interface StaleAccountCandidate { + userId: mongoose.Types.ObjectId; + puuid: string; + lastRefreshRequestedAt?: Date; +} + +export function buildStaleAccountsPipeline(opts: { + olderThan: Date; + limit: number; +}): PipelineStage[] { + return [ + { $match: { isActivated: true } }, + { $unwind: '$riotAccounts' }, + { + $match: { + $or: [ + { 'riotAccounts.lastRefreshRequestedAt': { $exists: false } }, + { 'riotAccounts.lastRefreshRequestedAt': { $lt: opts.olderThan } }, + ], + }, + }, + { + $addFields: { + 'riotAccounts.lastRefreshRequestedAtSort': { + $ifNull: ['$riotAccounts.lastRefreshRequestedAt', new Date(0)], + }, + }, + }, + { $sort: { 'riotAccounts.lastRefreshRequestedAtSort': 1 } }, + { $limit: opts.limit }, + { + $project: { + _id: 0, + userId: '$_id', + puuid: '$riotAccounts.puuid', + lastRefreshRequestedAt: '$riotAccounts.lastRefreshRequestedAt', + }, + }, + ]; +} + +@Injectable() +export class StaleAccountRepository { + constructor(private readonly connectionService: ConnectionService) {} + + public async findStaleAccounts(opts: { + olderThan: Date; + limit: number; + }): Promise { + const UserModel = await this.connectionService.getModel('user'); + return UserModel.aggregate( + buildStaleAccountsPipeline(opts), + ).exec(); + } +}