import { Injectable, Logger, OnModuleDestroy, OnModuleInit, } from '@nestjs/common'; import { type EnrichResult, RiotAccountEnrichmentService, } from '../riot/riot-account-enrichment.service'; import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service'; import { type DistributedLockOwnership, DistributedLockService, } from '../global/redis/distributed-lock.service'; import { RedisService } from '../global/redis/redis.service'; import { isInKstBackgroundWindow } from './kst-window.util'; import { StaleAccountRepository } from './stale-account.repository'; const TICK_INTERVAL_MS = 60_000; const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader'; const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000; const LEADER_RENEW_INTERVAL_MS = 30_000; const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000; const PER_TICK_CANDIDATE_LIMIT = 20; const DAWN_USAGE_RATIO = 0.8; const OPPORTUNISTIC_USAGE_RATIO = 0.5; const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail'; const FAILURE_BACKOFF_THRESHOLD = 3; const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60; interface RefreshGatePolicy { mode: 'dawn' | 'opportunistic'; usageRatio: number; } @Injectable() export class BackgroundRefreshScheduler implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(BackgroundRefreshScheduler.name); private tickTimer?: NodeJS.Timeout; private leaderRenewTimer?: NodeJS.Timeout; private tickInProgress = false; private leaderLockOwnership: DistributedLockOwnership | undefined; constructor( private readonly redis: RedisService, private readonly distributedLockService: DistributedLockService, private readonly rateLimiter: RiotRateLimiterService, private readonly enrichmentService: RiotAccountEnrichmentService, private readonly staleAccountRepository: StaleAccountRepository, ) {} public onModuleInit(): void { this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS); this.leaderRenewTimer = setInterval( () => void this.renewLeaderIfHeld(), LEADER_RENEW_INTERVAL_MS, ); this.tickTimer.unref?.(); this.leaderRenewTimer.unref?.(); void this.safeTick(); } public async onModuleDestroy(): Promise { if (this.tickTimer) clearInterval(this.tickTimer); if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer); if (!this.leaderLockOwnership) return; await this.distributedLockService .releaseLockIfOwned(this.leaderLockOwnership) .catch((err: unknown) => { this.logger.warn( `Background refresh leader release failed: ${ err instanceof Error ? err.message : err }`, ); }); this.leaderLockOwnership = undefined; } private async safeTick(): Promise { if (this.tickInProgress) return; this.tickInProgress = true; try { await this.tick(); } catch (err) { this.logger.warn( `Background refresh tick failed: ${ err instanceof Error ? err.message : err }`, ); } finally { this.tickInProgress = false; } } private async tick(): Promise { const initialGate = this.currentRefreshGatePolicy(); if (!(await this.rateLimiter.isRefreshWindowOpen(initialGate.usageRatio))) { return; } const isLeader = await this.tryBecomeLeader(); if (!isLeader) return; const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS); const candidates = await this.staleAccountRepository.findStaleAccounts({ olderThan, limit: PER_TICK_CANDIDATE_LIMIT, }); if (candidates.length === 0) return; let processed = 0; let failed = 0; let skipped = 0; let currentGate = initialGate; for (const candidate of candidates) { currentGate = this.currentRefreshGatePolicy(); if ( !(await this.rateLimiter.isRefreshWindowOpen(currentGate.usageRatio)) ) { break; } if (await this.isInFailureBackoff(candidate.puuid)) { skipped += 1; continue; } await this.enrichmentService.markRefreshRequested( candidate.userId, candidate.puuid, ); try { const result = await this.enrichmentService.enrichRiotAccount({ userId: candidate.userId, puuid: candidate.puuid, options: { force: false }, }); if (this.isFailedEnrichment(result)) { failed += 1; await this.bumpFailureCounter(candidate.puuid); continue; } await this.clearFailureCounter(candidate.puuid); processed += 1; } catch (err) { failed += 1; await this.bumpFailureCounter(candidate.puuid); this.logger.warn( `Background enrichRiotAccount failed for ${candidate.puuid}: ${ err instanceof Error ? err.message : err }`, ); } } this.logger.debug( `Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length} mode=${currentGate.mode} ratio=${currentGate.usageRatio}`, ); } private currentRefreshGatePolicy(): RefreshGatePolicy { const isDawn = isInKstBackgroundWindow(); return { mode: isDawn ? 'dawn' : 'opportunistic', usageRatio: isDawn ? DAWN_USAGE_RATIO : OPPORTUNISTIC_USAGE_RATIO, }; } private async tryBecomeLeader(): Promise { if (this.leaderLockOwnership) { const didExtendLeaderLock = await this.distributedLockService.extendLockIfOwned( this.leaderLockOwnership, ); if (didExtendLeaderLock) return true; this.leaderLockOwnership = undefined; } const leaderLockOwnership = await this.distributedLockService.acquireLock({ lockKey: LEADER_LOCK_KEY, lockExpirationDurationMilliseconds: LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS, }); this.leaderLockOwnership = leaderLockOwnership ?? undefined; return Boolean(leaderLockOwnership); } private async renewLeaderIfHeld(): Promise { try { if (!this.leaderLockOwnership) return; const didExtendLeaderLock = await this.distributedLockService.extendLockIfOwned( this.leaderLockOwnership, ); if (didExtendLeaderLock) return; this.leaderLockOwnership = undefined; } catch (err) { this.logger.warn( `Background refresh leader renew failed: ${ err instanceof Error ? err.message : err }`, ); } } private async isInFailureBackoff(puuid: string): Promise { const count = await this.getFailureCount(puuid); return count >= FAILURE_BACKOFF_THRESHOLD; } private async getFailureCount(puuid: string): Promise { const raw = await this.redis.getString(this.failureBackoffKey(puuid)); const count = Number(raw ?? 0); if (!Number.isFinite(count) || count < 0) return 0; return count; } private async bumpFailureCounter(puuid: string): Promise { const key = this.failureBackoffKey(puuid); const client = this.redis.raw(); const response = await client.sendCommand(['INCR', key]); await client.sendCommand([ 'EXPIRE', key, String(FAILURE_BACKOFF_TTL_SECONDS), ]); const count = Number(response ?? 0); if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) { this.logger.warn( `Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`, ); } } private async clearFailureCounter(puuid: string): Promise { await this.redis.delete(this.failureBackoffKey(puuid)); } private isFailedEnrichment(result: EnrichResult): boolean { if (!result.account) return true; return [ result.summoner, result.league, result.masteries, result.recentMatches, ].every((status) => status === 'failed'); } private failureBackoffKey(puuid: string): string { return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`; } }