Files
lolympus-tournament-support/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts
T

261 lines
7.9 KiB
TypeScript

import {
Injectable,
Logger,
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import {
type EnrichResult,
RiotAccountEnrichmentService,
} from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
import {
type DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { isInKstBackgroundWindow } from './kst-window.util';
import { StaleAccountRepository } from './stale-account.repository';
const TICK_INTERVAL_MS = 60_000;
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
const LEADER_RENEW_INTERVAL_MS = 30_000;
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
const PER_TICK_CANDIDATE_LIMIT = 20;
const DAWN_USAGE_RATIO = 0.8;
const OPPORTUNISTIC_USAGE_RATIO = 0.5;
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
const FAILURE_BACKOFF_THRESHOLD = 3;
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
interface RefreshGatePolicy {
mode: 'dawn' | 'opportunistic';
usageRatio: number;
}
@Injectable()
export class BackgroundRefreshScheduler
implements OnModuleInit, OnModuleDestroy
{
private readonly logger = new Logger(BackgroundRefreshScheduler.name);
private tickTimer?: NodeJS.Timeout;
private leaderRenewTimer?: NodeJS.Timeout;
private tickInProgress = false;
private leaderLockOwnership: DistributedLockOwnership | undefined;
constructor(
private readonly redis: RedisService,
private readonly distributedLockService: DistributedLockService,
private readonly rateLimiter: RiotRateLimiterService,
private readonly enrichmentService: RiotAccountEnrichmentService,
private readonly staleAccountRepository: StaleAccountRepository,
) {}
public onModuleInit(): void {
this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS);
this.leaderRenewTimer = setInterval(
() => void this.renewLeaderIfHeld(),
LEADER_RENEW_INTERVAL_MS,
);
this.tickTimer.unref?.();
this.leaderRenewTimer.unref?.();
void this.safeTick();
}
public async onModuleDestroy(): Promise<void> {
if (this.tickTimer) clearInterval(this.tickTimer);
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
.catch((err: unknown) => {
this.logger.warn(
`Background refresh leader release failed: ${
err instanceof Error ? err.message : err
}`,
);
});
this.leaderLockOwnership = undefined;
}
private async safeTick(): Promise<void> {
if (this.tickInProgress) return;
this.tickInProgress = true;
try {
await this.tick();
} catch (err) {
this.logger.warn(
`Background refresh tick failed: ${
err instanceof Error ? err.message : err
}`,
);
} finally {
this.tickInProgress = false;
}
}
private async tick(): Promise<void> {
const initialGate = this.currentRefreshGatePolicy();
if (!(await this.rateLimiter.isRefreshWindowOpen(initialGate.usageRatio))) {
return;
}
const isLeader = await this.tryBecomeLeader();
if (!isLeader) return;
const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS);
const candidates = await this.staleAccountRepository.findStaleAccounts({
olderThan,
limit: PER_TICK_CANDIDATE_LIMIT,
});
if (candidates.length === 0) return;
let processed = 0;
let failed = 0;
let skipped = 0;
let currentGate = initialGate;
for (const candidate of candidates) {
currentGate = this.currentRefreshGatePolicy();
if (
!(await this.rateLimiter.isRefreshWindowOpen(currentGate.usageRatio))
) {
break;
}
if (await this.isInFailureBackoff(candidate.puuid)) {
skipped += 1;
continue;
}
await this.enrichmentService.markRefreshRequested(
candidate.userId,
candidate.puuid,
);
try {
const result = await this.enrichmentService.enrichRiotAccount({
userId: candidate.userId,
puuid: candidate.puuid,
options: { force: false },
});
if (this.isFailedEnrichment(result)) {
failed += 1;
await this.bumpFailureCounter(candidate.puuid);
continue;
}
await this.clearFailureCounter(candidate.puuid);
processed += 1;
} catch (err) {
failed += 1;
await this.bumpFailureCounter(candidate.puuid);
this.logger.warn(
`Background enrichRiotAccount failed for ${candidate.puuid}: ${
err instanceof Error ? err.message : err
}`,
);
}
}
this.logger.debug(
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length} mode=${currentGate.mode} ratio=${currentGate.usageRatio}`,
);
}
private currentRefreshGatePolicy(): RefreshGatePolicy {
const isDawn = isInKstBackgroundWindow();
return {
mode: isDawn ? 'dawn' : 'opportunistic',
usageRatio: isDawn ? DAWN_USAGE_RATIO : OPPORTUNISTIC_USAGE_RATIO,
};
}
private async tryBecomeLeader(): Promise<boolean> {
if (this.leaderLockOwnership) {
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return true;
this.leaderLockOwnership = undefined;
}
const leaderLockOwnership = await this.distributedLockService.acquireLock({
lockKey: LEADER_LOCK_KEY,
lockExpirationDurationMilliseconds:
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
});
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
return Boolean(leaderLockOwnership);
}
private async renewLeaderIfHeld(): Promise<void> {
try {
if (!this.leaderLockOwnership) return;
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return;
this.leaderLockOwnership = undefined;
} catch (err) {
this.logger.warn(
`Background refresh leader renew failed: ${
err instanceof Error ? err.message : err
}`,
);
}
}
private async isInFailureBackoff(puuid: string): Promise<boolean> {
const count = await this.getFailureCount(puuid);
return count >= FAILURE_BACKOFF_THRESHOLD;
}
private async getFailureCount(puuid: string): Promise<number> {
const raw = await this.redis.getString(this.failureBackoffKey(puuid));
const count = Number(raw ?? 0);
if (!Number.isFinite(count) || count < 0) return 0;
return count;
}
private async bumpFailureCounter(puuid: string): Promise<void> {
const key = this.failureBackoffKey(puuid);
const client = this.redis.raw();
const response = await client.sendCommand(['INCR', key]);
await client.sendCommand([
'EXPIRE',
key,
String(FAILURE_BACKOFF_TTL_SECONDS),
]);
const count = Number(response ?? 0);
if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) {
this.logger.warn(
`Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`,
);
}
}
private async clearFailureCounter(puuid: string): Promise<void> {
await this.redis.delete(this.failureBackoffKey(puuid));
}
private isFailedEnrichment(result: EnrichResult): boolean {
if (!result.account) return true;
return [
result.summoner,
result.league,
result.masteries,
result.recentMatches,
].every((status) => status === 'failed');
}
private failureBackoffKey(puuid: string): string {
return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`;
}
}