234 lines
6.7 KiB
TypeScript
234 lines
6.7 KiB
TypeScript
import {
|
|
Injectable,
|
|
Logger,
|
|
OnModuleDestroy,
|
|
OnModuleInit,
|
|
} from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import {
|
|
type EnrichResult,
|
|
RiotAccountEnrichmentService,
|
|
} from '../riot/riot-account-enrichment.service';
|
|
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
|
import { GarnetService } from '../global/garnet/garnet.service';
|
|
import { isInKstBackgroundWindow } from './kst-window.util';
|
|
import { StaleAccountRepository } from './stale-account.repository';
|
|
|
|
const TICK_INTERVAL_MS = 60_000;
|
|
const LEADER_KEY = 'riot:bg-refresh:leader';
|
|
const LEADER_TTL_SECONDS = 90;
|
|
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
|
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
|
const PER_TICK_CANDIDATE_LIMIT = 20;
|
|
const ENRICH_COST_TOKENS = 8;
|
|
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
|
|
const FAILURE_BACKOFF_THRESHOLD = 3;
|
|
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
|
|
|
|
@Injectable()
|
|
export class BackgroundRefreshScheduler
|
|
implements OnModuleInit, OnModuleDestroy
|
|
{
|
|
private readonly logger = new Logger(BackgroundRefreshScheduler.name);
|
|
private tickTimer?: NodeJS.Timeout;
|
|
private leaderRenewTimer?: NodeJS.Timeout;
|
|
private tickInProgress = false;
|
|
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
|
private readonly enabled: boolean;
|
|
|
|
constructor(
|
|
private readonly garnet: GarnetService,
|
|
private readonly rateLimiter: RiotRateLimiterService,
|
|
private readonly enrichmentService: RiotAccountEnrichmentService,
|
|
private readonly staleAccountRepository: StaleAccountRepository,
|
|
config: ConfigService,
|
|
) {
|
|
this.enabled = config.get<string>('RIOT_BG_REFRESH_ENABLED') === 'true';
|
|
}
|
|
|
|
public onModuleInit(): void {
|
|
if (!this.enabled) {
|
|
this.logger.log(
|
|
'Background Riot refresh is disabled (RIOT_BG_REFRESH_ENABLED).',
|
|
);
|
|
return;
|
|
}
|
|
|
|
this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS);
|
|
this.leaderRenewTimer = setInterval(
|
|
() => void this.renewLeaderIfHeld(),
|
|
LEADER_RENEW_INTERVAL_MS,
|
|
);
|
|
this.tickTimer.unref?.();
|
|
this.leaderRenewTimer.unref?.();
|
|
|
|
void this.safeTick();
|
|
}
|
|
|
|
public onModuleDestroy(): void {
|
|
if (this.tickTimer) clearInterval(this.tickTimer);
|
|
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
|
}
|
|
|
|
private async safeTick(): Promise<void> {
|
|
if (this.tickInProgress) return;
|
|
this.tickInProgress = true;
|
|
try {
|
|
await this.tick();
|
|
} catch (err) {
|
|
this.logger.warn(
|
|
`Background refresh tick failed: ${
|
|
err instanceof Error ? err.message : err
|
|
}`,
|
|
);
|
|
} finally {
|
|
this.tickInProgress = false;
|
|
}
|
|
}
|
|
|
|
private async tick(): Promise<void> {
|
|
if (!isInKstBackgroundWindow()) return;
|
|
const isLeader = await this.tryBecomeLeader();
|
|
if (!isLeader) return;
|
|
|
|
const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS);
|
|
const candidates = await this.staleAccountRepository.findStaleAccounts({
|
|
olderThan,
|
|
limit: PER_TICK_CANDIDATE_LIMIT,
|
|
});
|
|
if (candidates.length === 0) return;
|
|
|
|
let processed = 0;
|
|
let failed = 0;
|
|
let skipped = 0;
|
|
|
|
for (const candidate of candidates) {
|
|
if (!isInKstBackgroundWindow()) break;
|
|
if (await this.isInFailureBackoff(candidate.puuid)) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS);
|
|
await this.enrichmentService.markRefreshRequested(
|
|
candidate.userId,
|
|
candidate.puuid,
|
|
);
|
|
|
|
try {
|
|
const result = await this.enrichmentService.enrichRiotAccount({
|
|
userId: candidate.userId,
|
|
puuid: candidate.puuid,
|
|
options: { force: false },
|
|
});
|
|
|
|
if (this.isFailedEnrichment(result)) {
|
|
failed += 1;
|
|
await this.bumpFailureCounter(candidate.puuid);
|
|
continue;
|
|
}
|
|
|
|
await this.clearFailureCounter(candidate.puuid);
|
|
processed += 1;
|
|
} catch (err) {
|
|
failed += 1;
|
|
await this.bumpFailureCounter(candidate.puuid);
|
|
this.logger.warn(
|
|
`Background enrichRiotAccount failed for ${candidate.puuid}: ${
|
|
err instanceof Error ? err.message : err
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
this.logger.debug(
|
|
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`,
|
|
);
|
|
}
|
|
|
|
private async tryBecomeLeader(): Promise<boolean> {
|
|
const current = await this.garnet.getString(LEADER_KEY);
|
|
if (current === this.consumer) {
|
|
await this.garnet.setString(
|
|
LEADER_KEY,
|
|
this.consumer,
|
|
LEADER_TTL_SECONDS,
|
|
);
|
|
return true;
|
|
}
|
|
if (current) return false;
|
|
|
|
return this.garnet.setStringIfAbsent(
|
|
LEADER_KEY,
|
|
this.consumer,
|
|
LEADER_TTL_SECONDS,
|
|
);
|
|
}
|
|
|
|
private async renewLeaderIfHeld(): Promise<void> {
|
|
try {
|
|
const current = await this.garnet.getString(LEADER_KEY);
|
|
if (current !== this.consumer) return;
|
|
await this.garnet.setString(
|
|
LEADER_KEY,
|
|
this.consumer,
|
|
LEADER_TTL_SECONDS,
|
|
);
|
|
} catch (err) {
|
|
this.logger.warn(
|
|
`Background refresh leader renew failed: ${
|
|
err instanceof Error ? err.message : err
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async isInFailureBackoff(puuid: string): Promise<boolean> {
|
|
const count = await this.getFailureCount(puuid);
|
|
return count >= FAILURE_BACKOFF_THRESHOLD;
|
|
}
|
|
|
|
private async getFailureCount(puuid: string): Promise<number> {
|
|
const raw = await this.garnet.getString(this.failureBackoffKey(puuid));
|
|
const count = Number(raw ?? 0);
|
|
if (!Number.isFinite(count) || count < 0) return 0;
|
|
return count;
|
|
}
|
|
|
|
private async bumpFailureCounter(puuid: string): Promise<void> {
|
|
const key = this.failureBackoffKey(puuid);
|
|
const client = this.garnet.raw();
|
|
const response = await client.sendCommand(['INCR', key]);
|
|
await client.sendCommand([
|
|
'EXPIRE',
|
|
key,
|
|
String(FAILURE_BACKOFF_TTL_SECONDS),
|
|
]);
|
|
|
|
const count = Number(response ?? 0);
|
|
if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) {
|
|
this.logger.warn(
|
|
`Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async clearFailureCounter(puuid: string): Promise<void> {
|
|
await this.garnet.delete(this.failureBackoffKey(puuid));
|
|
}
|
|
|
|
private isFailedEnrichment(result: EnrichResult): boolean {
|
|
if (!result.account) return true;
|
|
return [
|
|
result.summoner,
|
|
result.league,
|
|
result.masteries,
|
|
result.recentMatches,
|
|
].every((status) => status === 'failed');
|
|
}
|
|
|
|
private failureBackoffKey(puuid: string): string {
|
|
return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`;
|
|
}
|
|
}
|