feat(api): add background refresh
This commit is contained in:
1 parent
4bc2df7bcc
commit
7e78b1f92d
7 files changed
+614
No files matched your search
@@ -0,0 +1,233 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import {
|
||||
type EnrichResult,
|
||||
RiotAccountEnrichmentService,
|
||||
} from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
import { GarnetService } from '../global/garnet/garnet.service';
|
||||
import { isInKstBackgroundWindow } from './kst-window.util';
|
||||
import { StaleAccountRepository } from './stale-account.repository';
|
||||
|
||||
const TICK_INTERVAL_MS = 60_000;
|
||||
const LEADER_KEY = 'riot:bg-refresh:leader';
|
||||
const LEADER_TTL_SECONDS = 90;
|
||||
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
||||
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
||||
const PER_TICK_CANDIDATE_LIMIT = 20;
|
||||
const ENRICH_COST_TOKENS = 8;
|
||||
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
|
||||
const FAILURE_BACKOFF_THRESHOLD = 3;
|
||||
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
|
||||
|
||||
@Injectable()
|
||||
export class BackgroundRefreshScheduler
|
||||
implements OnModuleInit, OnModuleDestroy
|
||||
{
|
||||
private readonly logger = new Logger(BackgroundRefreshScheduler.name);
|
||||
private tickTimer?: NodeJS.Timeout;
|
||||
private leaderRenewTimer?: NodeJS.Timeout;
|
||||
private tickInProgress = false;
|
||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(
|
||||
private readonly garnet: GarnetService,
|
||||
private readonly rateLimiter: RiotRateLimiterService,
|
||||
private readonly enrichmentService: RiotAccountEnrichmentService,
|
||||
private readonly staleAccountRepository: StaleAccountRepository,
|
||||
config: ConfigService,
|
||||
) {
|
||||
this.enabled = config.get<string>('RIOT_BG_REFRESH_ENABLED') === 'true';
|
||||
}
|
||||
|
||||
public onModuleInit(): void {
|
||||
if (!this.enabled) {
|
||||
this.logger.log(
|
||||
'Background Riot refresh is disabled (RIOT_BG_REFRESH_ENABLED).',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS);
|
||||
this.leaderRenewTimer = setInterval(
|
||||
() => void this.renewLeaderIfHeld(),
|
||||
LEADER_RENEW_INTERVAL_MS,
|
||||
);
|
||||
this.tickTimer.unref?.();
|
||||
this.leaderRenewTimer.unref?.();
|
||||
|
||||
void this.safeTick();
|
||||
}
|
||||
|
||||
public onModuleDestroy(): void {
|
||||
if (this.tickTimer) clearInterval(this.tickTimer);
|
||||
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
||||
}
|
||||
|
||||
private async safeTick(): Promise<void> {
|
||||
if (this.tickInProgress) return;
|
||||
this.tickInProgress = true;
|
||||
try {
|
||||
await this.tick();
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Background refresh tick failed: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
} finally {
|
||||
this.tickInProgress = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async tick(): Promise<void> {
|
||||
if (!isInKstBackgroundWindow()) return;
|
||||
const isLeader = await this.tryBecomeLeader();
|
||||
if (!isLeader) return;
|
||||
|
||||
const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS);
|
||||
const candidates = await this.staleAccountRepository.findStaleAccounts({
|
||||
olderThan,
|
||||
limit: PER_TICK_CANDIDATE_LIMIT,
|
||||
});
|
||||
if (candidates.length === 0) return;
|
||||
|
||||
let processed = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (!isInKstBackgroundWindow()) break;
|
||||
if (await this.isInFailureBackoff(candidate.puuid)) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.rateLimiter.acquireBackground(ENRICH_COST_TOKENS);
|
||||
await this.enrichmentService.markRefreshRequested(
|
||||
candidate.userId,
|
||||
candidate.puuid,
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await this.enrichmentService.enrichRiotAccount({
|
||||
userId: candidate.userId,
|
||||
puuid: candidate.puuid,
|
||||
options: { force: false },
|
||||
});
|
||||
|
||||
if (this.isFailedEnrichment(result)) {
|
||||
failed += 1;
|
||||
await this.bumpFailureCounter(candidate.puuid);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.clearFailureCounter(candidate.puuid);
|
||||
processed += 1;
|
||||
} catch (err) {
|
||||
failed += 1;
|
||||
await this.bumpFailureCounter(candidate.puuid);
|
||||
this.logger.warn(
|
||||
`Background enrichRiotAccount failed for ${candidate.puuid}: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length}`,
|
||||
);
|
||||
}
|
||||
|
||||
private async tryBecomeLeader(): Promise<boolean> {
|
||||
const current = await this.garnet.getString(LEADER_KEY);
|
||||
if (current === this.consumer) {
|
||||
await this.garnet.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if (current) return false;
|
||||
|
||||
return this.garnet.setStringIfAbsent(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
}
|
||||
|
||||
private async renewLeaderIfHeld(): Promise<void> {
|
||||
try {
|
||||
const current = await this.garnet.getString(LEADER_KEY);
|
||||
if (current !== this.consumer) return;
|
||||
await this.garnet.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Background refresh leader renew failed: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async isInFailureBackoff(puuid: string): Promise<boolean> {
|
||||
const count = await this.getFailureCount(puuid);
|
||||
return count >= FAILURE_BACKOFF_THRESHOLD;
|
||||
}
|
||||
|
||||
private async getFailureCount(puuid: string): Promise<number> {
|
||||
const raw = await this.garnet.getString(this.failureBackoffKey(puuid));
|
||||
const count = Number(raw ?? 0);
|
||||
if (!Number.isFinite(count) || count < 0) return 0;
|
||||
return count;
|
||||
}
|
||||
|
||||
private async bumpFailureCounter(puuid: string): Promise<void> {
|
||||
const key = this.failureBackoffKey(puuid);
|
||||
const client = this.garnet.raw();
|
||||
const response = await client.sendCommand(['INCR', key]);
|
||||
await client.sendCommand([
|
||||
'EXPIRE',
|
||||
key,
|
||||
String(FAILURE_BACKOFF_TTL_SECONDS),
|
||||
]);
|
||||
|
||||
const count = Number(response ?? 0);
|
||||
if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) {
|
||||
this.logger.warn(
|
||||
`Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async clearFailureCounter(puuid: string): Promise<void> {
|
||||
await this.garnet.delete(this.failureBackoffKey(puuid));
|
||||
}
|
||||
|
||||
private isFailedEnrichment(result: EnrichResult): boolean {
|
||||
if (!result.account) return true;
|
||||
return [
|
||||
result.summoner,
|
||||
result.league,
|
||||
result.masteries,
|
||||
result.recentMatches,
|
||||
].every((status) => status === 'failed');
|
||||
}
|
||||
|
||||
private failureBackoffKey(puuid: string): string {
|
||||
return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user