287 lines
8.6 KiB
TypeScript
287 lines
8.6 KiB
TypeScript
import {
|
|
Injectable,
|
|
Logger,
|
|
OnModuleDestroy,
|
|
OnModuleInit,
|
|
} from '@nestjs/common';
|
|
import {
|
|
type EnrichResult,
|
|
RiotAccountEnrichmentService,
|
|
} from '../riot/riot-account-enrichment.service';
|
|
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
|
import {
|
|
type DistributedLockOwnership,
|
|
DistributedLockService,
|
|
} from '../global/redis/distributed-lock.service';
|
|
import { RedisService } from '../global/redis/redis.service';
|
|
import { isInKstBackgroundWindow } from './kst-window.util';
|
|
import {
|
|
type StaleAccountCandidate,
|
|
StaleAccountRepository,
|
|
} from './stale-account.repository';
|
|
|
|
const TICK_INTERVAL_MS = 60_000;
|
|
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
|
|
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
|
|
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
|
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
|
const PER_TICK_CANDIDATE_LIMIT = 20;
|
|
const DAWN_USAGE_RATIO = 0.8;
|
|
const OPPORTUNISTIC_USAGE_RATIO = 0.5;
|
|
const FAILURE_BACKOFF_KEY_PREFIX = 'riot:bg:fail';
|
|
const FAILURE_BACKOFF_THRESHOLD = 3;
|
|
const FAILURE_BACKOFF_TTL_SECONDS = 24 * 60 * 60;
|
|
|
|
interface RefreshGatePolicy {
|
|
mode: 'dawn' | 'opportunistic';
|
|
usageRatio: number;
|
|
}
|
|
|
|
@Injectable()
|
|
export class BackgroundRefreshScheduler
|
|
implements OnModuleInit, OnModuleDestroy
|
|
{
|
|
private readonly logger = new Logger(BackgroundRefreshScheduler.name);
|
|
private tickTimer?: NodeJS.Timeout;
|
|
private leaderRenewTimer?: NodeJS.Timeout;
|
|
private tickInProgress = false;
|
|
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
|
|
|
constructor(
|
|
private readonly redis: RedisService,
|
|
private readonly distributedLockService: DistributedLockService,
|
|
private readonly rateLimiter: RiotRateLimiterService,
|
|
private readonly enrichmentService: RiotAccountEnrichmentService,
|
|
private readonly staleAccountRepository: StaleAccountRepository,
|
|
) {}
|
|
|
|
public onModuleInit(): void {
|
|
this.tickTimer = setInterval(() => void this.safeTick(), TICK_INTERVAL_MS);
|
|
this.leaderRenewTimer = setInterval(
|
|
() => void this.renewLeaderIfHeld(),
|
|
LEADER_RENEW_INTERVAL_MS,
|
|
);
|
|
this.tickTimer.unref?.();
|
|
this.leaderRenewTimer.unref?.();
|
|
|
|
void this.safeTick();
|
|
}
|
|
|
|
public async onModuleDestroy(): Promise<void> {
|
|
if (this.tickTimer) clearInterval(this.tickTimer);
|
|
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
|
if (!this.leaderLockOwnership) return;
|
|
await this.distributedLockService
|
|
.releaseLockIfOwned(this.leaderLockOwnership)
|
|
.catch((err: unknown) => {
|
|
this.logger.warn(
|
|
`Background refresh leader release failed: ${
|
|
err instanceof Error ? err.message : String(err)
|
|
}`,
|
|
);
|
|
});
|
|
this.leaderLockOwnership = undefined;
|
|
}
|
|
|
|
private async safeTick(): Promise<void> {
|
|
if (this.tickInProgress) return;
|
|
this.tickInProgress = true;
|
|
try {
|
|
await this.tick();
|
|
} catch (err) {
|
|
this.logger.warn(
|
|
`Background refresh tick failed: ${
|
|
err instanceof Error ? err.message : err
|
|
}`,
|
|
);
|
|
} finally {
|
|
this.tickInProgress = false;
|
|
}
|
|
}
|
|
|
|
private async tick(): Promise<void> {
|
|
const initialGate = this.currentRefreshGatePolicy();
|
|
if (!(await this.rateLimiter.isRefreshWindowOpen(initialGate.usageRatio))) {
|
|
return;
|
|
}
|
|
|
|
const isLeader = await this.tryBecomeLeader();
|
|
if (!isLeader) return;
|
|
|
|
const olderThan = new Date(Date.now() - STALE_THRESHOLD_MS);
|
|
const candidates = await this.staleAccountRepository.findStaleAccounts({
|
|
olderThan,
|
|
limit: PER_TICK_CANDIDATE_LIMIT,
|
|
});
|
|
if (candidates.length === 0) return;
|
|
|
|
let processed = 0;
|
|
let failed = 0;
|
|
let skipped = 0;
|
|
let currentGate = initialGate;
|
|
|
|
for (const candidate of candidates) {
|
|
currentGate = this.currentRefreshGatePolicy();
|
|
if (
|
|
!(await this.rateLimiter.isRefreshWindowOpen(currentGate.usageRatio))
|
|
) {
|
|
break;
|
|
}
|
|
if (await this.isInFailureBackoff(candidate.puuid)) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
await this.enrichmentService.markRefreshRequested(
|
|
candidate.userId,
|
|
candidate.puuid,
|
|
);
|
|
await this.refreshRiotIdentity(candidate);
|
|
|
|
try {
|
|
const result = await this.enrichmentService.enrichRiotAccount({
|
|
userId: candidate.userId,
|
|
puuid: candidate.puuid,
|
|
options: { force: false },
|
|
});
|
|
|
|
if (this.isFailedEnrichment(result)) {
|
|
failed += 1;
|
|
await this.bumpFailureCounter(candidate.puuid);
|
|
continue;
|
|
}
|
|
|
|
await this.clearFailureCounter(candidate.puuid);
|
|
processed += 1;
|
|
} catch (err) {
|
|
failed += 1;
|
|
await this.bumpFailureCounter(candidate.puuid);
|
|
this.logger.warn(
|
|
`Background enrichRiotAccount failed for ${candidate.puuid}: ${
|
|
err instanceof Error ? err.message : err
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
this.logger.debug(
|
|
`Background refresh tick: processed=${processed} failed=${failed} skipped=${skipped} candidates=${candidates.length} mode=${currentGate.mode} ratio=${currentGate.usageRatio}`,
|
|
);
|
|
}
|
|
|
|
private async refreshRiotIdentity(
|
|
candidate: StaleAccountCandidate,
|
|
): Promise<void> {
|
|
try {
|
|
const result = await this.enrichmentService.refreshRiotIdentity({
|
|
userId: candidate.userId,
|
|
puuid: candidate.puuid,
|
|
});
|
|
if (result !== 'failed') return;
|
|
|
|
this.logger.warn(
|
|
`Background Riot identity refresh failed for ${candidate.puuid}.`,
|
|
);
|
|
} catch (err) {
|
|
this.logger.warn(
|
|
`Background Riot identity refresh failed for ${candidate.puuid}: ${
|
|
err instanceof Error ? err.message : String(err)
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private currentRefreshGatePolicy(): RefreshGatePolicy {
|
|
const isDawn = isInKstBackgroundWindow();
|
|
return {
|
|
mode: isDawn ? 'dawn' : 'opportunistic',
|
|
usageRatio: isDawn ? DAWN_USAGE_RATIO : OPPORTUNISTIC_USAGE_RATIO,
|
|
};
|
|
}
|
|
|
|
private async tryBecomeLeader(): Promise<boolean> {
|
|
if (this.leaderLockOwnership) {
|
|
const didExtendLeaderLock =
|
|
await this.distributedLockService.extendLockIfOwned(
|
|
this.leaderLockOwnership,
|
|
);
|
|
if (didExtendLeaderLock) return true;
|
|
this.leaderLockOwnership = undefined;
|
|
}
|
|
|
|
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
|
lockKey: LEADER_LOCK_KEY,
|
|
lockExpirationDurationMilliseconds:
|
|
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
|
});
|
|
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
|
return Boolean(leaderLockOwnership);
|
|
}
|
|
|
|
private async renewLeaderIfHeld(): Promise<void> {
|
|
try {
|
|
if (!this.leaderLockOwnership) return;
|
|
const didExtendLeaderLock =
|
|
await this.distributedLockService.extendLockIfOwned(
|
|
this.leaderLockOwnership,
|
|
);
|
|
if (didExtendLeaderLock) return;
|
|
this.leaderLockOwnership = undefined;
|
|
} catch (err) {
|
|
this.logger.warn(
|
|
`Background refresh leader renew failed: ${
|
|
err instanceof Error ? err.message : err
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async isInFailureBackoff(puuid: string): Promise<boolean> {
|
|
const count = await this.getFailureCount(puuid);
|
|
return count >= FAILURE_BACKOFF_THRESHOLD;
|
|
}
|
|
|
|
private async getFailureCount(puuid: string): Promise<number> {
|
|
const raw = await this.redis.getString(this.failureBackoffKey(puuid));
|
|
const count = Number(raw ?? 0);
|
|
if (!Number.isFinite(count) || count < 0) return 0;
|
|
return count;
|
|
}
|
|
|
|
private async bumpFailureCounter(puuid: string): Promise<void> {
|
|
const key = this.failureBackoffKey(puuid);
|
|
const client = this.redis.raw();
|
|
const response = await client.sendCommand(['INCR', key]);
|
|
await client.sendCommand([
|
|
'EXPIRE',
|
|
key,
|
|
String(FAILURE_BACKOFF_TTL_SECONDS),
|
|
]);
|
|
|
|
const count = Number(response ?? 0);
|
|
if (Number.isFinite(count) && count >= FAILURE_BACKOFF_THRESHOLD) {
|
|
this.logger.warn(
|
|
`Background Riot refresh backoff enabled for ${puuid} after ${count} failures.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async clearFailureCounter(puuid: string): Promise<void> {
|
|
await this.redis.delete(this.failureBackoffKey(puuid));
|
|
}
|
|
|
|
private isFailedEnrichment(result: EnrichResult): boolean {
|
|
if (!result.account) return true;
|
|
return [
|
|
result.summoner,
|
|
result.league,
|
|
result.masteries,
|
|
result.recentMatches,
|
|
].every((status) => status === 'failed');
|
|
}
|
|
|
|
private failureBackoffKey(puuid: string): string {
|
|
return `${FAILURE_BACKOFF_KEY_PREFIX}:${puuid}`;
|
|
}
|
|
}
|