fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -9,13 +9,17 @@ import {
RiotAccountEnrichmentService,
} from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
import {
type DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { isInKstBackgroundWindow } from './kst-window.util';
import { StaleAccountRepository } from './stale-account.repository';
const TICK_INTERVAL_MS = 60_000;
const LEADER_KEY = 'riot:bg-refresh:leader';
const LEADER_TTL_SECONDS = 90;
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
const LEADER_RENEW_INTERVAL_MS = 30_000;
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
const PER_TICK_CANDIDATE_LIMIT = 20;
@@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler
private tickTimer?: NodeJS.Timeout;
private leaderRenewTimer?: NodeJS.Timeout;
private tickInProgress = false;
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private leaderLockOwnership: DistributedLockOwnership | undefined;
constructor(
private readonly redis: RedisService,
private readonly distributedLockService: DistributedLockService,
private readonly rateLimiter: RiotRateLimiterService,
private readonly enrichmentService: RiotAccountEnrichmentService,
private readonly staleAccountRepository: StaleAccountRepository,
@@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler
void this.safeTick();
}
public onModuleDestroy(): void {
public async onModuleDestroy(): Promise<void> {
if (this.tickTimer) clearInterval(this.tickTimer);
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
.catch((err: unknown) => {
this.logger.warn(
`Background refresh leader release failed: ${
err instanceof Error ? err.message : err
}`,
);
});
this.leaderLockOwnership = undefined;
}
private async safeTick(): Promise<void> {
@@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler
}
private async tryBecomeLeader(): Promise<boolean> {
const current = await this.redis.getString(LEADER_KEY);
if (current === this.consumer) {
await this.redis.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
return true;
if (this.leaderLockOwnership) {
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return true;
this.leaderLockOwnership = undefined;
}
if (current) return false;
return this.redis.setStringIfAbsent(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
const leaderLockOwnership = await this.distributedLockService.acquireLock({
lockKey: LEADER_LOCK_KEY,
lockExpirationDurationMilliseconds:
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
});
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
return Boolean(leaderLockOwnership);
}
private async renewLeaderIfHeld(): Promise<void> {
try {
const current = await this.redis.getString(LEADER_KEY);
if (current !== this.consumer) return;
await this.redis.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
if (!this.leaderLockOwnership) return;
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return;
this.leaderLockOwnership = undefined;
} catch (err) {
this.logger.warn(
`Background refresh leader renew failed: ${