fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -9,13 +9,17 @@ import {
|
||||
RiotAccountEnrichmentService,
|
||||
} from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
import {
|
||||
type DistributedLockOwnership,
|
||||
DistributedLockService,
|
||||
} from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { isInKstBackgroundWindow } from './kst-window.util';
|
||||
import { StaleAccountRepository } from './stale-account.repository';
|
||||
|
||||
const TICK_INTERVAL_MS = 60_000;
|
||||
const LEADER_KEY = 'riot:bg-refresh:leader';
|
||||
const LEADER_TTL_SECONDS = 90;
|
||||
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
|
||||
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
|
||||
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
||||
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
||||
const PER_TICK_CANDIDATE_LIMIT = 20;
|
||||
@@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler
|
||||
private tickTimer?: NodeJS.Timeout;
|
||||
private leaderRenewTimer?: NodeJS.Timeout;
|
||||
private tickInProgress = false;
|
||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly redis: RedisService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
private readonly rateLimiter: RiotRateLimiterService,
|
||||
private readonly enrichmentService: RiotAccountEnrichmentService,
|
||||
private readonly staleAccountRepository: StaleAccountRepository,
|
||||
@@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler
|
||||
void this.safeTick();
|
||||
}
|
||||
|
||||
public onModuleDestroy(): void {
|
||||
public async onModuleDestroy(): Promise<void> {
|
||||
if (this.tickTimer) clearInterval(this.tickTimer);
|
||||
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
||||
if (!this.leaderLockOwnership) return;
|
||||
await this.distributedLockService
|
||||
.releaseLockIfOwned(this.leaderLockOwnership)
|
||||
.catch((err: unknown) => {
|
||||
this.logger.warn(
|
||||
`Background refresh leader release failed: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
});
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
|
||||
private async safeTick(): Promise<void> {
|
||||
@@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler
|
||||
}
|
||||
|
||||
private async tryBecomeLeader(): Promise<boolean> {
|
||||
const current = await this.redis.getString(LEADER_KEY);
|
||||
if (current === this.consumer) {
|
||||
await this.redis.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
return true;
|
||||
if (this.leaderLockOwnership) {
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return true;
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
if (current) return false;
|
||||
|
||||
return this.redis.setStringIfAbsent(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
||||
lockKey: LEADER_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
});
|
||||
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
||||
return Boolean(leaderLockOwnership);
|
||||
}
|
||||
|
||||
private async renewLeaderIfHeld(): Promise<void> {
|
||||
try {
|
||||
const current = await this.redis.getString(LEADER_KEY);
|
||||
if (current !== this.consumer) return;
|
||||
await this.redis.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
if (!this.leaderLockOwnership) return;
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return;
|
||||
this.leaderLockOwnership = undefined;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Background refresh leader renew failed: ${
|
||||
|
||||
Reference in new issue
Block a user