fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -1,4 +1,5 @@
import mongoose from 'mongoose';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
@@ -35,11 +36,18 @@ function makeScheduler() {
};
const redis = {
getString: jest.fn().mockResolvedValue(null),
setString: jest.fn().mockResolvedValue(undefined),
setStringIfAbsent: jest.fn().mockResolvedValue(true),
delete: jest.fn().mockResolvedValue(1),
raw: jest.fn().mockReturnValue(rawClient),
} as unknown as jest.Mocked<RedisService> & { raw: jest.Mock };
const distributedLockService = {
acquireLock: jest.fn().mockResolvedValue({
lockKey: 'distributed-lock:riot-background-refresh:leader',
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds: 90_000,
}),
extendLockIfOwned: jest.fn().mockResolvedValue(true),
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
} as unknown as jest.Mocked<DistributedLockService>;
const rateLimiter = {
isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
} as unknown as jest.Mocked<RiotRateLimiterService>;
@@ -52,6 +60,7 @@ function makeScheduler() {
} as unknown as jest.Mocked<StaleAccountRepository>;
const scheduler = new BackgroundRefreshScheduler(
redis,
distributedLockService,
rateLimiter,
enrichmentService,
staleAccountRepository,
@@ -60,6 +69,7 @@ function makeScheduler() {
return {
scheduler,
redis,
distributedLockService,
rawClient,
rateLimiter,
enrichmentService,
@@ -115,8 +125,9 @@ describe('BackgroundRefreshScheduler', () => {
it('does not query stale accounts when another leader holds the lock', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { scheduler, redis, staleAccountRepository } = makeScheduler();
redis.getString.mockResolvedValue('another-consumer');
const { scheduler, distributedLockService, staleAccountRepository } =
makeScheduler();
distributedLockService.acquireLock.mockResolvedValueOnce(null);
await scheduler.tick();
@@ -9,13 +9,17 @@ import {
RiotAccountEnrichmentService,
} from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
import {
type DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { isInKstBackgroundWindow } from './kst-window.util';
import { StaleAccountRepository } from './stale-account.repository';
const TICK_INTERVAL_MS = 60_000;
const LEADER_KEY = 'riot:bg-refresh:leader';
const LEADER_TTL_SECONDS = 90;
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
const LEADER_RENEW_INTERVAL_MS = 30_000;
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
const PER_TICK_CANDIDATE_LIMIT = 20;
@@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler
private tickTimer?: NodeJS.Timeout;
private leaderRenewTimer?: NodeJS.Timeout;
private tickInProgress = false;
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private leaderLockOwnership: DistributedLockOwnership | undefined;
constructor(
private readonly redis: RedisService,
private readonly distributedLockService: DistributedLockService,
private readonly rateLimiter: RiotRateLimiterService,
private readonly enrichmentService: RiotAccountEnrichmentService,
private readonly staleAccountRepository: StaleAccountRepository,
@@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler
void this.safeTick();
}
public onModuleDestroy(): void {
public async onModuleDestroy(): Promise<void> {
if (this.tickTimer) clearInterval(this.tickTimer);
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
.catch((err: unknown) => {
this.logger.warn(
`Background refresh leader release failed: ${
err instanceof Error ? err.message : err
}`,
);
});
this.leaderLockOwnership = undefined;
}
private async safeTick(): Promise<void> {
@@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler
}
private async tryBecomeLeader(): Promise<boolean> {
const current = await this.redis.getString(LEADER_KEY);
if (current === this.consumer) {
await this.redis.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
return true;
if (this.leaderLockOwnership) {
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return true;
this.leaderLockOwnership = undefined;
}
if (current) return false;
return this.redis.setStringIfAbsent(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
const leaderLockOwnership = await this.distributedLockService.acquireLock({
lockKey: LEADER_LOCK_KEY,
lockExpirationDurationMilliseconds:
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
});
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
return Boolean(leaderLockOwnership);
}
private async renewLeaderIfHeld(): Promise<void> {
try {
const current = await this.redis.getString(LEADER_KEY);
if (current !== this.consumer) return;
await this.redis.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
if (!this.leaderLockOwnership) return;
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return;
this.leaderLockOwnership = undefined;
} catch (err) {
this.logger.warn(
`Background refresh leader renew failed: ${