fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
@@ -35,11 +36,18 @@ function makeScheduler() {
|
||||
};
|
||||
const redis = {
|
||||
getString: jest.fn().mockResolvedValue(null),
|
||||
setString: jest.fn().mockResolvedValue(undefined),
|
||||
setStringIfAbsent: jest.fn().mockResolvedValue(true),
|
||||
delete: jest.fn().mockResolvedValue(1),
|
||||
raw: jest.fn().mockReturnValue(rawClient),
|
||||
} as unknown as jest.Mocked<RedisService> & { raw: jest.Mock };
|
||||
const distributedLockService = {
|
||||
acquireLock: jest.fn().mockResolvedValue({
|
||||
lockKey: 'distributed-lock:riot-background-refresh:leader',
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds: 90_000,
|
||||
}),
|
||||
extendLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const rateLimiter = {
|
||||
isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
|
||||
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
||||
@@ -52,6 +60,7 @@ function makeScheduler() {
|
||||
} as unknown as jest.Mocked<StaleAccountRepository>;
|
||||
const scheduler = new BackgroundRefreshScheduler(
|
||||
redis,
|
||||
distributedLockService,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
staleAccountRepository,
|
||||
@@ -60,6 +69,7 @@ function makeScheduler() {
|
||||
return {
|
||||
scheduler,
|
||||
redis,
|
||||
distributedLockService,
|
||||
rawClient,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
@@ -115,8 +125,9 @@ describe('BackgroundRefreshScheduler', () => {
|
||||
|
||||
it('does not query stale accounts when another leader holds the lock', async () => {
|
||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||
const { scheduler, redis, staleAccountRepository } = makeScheduler();
|
||||
redis.getString.mockResolvedValue('another-consumer');
|
||||
const { scheduler, distributedLockService, staleAccountRepository } =
|
||||
makeScheduler();
|
||||
distributedLockService.acquireLock.mockResolvedValueOnce(null);
|
||||
|
||||
await scheduler.tick();
|
||||
|
||||
|
||||
@@ -9,13 +9,17 @@ import {
|
||||
RiotAccountEnrichmentService,
|
||||
} from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
import {
|
||||
type DistributedLockOwnership,
|
||||
DistributedLockService,
|
||||
} from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { isInKstBackgroundWindow } from './kst-window.util';
|
||||
import { StaleAccountRepository } from './stale-account.repository';
|
||||
|
||||
const TICK_INTERVAL_MS = 60_000;
|
||||
const LEADER_KEY = 'riot:bg-refresh:leader';
|
||||
const LEADER_TTL_SECONDS = 90;
|
||||
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
|
||||
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
|
||||
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
||||
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
||||
const PER_TICK_CANDIDATE_LIMIT = 20;
|
||||
@@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler
|
||||
private tickTimer?: NodeJS.Timeout;
|
||||
private leaderRenewTimer?: NodeJS.Timeout;
|
||||
private tickInProgress = false;
|
||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly redis: RedisService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
private readonly rateLimiter: RiotRateLimiterService,
|
||||
private readonly enrichmentService: RiotAccountEnrichmentService,
|
||||
private readonly staleAccountRepository: StaleAccountRepository,
|
||||
@@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler
|
||||
void this.safeTick();
|
||||
}
|
||||
|
||||
public onModuleDestroy(): void {
|
||||
public async onModuleDestroy(): Promise<void> {
|
||||
if (this.tickTimer) clearInterval(this.tickTimer);
|
||||
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
||||
if (!this.leaderLockOwnership) return;
|
||||
await this.distributedLockService
|
||||
.releaseLockIfOwned(this.leaderLockOwnership)
|
||||
.catch((err: unknown) => {
|
||||
this.logger.warn(
|
||||
`Background refresh leader release failed: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
});
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
|
||||
private async safeTick(): Promise<void> {
|
||||
@@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler
|
||||
}
|
||||
|
||||
private async tryBecomeLeader(): Promise<boolean> {
|
||||
const current = await this.redis.getString(LEADER_KEY);
|
||||
if (current === this.consumer) {
|
||||
await this.redis.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
return true;
|
||||
if (this.leaderLockOwnership) {
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return true;
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
if (current) return false;
|
||||
|
||||
return this.redis.setStringIfAbsent(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
||||
lockKey: LEADER_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
});
|
||||
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
||||
return Boolean(leaderLockOwnership);
|
||||
}
|
||||
|
||||
private async renewLeaderIfHeld(): Promise<void> {
|
||||
try {
|
||||
const current = await this.redis.getString(LEADER_KEY);
|
||||
if (current !== this.consumer) return;
|
||||
await this.redis.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
if (!this.leaderLockOwnership) return;
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return;
|
||||
this.leaderLockOwnership = undefined;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Background refresh leader renew failed: ${
|
||||
|
||||
Reference in new issue
Block a user