fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -6,6 +6,10 @@ import {
} from '@nestjs/common';
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import {
DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import { RedisService, StreamEntry } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
@@ -23,17 +27,24 @@ import {
import { RiotJobQueueService } from './riot-job-queue.service';
import { RiotRefreshGateway } from './riot-refresh.gateway';
const LEADER_KEY = 'riot:worker:leader';
const LEADER_TTL_SECONDS = 60;
const RIOT_JOB_WORKER_LEADER_LOCK_KEY =
'distributed-lock:riot-job-worker:leader';
const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000;
const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000;
const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000;
const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1;
@Injectable()
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(RiotJobWorkerService.name);
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private leaderLockOwnership: DistributedLockOwnership | undefined;
private stopped = false;
constructor(
private readonly redisService: RedisService,
private readonly distributedLockService: DistributedLockService,
private readonly jobQueueService: RiotJobQueueService,
private readonly recentMatchesService: RiotRecentMatchesService,
private readonly connectionService: ConnectionService,
@@ -48,8 +59,19 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
void this.loop();
}
public onModuleDestroy() {
public async onModuleDestroy() {
this.stopped = true;
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
.catch((error: unknown) => {
this.logger.warn(
`Failed to release Riot job worker leader lock: ${
error instanceof Error ? error.message : error
}`,
);
});
this.leaderLockOwnership = undefined;
}
private async loop(): Promise<void> {
@@ -63,7 +85,9 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
await this.jobQueueService.promoteDueScheduledJobs();
const entry =
(await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ??
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
(await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ??
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
if (!entry) continue;
@@ -89,6 +113,31 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
return;
}
const didProcessJob = await this.distributedLockService.runWithRenewedLock(
{
lockKey: this.jobProcessingLockKey(job),
lockExpirationDurationMilliseconds:
RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.processJob(job);
return true;
},
);
if (!didProcessJob) {
this.logger.debug(
`Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`,
);
return;
}
await this.acknowledgeAndDelete(stream, entry.id);
}
private async processJob(job: RiotJobPayload): Promise<void> {
try {
if (this.isTournamentResultJob(job)) {
await this.tournamentResultService.processPollJob(job);
@@ -136,8 +185,6 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
error instanceof Error ? error.message : error
}`,
);
} finally {
await this.acknowledgeAndDelete(stream, entry.id);
}
}
@@ -228,15 +275,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.readGroup(
stream,
RIOT_JOB_GROUP,
this.consumer,
'>',
{ count: 1, blockMs },
RIOT_JOB_GROUP,
this.consumerIdentifier,
'>',
{ count: 1, blockMs },
);
const message = result?.[0]?.messages[0];
return message ? { stream, message } : null;
}
private async readOneStalePendingMessage(
stream: string,
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.claimStalePendingStreamMessages({
key: stream,
group: RIOT_JOB_GROUP,
consumer: this.consumerIdentifier,
stalePendingMessageMinimumIdleMilliseconds:
STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
count: STALE_PENDING_MESSAGE_CLAIM_COUNT,
});
const message = result.messages[0];
return message ? { stream, message } : null;
}
private async ensureConsumerGroups(): Promise<void> {
await Promise.all(
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
@@ -252,21 +314,26 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
}
private async tryBecomeLeader(): Promise<boolean> {
const current = await this.redisService.getString(LEADER_KEY);
if (current === this.consumer) {
await this.redisService.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
return true;
if (this.leaderLockOwnership) {
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return true;
this.leaderLockOwnership = undefined;
}
if (current) return false;
return this.redisService.setStringIfAbsent(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
const leaderLockOwnership = await this.distributedLockService.acquireLock({
lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY,
lockExpirationDurationMilliseconds:
RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
});
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
return Boolean(leaderLockOwnership);
}
private jobProcessingLockKey(job: RiotJobPayload): string {
return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`;
}
private sleep(ms: number): Promise<void> {