fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -1,5 +1,6 @@
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
@@ -19,6 +20,16 @@ function makeWorker() {
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RedisService>;
const distributedLockService = {
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
} as unknown as jest.Mocked<DistributedLockService>;
const jobQueueService = {
markProcessing: jest.fn().mockResolvedValue(undefined),
markCompleted: jest.fn().mockResolvedValue({
@@ -54,6 +65,7 @@ function makeWorker() {
const worker = new RiotJobWorkerService(
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
@@ -66,6 +78,7 @@ function makeWorker() {
return {
worker,
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
@@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => {
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
);
});
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
const { worker, distributedLockService, redisService } = makeWorker();
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
const userId = new mongoose.Types.ObjectId().toString();
const job: RiotRefreshSubJob = {
jobId: 'job-1',
parentJobId: 'parent-1',
userId,
puuid: 'puuid-1',
region: 'asia',
platform: 'kr',
queueGroup: 'summoners-rift',
queueIds: [420],
requestedAt: Date.now(),
priority: 'high',
};
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: '1-0',
message: { job: JSON.stringify(job) },
});
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
});
});
@@ -6,6 +6,10 @@ import {
} from '@nestjs/common';
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import {
DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import { RedisService, StreamEntry } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
@@ -23,17 +27,24 @@ import {
import { RiotJobQueueService } from './riot-job-queue.service';
import { RiotRefreshGateway } from './riot-refresh.gateway';
const LEADER_KEY = 'riot:worker:leader';
const LEADER_TTL_SECONDS = 60;
const RIOT_JOB_WORKER_LEADER_LOCK_KEY =
'distributed-lock:riot-job-worker:leader';
const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000;
const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000;
const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000;
const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1;
@Injectable()
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(RiotJobWorkerService.name);
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private leaderLockOwnership: DistributedLockOwnership | undefined;
private stopped = false;
constructor(
private readonly redisService: RedisService,
private readonly distributedLockService: DistributedLockService,
private readonly jobQueueService: RiotJobQueueService,
private readonly recentMatchesService: RiotRecentMatchesService,
private readonly connectionService: ConnectionService,
@@ -48,8 +59,19 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
void this.loop();
}
public onModuleDestroy() {
public async onModuleDestroy() {
this.stopped = true;
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
.catch((error: unknown) => {
this.logger.warn(
`Failed to release Riot job worker leader lock: ${
error instanceof Error ? error.message : error
}`,
);
});
this.leaderLockOwnership = undefined;
}
private async loop(): Promise<void> {
@@ -63,7 +85,9 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
await this.jobQueueService.promoteDueScheduledJobs();
const entry =
(await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ??
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
(await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ??
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
if (!entry) continue;
@@ -89,6 +113,31 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
return;
}
const didProcessJob = await this.distributedLockService.runWithRenewedLock(
{
lockKey: this.jobProcessingLockKey(job),
lockExpirationDurationMilliseconds:
RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.processJob(job);
return true;
},
);
if (!didProcessJob) {
this.logger.debug(
`Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`,
);
return;
}
await this.acknowledgeAndDelete(stream, entry.id);
}
private async processJob(job: RiotJobPayload): Promise<void> {
try {
if (this.isTournamentResultJob(job)) {
await this.tournamentResultService.processPollJob(job);
@@ -136,8 +185,6 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
error instanceof Error ? error.message : error
}`,
);
} finally {
await this.acknowledgeAndDelete(stream, entry.id);
}
}
@@ -228,15 +275,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.readGroup(
stream,
RIOT_JOB_GROUP,
this.consumer,
'>',
{ count: 1, blockMs },
RIOT_JOB_GROUP,
this.consumerIdentifier,
'>',
{ count: 1, blockMs },
);
const message = result?.[0]?.messages[0];
return message ? { stream, message } : null;
}
private async readOneStalePendingMessage(
stream: string,
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.claimStalePendingStreamMessages({
key: stream,
group: RIOT_JOB_GROUP,
consumer: this.consumerIdentifier,
stalePendingMessageMinimumIdleMilliseconds:
STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
count: STALE_PENDING_MESSAGE_CLAIM_COUNT,
});
const message = result.messages[0];
return message ? { stream, message } : null;
}
private async ensureConsumerGroups(): Promise<void> {
await Promise.all(
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
@@ -252,21 +314,26 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
}
private async tryBecomeLeader(): Promise<boolean> {
const current = await this.redisService.getString(LEADER_KEY);
if (current === this.consumer) {
await this.redisService.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
return true;
if (this.leaderLockOwnership) {
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return true;
this.leaderLockOwnership = undefined;
}
if (current) return false;
return this.redisService.setStringIfAbsent(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
const leaderLockOwnership = await this.distributedLockService.acquireLock({
lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY,
lockExpirationDurationMilliseconds:
RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
});
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
return Boolean(leaderLockOwnership);
}
private jobProcessingLockKey(job: RiotJobPayload): string {
return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`;
}
private sleep(ms: number): Promise<void> {