fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -1,5 +1,6 @@
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
@@ -19,6 +20,16 @@ function makeWorker() {
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RedisService>;
const distributedLockService = {
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
} as unknown as jest.Mocked<DistributedLockService>;
const jobQueueService = {
markProcessing: jest.fn().mockResolvedValue(undefined),
markCompleted: jest.fn().mockResolvedValue({
@@ -54,6 +65,7 @@ function makeWorker() {
const worker = new RiotJobWorkerService(
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
@@ -66,6 +78,7 @@ function makeWorker() {
return {
worker,
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
@@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => {
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
);
});
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
const { worker, distributedLockService, redisService } = makeWorker();
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
const userId = new mongoose.Types.ObjectId().toString();
const job: RiotRefreshSubJob = {
jobId: 'job-1',
parentJobId: 'parent-1',
userId,
puuid: 'puuid-1',
region: 'asia',
platform: 'kr',
queueGroup: 'summoners-rift',
queueIds: [420],
requestedAt: Date.now(),
priority: 'high',
};
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: '1-0',
message: { job: JSON.stringify(job) },
});
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
});
});