fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||
@@ -19,6 +20,16 @@ function makeWorker() {
|
||||
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
||||
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
|
||||
} as unknown as jest.Mocked<RedisService>;
|
||||
const distributedLockService = {
|
||||
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||
operation({
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
}),
|
||||
),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const jobQueueService = {
|
||||
markProcessing: jest.fn().mockResolvedValue(undefined),
|
||||
markCompleted: jest.fn().mockResolvedValue({
|
||||
@@ -54,6 +65,7 @@ function makeWorker() {
|
||||
|
||||
const worker = new RiotJobWorkerService(
|
||||
redisService,
|
||||
distributedLockService,
|
||||
jobQueueService,
|
||||
recentMatchesService,
|
||||
connectionService,
|
||||
@@ -66,6 +78,7 @@ function makeWorker() {
|
||||
return {
|
||||
worker,
|
||||
redisService,
|
||||
distributedLockService,
|
||||
jobQueueService,
|
||||
recentMatchesService,
|
||||
connectionService,
|
||||
@@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => {
|
||||
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
|
||||
const { worker, distributedLockService, redisService } = makeWorker();
|
||||
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
const job: RiotRefreshSubJob = {
|
||||
jobId: 'job-1',
|
||||
parentJobId: 'parent-1',
|
||||
userId,
|
||||
puuid: 'puuid-1',
|
||||
region: 'asia',
|
||||
platform: 'kr',
|
||||
queueGroup: 'summoners-rift',
|
||||
queueIds: [420],
|
||||
requestedAt: Date.now(),
|
||||
priority: 'high',
|
||||
};
|
||||
|
||||
await (
|
||||
worker as unknown as {
|
||||
processEntry: (
|
||||
stream: string,
|
||||
entry: { id: string; message: Record<string, string> },
|
||||
) => Promise<void>;
|
||||
}
|
||||
).processEntry('riot:queue:high', {
|
||||
id: '1-0',
|
||||
message: { job: JSON.stringify(job) },
|
||||
});
|
||||
|
||||
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
|
||||
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user