fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -0,0 +1,121 @@
import { DistributedLockService } from './distributed-lock.service';
import { RedisService } from './redis.service';
function createService(sendCommand: jest.Mock) {
const redisService = {
raw: jest.fn().mockReturnValue({ sendCommand }),
} as unknown as RedisService;
return new DistributedLockService(redisService);
}
describe('DistributedLockService', () => {
it('acquires a lock with a token and millisecond expiration', async () => {
const sendCommand = jest.fn().mockResolvedValue('OK');
const service = createService(sendCommand);
const lockOwnership = await service.acquireLock({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
});
expect(lockOwnership).toEqual({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
});
expect(sendCommand).toHaveBeenCalledWith([
'SET',
'distributed-lock:test',
'owner-token',
'NX',
'PX',
'30000',
]);
});
it('returns null when another owner already holds the lock', async () => {
const sendCommand = jest.fn().mockResolvedValue(null);
const service = createService(sendCommand);
await expect(
service.acquireLock({
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
}),
).resolves.toBeNull();
});
it('releases a lock only when the ownership token matches', async () => {
const sendCommand = jest.fn().mockResolvedValue(1);
const service = createService(sendCommand);
await expect(
service.releaseLockIfOwned({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
}),
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'),
'1',
'distributed-lock:test',
'owner-token',
]);
});
it('extends a lock only when the ownership token matches', async () => {
const sendCommand = jest.fn().mockResolvedValue(1);
const service = createService(sendCommand);
await expect(
service.extendLockIfOwned(
{
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
},
45_000,
),
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('PEXPIRE'),
'1',
'distributed-lock:test',
'owner-token',
'45000',
]);
});
it('runs the operation and releases the lock afterward', async () => {
const sendCommand = jest
.fn()
.mockResolvedValueOnce('OK')
.mockResolvedValueOnce(1);
const service = createService(sendCommand);
const operation = jest.fn().mockResolvedValue('done');
await expect(
service.runWithLock(
{
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
},
operation,
),
).resolves.toBe('done');
expect(operation).toHaveBeenCalledWith(
expect.objectContaining({
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
}),
);
expect(sendCommand).toHaveBeenCalledTimes(2);
});
});