fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -0,0 +1,121 @@
|
||||
import { DistributedLockService } from './distributed-lock.service';
|
||||
import { RedisService } from './redis.service';
|
||||
|
||||
function createService(sendCommand: jest.Mock) {
|
||||
const redisService = {
|
||||
raw: jest.fn().mockReturnValue({ sendCommand }),
|
||||
} as unknown as RedisService;
|
||||
return new DistributedLockService(redisService);
|
||||
}
|
||||
|
||||
describe('DistributedLockService', () => {
|
||||
it('acquires a lock with a token and millisecond expiration', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue('OK');
|
||||
const service = createService(sendCommand);
|
||||
|
||||
const lockOwnership = await service.acquireLock({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
});
|
||||
|
||||
expect(lockOwnership).toEqual({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
});
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'SET',
|
||||
'distributed-lock:test',
|
||||
'owner-token',
|
||||
'NX',
|
||||
'PX',
|
||||
'30000',
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns null when another owner already holds the lock', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue(null);
|
||||
const service = createService(sendCommand);
|
||||
|
||||
await expect(
|
||||
service.acquireLock({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
}),
|
||||
).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('releases a lock only when the ownership token matches', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue(1);
|
||||
const service = createService(sendCommand);
|
||||
|
||||
await expect(
|
||||
service.releaseLockIfOwned({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'),
|
||||
'1',
|
||||
'distributed-lock:test',
|
||||
'owner-token',
|
||||
]);
|
||||
});
|
||||
|
||||
it('extends a lock only when the ownership token matches', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue(1);
|
||||
const service = createService(sendCommand);
|
||||
|
||||
await expect(
|
||||
service.extendLockIfOwned(
|
||||
{
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
},
|
||||
45_000,
|
||||
),
|
||||
).resolves.toBe(true);
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining('PEXPIRE'),
|
||||
'1',
|
||||
'distributed-lock:test',
|
||||
'owner-token',
|
||||
'45000',
|
||||
]);
|
||||
});
|
||||
|
||||
it('runs the operation and releases the lock afterward', async () => {
|
||||
const sendCommand = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce(1);
|
||||
const service = createService(sendCommand);
|
||||
const operation = jest.fn().mockResolvedValue('done');
|
||||
|
||||
await expect(
|
||||
service.runWithLock(
|
||||
{
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
},
|
||||
operation,
|
||||
),
|
||||
).resolves.toBe('done');
|
||||
|
||||
expect(operation).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
}),
|
||||
);
|
||||
expect(sendCommand).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,199 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { RedisService } from './redis.service';
|
||||
|
||||
const RELEASE_LOCK_IF_OWNED_SCRIPT = `
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
const EXTEND_LOCK_IF_OWNED_SCRIPT = `
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
export interface DistributedLockOwnership {
|
||||
lockKey: string;
|
||||
lockOwnershipToken: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
}
|
||||
|
||||
export interface AcquireDistributedLockOptions {
|
||||
lockKey: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
lockOwnershipToken?: string;
|
||||
}
|
||||
|
||||
export interface RunWithDistributedLockOptions {
|
||||
lockKey: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
lockAcquireTimeoutMilliseconds?: number;
|
||||
lockRetryDelayMilliseconds?: number;
|
||||
}
|
||||
|
||||
export interface RunWithRenewedDistributedLockOptions
|
||||
extends RunWithDistributedLockOptions {
|
||||
lockRenewalIntervalMilliseconds: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DistributedLockService {
|
||||
private readonly logger = new Logger(DistributedLockService.name);
|
||||
|
||||
constructor(private readonly redisService: RedisService) {}
|
||||
|
||||
public async acquireLock(
|
||||
options: AcquireDistributedLockOptions,
|
||||
): Promise<DistributedLockOwnership | null> {
|
||||
this.assertPositiveDuration(
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
'lockExpirationDurationMilliseconds',
|
||||
);
|
||||
|
||||
const lockOwnershipToken = options.lockOwnershipToken ?? randomUUID();
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'SET',
|
||||
options.lockKey,
|
||||
lockOwnershipToken,
|
||||
'NX',
|
||||
'PX',
|
||||
String(options.lockExpirationDurationMilliseconds),
|
||||
]);
|
||||
|
||||
if (String(result) !== 'OK') return null;
|
||||
return {
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken,
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
};
|
||||
}
|
||||
|
||||
public async releaseLockIfOwned(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
): Promise<boolean> {
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'EVAL',
|
||||
RELEASE_LOCK_IF_OWNED_SCRIPT,
|
||||
'1',
|
||||
lockOwnership.lockKey,
|
||||
lockOwnership.lockOwnershipToken,
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
|
||||
public async extendLockIfOwned(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
lockExpirationDurationMilliseconds =
|
||||
lockOwnership.lockExpirationDurationMilliseconds,
|
||||
): Promise<boolean> {
|
||||
this.assertPositiveDuration(
|
||||
lockExpirationDurationMilliseconds,
|
||||
'lockExpirationDurationMilliseconds',
|
||||
);
|
||||
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'EVAL',
|
||||
EXTEND_LOCK_IF_OWNED_SCRIPT,
|
||||
'1',
|
||||
lockOwnership.lockKey,
|
||||
lockOwnership.lockOwnershipToken,
|
||||
String(lockExpirationDurationMilliseconds),
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
|
||||
public async runWithLock<TResult>(
|
||||
options: RunWithDistributedLockOptions,
|
||||
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||
): Promise<TResult | null> {
|
||||
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||
if (!lockOwnership) return null;
|
||||
|
||||
try {
|
||||
return await operation(lockOwnership);
|
||||
} finally {
|
||||
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public async runWithRenewedLock<TResult>(
|
||||
options: RunWithRenewedDistributedLockOptions,
|
||||
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||
): Promise<TResult | null> {
|
||||
this.assertPositiveDuration(
|
||||
options.lockRenewalIntervalMilliseconds,
|
||||
'lockRenewalIntervalMilliseconds',
|
||||
);
|
||||
|
||||
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||
if (!lockOwnership) return null;
|
||||
|
||||
let lockRenewalTimer: NodeJS.Timeout | undefined;
|
||||
lockRenewalTimer = setInterval(() => {
|
||||
void this.extendLockIfOwned(lockOwnership).then((wasExtended) => {
|
||||
if (wasExtended) return;
|
||||
this.logger.warn(
|
||||
`Lost distributed lock ownership before operation finished: ${lockOwnership.lockKey}`,
|
||||
);
|
||||
});
|
||||
}, options.lockRenewalIntervalMilliseconds);
|
||||
lockRenewalTimer.unref?.();
|
||||
|
||||
try {
|
||||
return await operation(lockOwnership);
|
||||
} finally {
|
||||
if (lockRenewalTimer) clearInterval(lockRenewalTimer);
|
||||
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async acquireLockWithRetry(
|
||||
options: RunWithDistributedLockOptions,
|
||||
): Promise<DistributedLockOwnership | null> {
|
||||
const lockAcquireTimeoutMilliseconds =
|
||||
options.lockAcquireTimeoutMilliseconds ?? 0;
|
||||
const lockRetryDelayMilliseconds =
|
||||
options.lockRetryDelayMilliseconds ?? 100;
|
||||
const acquisitionStartedAtTimestampMilliseconds = Date.now();
|
||||
|
||||
while (true) {
|
||||
const lockOwnership = await this.acquireLock(options);
|
||||
if (lockOwnership) return lockOwnership;
|
||||
|
||||
if (lockAcquireTimeoutMilliseconds <= 0) return null;
|
||||
const elapsedMilliseconds =
|
||||
Date.now() - acquisitionStartedAtTimestampMilliseconds;
|
||||
if (elapsedMilliseconds >= lockAcquireTimeoutMilliseconds) return null;
|
||||
|
||||
await this.sleep(
|
||||
Math.min(lockRetryDelayMilliseconds, lockAcquireTimeoutMilliseconds),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private assertPositiveDuration(value: number, fieldName: string): void {
|
||||
if (Number.isFinite(value) && value > 0) return;
|
||||
throw new Error(`${fieldName} must be a positive finite number.`);
|
||||
}
|
||||
|
||||
private sleep(durationMilliseconds: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { SecretModule } from '../secret/secret.module';
|
||||
import { SecretService } from '../secret/secret.service';
|
||||
import { REDIS_CLIENT } from './redis.constants';
|
||||
import { resolveRedisClientOptions } from './redis.config';
|
||||
import { DistributedLockService } from './distributed-lock.service';
|
||||
import { RedisService, type RedisClient } from './redis.service';
|
||||
|
||||
async function readFirstSecret(
|
||||
@@ -62,7 +63,8 @@ async function readFirstSecret(
|
||||
},
|
||||
},
|
||||
RedisService,
|
||||
DistributedLockService,
|
||||
],
|
||||
exports: [RedisService, REDIS_CLIENT],
|
||||
exports: [RedisService, DistributedLockService, REDIS_CLIENT],
|
||||
})
|
||||
export class RedisModule {}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { RedisService } from './redis.service';
|
||||
|
||||
function createRedisService(client: Record<string, unknown>) {
|
||||
return new RedisService(client as never);
|
||||
}
|
||||
|
||||
describe('RedisService stream pending recovery', () => {
|
||||
it('normalizes XAUTOCLAIM array replies into stream entries', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue([
|
||||
'0-0',
|
||||
[
|
||||
[
|
||||
'1700000000000-0',
|
||||
['job', '{"kind":"poll-tournament-result"}', 'source', 'test'],
|
||||
],
|
||||
],
|
||||
]);
|
||||
const service = createRedisService({ sendCommand });
|
||||
|
||||
await expect(
|
||||
service.claimStalePendingStreamMessages({
|
||||
key: 'riot:queue:high',
|
||||
group: 'riot-refresh-worker',
|
||||
consumer: 'worker-1',
|
||||
stalePendingMessageMinimumIdleMilliseconds: 1_800_000,
|
||||
count: 1,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
nextStartId: '0-0',
|
||||
messages: [
|
||||
{
|
||||
id: '1700000000000-0',
|
||||
message: {
|
||||
job: '{"kind":"poll-tournament-result"}',
|
||||
source: 'test',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'XAUTOCLAIM',
|
||||
'riot:queue:high',
|
||||
'riot-refresh-worker',
|
||||
'worker-1',
|
||||
'1800000',
|
||||
'0-0',
|
||||
'COUNT',
|
||||
'1',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,11 @@ export interface StreamReadResult {
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface ClaimStalePendingStreamMessagesResult {
|
||||
messages: StreamEntry[];
|
||||
nextStartId: string;
|
||||
}
|
||||
|
||||
interface RawStreamMessage {
|
||||
deliveriesCounter?: number;
|
||||
id: string;
|
||||
@@ -29,6 +34,16 @@ interface RawStreamReadResult {
|
||||
}
|
||||
|
||||
function normalizeStreamFields(fields: unknown): Record<string, string> {
|
||||
if (Array.isArray(fields)) {
|
||||
const normalized: Record<string, string> = {};
|
||||
for (let index = 0; index < fields.length; index += 2) {
|
||||
const key = fields[index];
|
||||
if (key === undefined) continue;
|
||||
normalized[String(key)] = String(fields[index + 1] ?? '');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (fields instanceof Map) {
|
||||
return Object.fromEntries(
|
||||
[...fields.entries()].map(([key, value]) => [String(key), String(value)]),
|
||||
@@ -188,6 +203,38 @@ export class RedisService implements OnModuleDestroy {
|
||||
}));
|
||||
}
|
||||
|
||||
public async claimStalePendingStreamMessages(options: {
|
||||
key: string;
|
||||
group: string;
|
||||
consumer: string;
|
||||
stalePendingMessageMinimumIdleMilliseconds: number;
|
||||
startId?: string;
|
||||
count?: number;
|
||||
}): Promise<ClaimStalePendingStreamMessagesResult> {
|
||||
const response = (await this.client.sendCommand([
|
||||
'XAUTOCLAIM',
|
||||
options.key,
|
||||
options.group,
|
||||
options.consumer,
|
||||
String(options.stalePendingMessageMinimumIdleMilliseconds),
|
||||
options.startId ?? '0-0',
|
||||
'COUNT',
|
||||
String(options.count ?? 1),
|
||||
])) as unknown;
|
||||
|
||||
const responseItems = Array.isArray(response) ? response : [];
|
||||
const rawMessages = Array.isArray(responseItems[1])
|
||||
? responseItems[1]
|
||||
: [];
|
||||
|
||||
return {
|
||||
nextStartId: String(responseItems[0] ?? '0-0'),
|
||||
messages: rawMessages
|
||||
.map((rawMessage) => this.normalizeClaimedStreamMessage(rawMessage))
|
||||
.filter((message): message is StreamEntry => Boolean(message)),
|
||||
};
|
||||
}
|
||||
|
||||
public async acknowledgeStream(
|
||||
key: string,
|
||||
group: string,
|
||||
@@ -210,4 +257,37 @@ export class RedisService implements OnModuleDestroy {
|
||||
public async publish(channel: string, message: string): Promise<number> {
|
||||
return this.client.publish(channel, message);
|
||||
}
|
||||
|
||||
private normalizeClaimedStreamMessage(rawMessage: unknown): StreamEntry | null {
|
||||
if (Array.isArray(rawMessage)) {
|
||||
const [id, fields] = rawMessage;
|
||||
if (id === undefined) return null;
|
||||
return {
|
||||
id: String(id),
|
||||
message: normalizeStreamFields(fields),
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
typeof rawMessage === 'object' &&
|
||||
rawMessage !== null &&
|
||||
'id' in rawMessage
|
||||
) {
|
||||
const message = rawMessage as RawStreamMessage;
|
||||
return {
|
||||
deliveriesCounter:
|
||||
typeof message.deliveriesCounter === 'number'
|
||||
? message.deliveriesCounter
|
||||
: undefined,
|
||||
id: String(message.id),
|
||||
message: normalizeStreamFields(message.message),
|
||||
millisElapsedFromDelivery:
|
||||
typeof message.millisElapsedFromDelivery === 'number'
|
||||
? message.millisElapsedFromDelivery
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user