fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -0,0 +1,199 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { RedisService } from './redis.service';
|
||||
|
||||
const RELEASE_LOCK_IF_OWNED_SCRIPT = `
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
const EXTEND_LOCK_IF_OWNED_SCRIPT = `
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
export interface DistributedLockOwnership {
|
||||
lockKey: string;
|
||||
lockOwnershipToken: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
}
|
||||
|
||||
export interface AcquireDistributedLockOptions {
|
||||
lockKey: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
lockOwnershipToken?: string;
|
||||
}
|
||||
|
||||
export interface RunWithDistributedLockOptions {
|
||||
lockKey: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
lockAcquireTimeoutMilliseconds?: number;
|
||||
lockRetryDelayMilliseconds?: number;
|
||||
}
|
||||
|
||||
export interface RunWithRenewedDistributedLockOptions
|
||||
extends RunWithDistributedLockOptions {
|
||||
lockRenewalIntervalMilliseconds: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DistributedLockService {
|
||||
private readonly logger = new Logger(DistributedLockService.name);
|
||||
|
||||
constructor(private readonly redisService: RedisService) {}
|
||||
|
||||
public async acquireLock(
|
||||
options: AcquireDistributedLockOptions,
|
||||
): Promise<DistributedLockOwnership | null> {
|
||||
this.assertPositiveDuration(
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
'lockExpirationDurationMilliseconds',
|
||||
);
|
||||
|
||||
const lockOwnershipToken = options.lockOwnershipToken ?? randomUUID();
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'SET',
|
||||
options.lockKey,
|
||||
lockOwnershipToken,
|
||||
'NX',
|
||||
'PX',
|
||||
String(options.lockExpirationDurationMilliseconds),
|
||||
]);
|
||||
|
||||
if (String(result) !== 'OK') return null;
|
||||
return {
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken,
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
};
|
||||
}
|
||||
|
||||
public async releaseLockIfOwned(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
): Promise<boolean> {
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'EVAL',
|
||||
RELEASE_LOCK_IF_OWNED_SCRIPT,
|
||||
'1',
|
||||
lockOwnership.lockKey,
|
||||
lockOwnership.lockOwnershipToken,
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
|
||||
public async extendLockIfOwned(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
lockExpirationDurationMilliseconds =
|
||||
lockOwnership.lockExpirationDurationMilliseconds,
|
||||
): Promise<boolean> {
|
||||
this.assertPositiveDuration(
|
||||
lockExpirationDurationMilliseconds,
|
||||
'lockExpirationDurationMilliseconds',
|
||||
);
|
||||
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'EVAL',
|
||||
EXTEND_LOCK_IF_OWNED_SCRIPT,
|
||||
'1',
|
||||
lockOwnership.lockKey,
|
||||
lockOwnership.lockOwnershipToken,
|
||||
String(lockExpirationDurationMilliseconds),
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
|
||||
public async runWithLock<TResult>(
|
||||
options: RunWithDistributedLockOptions,
|
||||
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||
): Promise<TResult | null> {
|
||||
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||
if (!lockOwnership) return null;
|
||||
|
||||
try {
|
||||
return await operation(lockOwnership);
|
||||
} finally {
|
||||
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public async runWithRenewedLock<TResult>(
|
||||
options: RunWithRenewedDistributedLockOptions,
|
||||
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||
): Promise<TResult | null> {
|
||||
this.assertPositiveDuration(
|
||||
options.lockRenewalIntervalMilliseconds,
|
||||
'lockRenewalIntervalMilliseconds',
|
||||
);
|
||||
|
||||
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||
if (!lockOwnership) return null;
|
||||
|
||||
let lockRenewalTimer: NodeJS.Timeout | undefined;
|
||||
lockRenewalTimer = setInterval(() => {
|
||||
void this.extendLockIfOwned(lockOwnership).then((wasExtended) => {
|
||||
if (wasExtended) return;
|
||||
this.logger.warn(
|
||||
`Lost distributed lock ownership before operation finished: ${lockOwnership.lockKey}`,
|
||||
);
|
||||
});
|
||||
}, options.lockRenewalIntervalMilliseconds);
|
||||
lockRenewalTimer.unref?.();
|
||||
|
||||
try {
|
||||
return await operation(lockOwnership);
|
||||
} finally {
|
||||
if (lockRenewalTimer) clearInterval(lockRenewalTimer);
|
||||
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async acquireLockWithRetry(
|
||||
options: RunWithDistributedLockOptions,
|
||||
): Promise<DistributedLockOwnership | null> {
|
||||
const lockAcquireTimeoutMilliseconds =
|
||||
options.lockAcquireTimeoutMilliseconds ?? 0;
|
||||
const lockRetryDelayMilliseconds =
|
||||
options.lockRetryDelayMilliseconds ?? 100;
|
||||
const acquisitionStartedAtTimestampMilliseconds = Date.now();
|
||||
|
||||
while (true) {
|
||||
const lockOwnership = await this.acquireLock(options);
|
||||
if (lockOwnership) return lockOwnership;
|
||||
|
||||
if (lockAcquireTimeoutMilliseconds <= 0) return null;
|
||||
const elapsedMilliseconds =
|
||||
Date.now() - acquisitionStartedAtTimestampMilliseconds;
|
||||
if (elapsedMilliseconds >= lockAcquireTimeoutMilliseconds) return null;
|
||||
|
||||
await this.sleep(
|
||||
Math.min(lockRetryDelayMilliseconds, lockAcquireTimeoutMilliseconds),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private assertPositiveDuration(value: number, fieldName: string): void {
|
||||
if (Number.isFinite(value) && value > 0) return;
|
||||
throw new Error(`${fieldName} must be a positive finite number.`);
|
||||
}
|
||||
|
||||
private sleep(durationMilliseconds: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user