fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -42,4 +42,21 @@ describe('resolveMongoConnectionOptions', () => {
autoCreate: false,
});
});
it('enables automatic index creation when explicitly requested', () => {
expect(
resolveMongoConnectionOptions(
'lolympus-staging',
'lolympus-api:staging:1',
'true',
),
).toEqual({
dbName: 'lolympus-staging',
appName: 'lolympus-api:staging:1',
minPoolSize: 10,
maxPoolSize: 20,
autoIndex: true,
autoCreate: false,
});
});
});
@@ -19,13 +19,17 @@ export function resolveMongoAppName(
export function resolveMongoConnectionOptions(
dbName: string,
appName: string,
mongoAutoIndex = process.env.MONGODB_AUTO_INDEX,
): mongoose.ConnectOptions {
const shouldEnableMongoAutoIndex =
mongoAutoIndex?.trim().toLowerCase() === 'true';
return {
dbName,
appName,
minPoolSize: 10,
maxPoolSize: 20,
autoIndex: false,
autoIndex: shouldEnableMongoAutoIndex,
autoCreate: false,
};
}
@@ -392,6 +392,8 @@ export interface DakaoOutboxMessage extends MongooseTimestamps {
payload: Record<string, unknown>;
state: DakaoOutboxMessageState;
publishAttempts: number;
publishLeaseOwnerIdentifier?: string;
publishLeaseExpiresAt?: Date;
publishedAt?: Date;
terminalAt?: Date;
resultEventId?: string;
@@ -733,6 +735,7 @@ export interface LegacyImportJob extends MongooseTimestamps {
totalSkippedDuplicate: number;
totalErrors: number;
averageMemberSeconds?: number;
completedMemberJobIds: string[];
errors?: LegacyImportJobError[];
}
@@ -1282,6 +1285,8 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
default: 'pending',
},
publishAttempts: { type: Number, required: true, default: 0 },
publishLeaseOwnerIdentifier: { type: String },
publishLeaseExpiresAt: { type: Date },
publishedAt: { type: Date },
terminalAt: { type: Date },
resultEventId: { type: String },
@@ -1302,6 +1307,11 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
);
DakaoOutboxMessageSchema.index({ state: 1, updatedAt: 1 });
DakaoOutboxMessageSchema.index({
state: 1,
publishLeaseExpiresAt: 1,
updatedAt: 1,
});
DakaoOutboxMessageSchema.index({ correlationId: 1 }, { sparse: true });
export const DakaoSendResultEventSchema = new Schema<DakaoSendResultEvent>(
@@ -1988,6 +1998,11 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
totalSkippedDuplicate: { type: Number, required: true, default: 0 },
totalErrors: { type: Number, required: true, default: 0 },
averageMemberSeconds: { type: Number },
completedMemberJobIds: {
type: [String],
required: true,
default: [],
},
errors: {
type: [LegacyImportJobErrorSubSchema],
required: true,
@@ -2003,6 +2018,7 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
LegacyImportJobSchema.index({ communityId: 1, startedAt: -1 });
LegacyImportJobSchema.index({ communityId: 1, state: 1 });
LegacyImportJobSchema.index({ jobId: 1, completedMemberJobIds: 1 });
const AuctionPlayerSnapshotSubSchema = new Schema<AuctionPlayerSnapshot>(
{
@@ -56,20 +56,25 @@ export interface JetStreamConsumeOptions {
maxMessages?: number;
expiresMs?: number;
nakDelayMs?: number;
restartDelayMilliseconds?: number;
}
const JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS = 1_000;
@Injectable()
export class NatsService implements OnModuleDestroy {
private readonly logger = new Logger(NatsService.name);
private readonly stringCodec = StringCodec();
private readonly subscriptions = new Set<Subscription>();
private readonly jetStreamConsumers = new Set<ConsumerMessages>();
private isDestroying = false;
constructor(
@Inject(NATS_CLIENT) private readonly natsClient: NatsConnection,
) {}
async onModuleDestroy() {
this.isDestroying = true;
for (const consumer of this.jetStreamConsumers) {
try {
await consumer.close();
@@ -199,28 +204,36 @@ export class NatsService implements OnModuleDestroy {
options: JetStreamConsumeOptions,
handler: JetStreamMessageHandler<T>,
): Promise<void> {
try {
const consumer = await this.natsClient
.jetstream()
.consumers.get(options.stream, options.durable);
const messages = await consumer.consume({
max_messages: options.maxMessages ?? 10,
expires: options.expiresMs ?? 30000,
});
this.jetStreamConsumers.add(messages);
while (!this.isDestroying) {
try {
for await (const msg of messages) {
await this.dispatchJetStream(msg, handler, options);
const consumer = await this.natsClient
.jetstream()
.consumers.get(options.stream, options.durable);
const messages = await consumer.consume({
max_messages: options.maxMessages ?? 10,
expires: options.expiresMs ?? 30000,
});
this.jetStreamConsumers.add(messages);
try {
for await (const msg of messages) {
await this.dispatchJetStream(msg, handler, options);
}
} finally {
this.jetStreamConsumers.delete(messages);
}
} finally {
this.jetStreamConsumers.delete(messages);
} catch (err) {
if (this.isDestroying) return;
this.logger.error(
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
(err as Error)?.stack ?? err
}`,
);
}
} catch (err) {
this.logger.error(
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
(err as Error)?.stack ?? err
}`,
await this.sleep(
options.restartDelayMilliseconds ??
JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS,
);
}
}
@@ -313,4 +326,8 @@ export class NatsService implements OnModuleDestroy {
? RetentionPolicy.Workqueue
: RetentionPolicy.Limits;
}
private sleep(durationMilliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
}
}
@@ -0,0 +1,121 @@
import { DistributedLockService } from './distributed-lock.service';
import { RedisService } from './redis.service';
function createService(sendCommand: jest.Mock) {
const redisService = {
raw: jest.fn().mockReturnValue({ sendCommand }),
} as unknown as RedisService;
return new DistributedLockService(redisService);
}
describe('DistributedLockService', () => {
it('acquires a lock with a token and millisecond expiration', async () => {
const sendCommand = jest.fn().mockResolvedValue('OK');
const service = createService(sendCommand);
const lockOwnership = await service.acquireLock({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
});
expect(lockOwnership).toEqual({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
});
expect(sendCommand).toHaveBeenCalledWith([
'SET',
'distributed-lock:test',
'owner-token',
'NX',
'PX',
'30000',
]);
});
it('returns null when another owner already holds the lock', async () => {
const sendCommand = jest.fn().mockResolvedValue(null);
const service = createService(sendCommand);
await expect(
service.acquireLock({
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
}),
).resolves.toBeNull();
});
it('releases a lock only when the ownership token matches', async () => {
const sendCommand = jest.fn().mockResolvedValue(1);
const service = createService(sendCommand);
await expect(
service.releaseLockIfOwned({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
}),
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'),
'1',
'distributed-lock:test',
'owner-token',
]);
});
it('extends a lock only when the ownership token matches', async () => {
const sendCommand = jest.fn().mockResolvedValue(1);
const service = createService(sendCommand);
await expect(
service.extendLockIfOwned(
{
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
},
45_000,
),
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('PEXPIRE'),
'1',
'distributed-lock:test',
'owner-token',
'45000',
]);
});
it('runs the operation and releases the lock afterward', async () => {
const sendCommand = jest
.fn()
.mockResolvedValueOnce('OK')
.mockResolvedValueOnce(1);
const service = createService(sendCommand);
const operation = jest.fn().mockResolvedValue('done');
await expect(
service.runWithLock(
{
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
},
operation,
),
).resolves.toBe('done');
expect(operation).toHaveBeenCalledWith(
expect.objectContaining({
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
}),
);
expect(sendCommand).toHaveBeenCalledTimes(2);
});
});
@@ -0,0 +1,199 @@
import { Injectable, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { RedisService } from './redis.service';
const RELEASE_LOCK_IF_OWNED_SCRIPT = `
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("DEL", KEYS[1])
end
return 0
`;
const EXTEND_LOCK_IF_OWNED_SCRIPT = `
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
end
return 0
`;
export interface DistributedLockOwnership {
lockKey: string;
lockOwnershipToken: string;
lockExpirationDurationMilliseconds: number;
}
export interface AcquireDistributedLockOptions {
lockKey: string;
lockExpirationDurationMilliseconds: number;
lockOwnershipToken?: string;
}
export interface RunWithDistributedLockOptions {
lockKey: string;
lockExpirationDurationMilliseconds: number;
lockAcquireTimeoutMilliseconds?: number;
lockRetryDelayMilliseconds?: number;
}
export interface RunWithRenewedDistributedLockOptions
extends RunWithDistributedLockOptions {
lockRenewalIntervalMilliseconds: number;
}
@Injectable()
export class DistributedLockService {
private readonly logger = new Logger(DistributedLockService.name);
constructor(private readonly redisService: RedisService) {}
public async acquireLock(
options: AcquireDistributedLockOptions,
): Promise<DistributedLockOwnership | null> {
this.assertPositiveDuration(
options.lockExpirationDurationMilliseconds,
'lockExpirationDurationMilliseconds',
);
const lockOwnershipToken = options.lockOwnershipToken ?? randomUUID();
const result = await this.redisService.raw().sendCommand([
'SET',
options.lockKey,
lockOwnershipToken,
'NX',
'PX',
String(options.lockExpirationDurationMilliseconds),
]);
if (String(result) !== 'OK') return null;
return {
lockKey: options.lockKey,
lockOwnershipToken,
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
};
}
public async releaseLockIfOwned(
lockOwnership: DistributedLockOwnership,
): Promise<boolean> {
const result = await this.redisService.raw().sendCommand([
'EVAL',
RELEASE_LOCK_IF_OWNED_SCRIPT,
'1',
lockOwnership.lockKey,
lockOwnership.lockOwnershipToken,
]);
return Number(result) === 1;
}
public async extendLockIfOwned(
lockOwnership: DistributedLockOwnership,
lockExpirationDurationMilliseconds =
lockOwnership.lockExpirationDurationMilliseconds,
): Promise<boolean> {
this.assertPositiveDuration(
lockExpirationDurationMilliseconds,
'lockExpirationDurationMilliseconds',
);
const result = await this.redisService.raw().sendCommand([
'EVAL',
EXTEND_LOCK_IF_OWNED_SCRIPT,
'1',
lockOwnership.lockKey,
lockOwnership.lockOwnershipToken,
String(lockExpirationDurationMilliseconds),
]);
return Number(result) === 1;
}
public async runWithLock<TResult>(
options: RunWithDistributedLockOptions,
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
): Promise<TResult | null> {
const lockOwnership = await this.acquireLockWithRetry(options);
if (!lockOwnership) return null;
try {
return await operation(lockOwnership);
} finally {
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
this.logger.warn(
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
error instanceof Error ? error.message : error
}`,
);
});
}
}
public async runWithRenewedLock<TResult>(
options: RunWithRenewedDistributedLockOptions,
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
): Promise<TResult | null> {
this.assertPositiveDuration(
options.lockRenewalIntervalMilliseconds,
'lockRenewalIntervalMilliseconds',
);
const lockOwnership = await this.acquireLockWithRetry(options);
if (!lockOwnership) return null;
let lockRenewalTimer: NodeJS.Timeout | undefined;
lockRenewalTimer = setInterval(() => {
void this.extendLockIfOwned(lockOwnership).then((wasExtended) => {
if (wasExtended) return;
this.logger.warn(
`Lost distributed lock ownership before operation finished: ${lockOwnership.lockKey}`,
);
});
}, options.lockRenewalIntervalMilliseconds);
lockRenewalTimer.unref?.();
try {
return await operation(lockOwnership);
} finally {
if (lockRenewalTimer) clearInterval(lockRenewalTimer);
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
this.logger.warn(
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
error instanceof Error ? error.message : error
}`,
);
});
}
}
private async acquireLockWithRetry(
options: RunWithDistributedLockOptions,
): Promise<DistributedLockOwnership | null> {
const lockAcquireTimeoutMilliseconds =
options.lockAcquireTimeoutMilliseconds ?? 0;
const lockRetryDelayMilliseconds =
options.lockRetryDelayMilliseconds ?? 100;
const acquisitionStartedAtTimestampMilliseconds = Date.now();
while (true) {
const lockOwnership = await this.acquireLock(options);
if (lockOwnership) return lockOwnership;
if (lockAcquireTimeoutMilliseconds <= 0) return null;
const elapsedMilliseconds =
Date.now() - acquisitionStartedAtTimestampMilliseconds;
if (elapsedMilliseconds >= lockAcquireTimeoutMilliseconds) return null;
await this.sleep(
Math.min(lockRetryDelayMilliseconds, lockAcquireTimeoutMilliseconds),
);
}
}
private assertPositiveDuration(value: number, fieldName: string): void {
if (Number.isFinite(value) && value > 0) return;
throw new Error(`${fieldName} must be a positive finite number.`);
}
private sleep(durationMilliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
}
}
@@ -4,6 +4,7 @@ import { SecretModule } from '../secret/secret.module';
import { SecretService } from '../secret/secret.service';
import { REDIS_CLIENT } from './redis.constants';
import { resolveRedisClientOptions } from './redis.config';
import { DistributedLockService } from './distributed-lock.service';
import { RedisService, type RedisClient } from './redis.service';
async function readFirstSecret(
@@ -62,7 +63,8 @@ async function readFirstSecret(
},
},
RedisService,
DistributedLockService,
],
exports: [RedisService, REDIS_CLIENT],
exports: [RedisService, DistributedLockService, REDIS_CLIENT],
})
export class RedisModule {}
@@ -0,0 +1,52 @@
import { RedisService } from './redis.service';
function createRedisService(client: Record<string, unknown>) {
return new RedisService(client as never);
}
describe('RedisService stream pending recovery', () => {
it('normalizes XAUTOCLAIM array replies into stream entries', async () => {
const sendCommand = jest.fn().mockResolvedValue([
'0-0',
[
[
'1700000000000-0',
['job', '{"kind":"poll-tournament-result"}', 'source', 'test'],
],
],
]);
const service = createRedisService({ sendCommand });
await expect(
service.claimStalePendingStreamMessages({
key: 'riot:queue:high',
group: 'riot-refresh-worker',
consumer: 'worker-1',
stalePendingMessageMinimumIdleMilliseconds: 1_800_000,
count: 1,
}),
).resolves.toEqual({
nextStartId: '0-0',
messages: [
{
id: '1700000000000-0',
message: {
job: '{"kind":"poll-tournament-result"}',
source: 'test',
},
},
],
});
expect(sendCommand).toHaveBeenCalledWith([
'XAUTOCLAIM',
'riot:queue:high',
'riot-refresh-worker',
'worker-1',
'1800000',
'0-0',
'COUNT',
'1',
]);
});
});
@@ -16,6 +16,11 @@ export interface StreamReadResult {
name: string;
}
export interface ClaimStalePendingStreamMessagesResult {
messages: StreamEntry[];
nextStartId: string;
}
interface RawStreamMessage {
deliveriesCounter?: number;
id: string;
@@ -29,6 +34,16 @@ interface RawStreamReadResult {
}
function normalizeStreamFields(fields: unknown): Record<string, string> {
if (Array.isArray(fields)) {
const normalized: Record<string, string> = {};
for (let index = 0; index < fields.length; index += 2) {
const key = fields[index];
if (key === undefined) continue;
normalized[String(key)] = String(fields[index + 1] ?? '');
}
return normalized;
}
if (fields instanceof Map) {
return Object.fromEntries(
[...fields.entries()].map(([key, value]) => [String(key), String(value)]),
@@ -188,6 +203,38 @@ export class RedisService implements OnModuleDestroy {
}));
}
public async claimStalePendingStreamMessages(options: {
key: string;
group: string;
consumer: string;
stalePendingMessageMinimumIdleMilliseconds: number;
startId?: string;
count?: number;
}): Promise<ClaimStalePendingStreamMessagesResult> {
const response = (await this.client.sendCommand([
'XAUTOCLAIM',
options.key,
options.group,
options.consumer,
String(options.stalePendingMessageMinimumIdleMilliseconds),
options.startId ?? '0-0',
'COUNT',
String(options.count ?? 1),
])) as unknown;
const responseItems = Array.isArray(response) ? response : [];
const rawMessages = Array.isArray(responseItems[1])
? responseItems[1]
: [];
return {
nextStartId: String(responseItems[0] ?? '0-0'),
messages: rawMessages
.map((rawMessage) => this.normalizeClaimedStreamMessage(rawMessage))
.filter((message): message is StreamEntry => Boolean(message)),
};
}
public async acknowledgeStream(
key: string,
group: string,
@@ -210,4 +257,37 @@ export class RedisService implements OnModuleDestroy {
public async publish(channel: string, message: string): Promise<number> {
return this.client.publish(channel, message);
}
private normalizeClaimedStreamMessage(rawMessage: unknown): StreamEntry | null {
if (Array.isArray(rawMessage)) {
const [id, fields] = rawMessage;
if (id === undefined) return null;
return {
id: String(id),
message: normalizeStreamFields(fields),
};
}
if (
typeof rawMessage === 'object' &&
rawMessage !== null &&
'id' in rawMessage
) {
const message = rawMessage as RawStreamMessage;
return {
deliveriesCounter:
typeof message.deliveriesCounter === 'number'
? message.deliveriesCounter
: undefined,
id: String(message.id),
message: normalizeStreamFields(message.message),
millisElapsedFromDelivery:
typeof message.millisElapsedFromDelivery === 'number'
? message.millisElapsedFromDelivery
: undefined,
};
}
return null;
}
}