fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -4,10 +4,13 @@ import {
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { DakaoOutboxService } from './dakao-outbox.service';
const SWEEP_INTERVAL_MS = 60 * 1000;
const STALE_OUTBOX_MS = 5 * 60 * 1000;
const SWEEP_LOCK_KEY = 'distributed-lock:dakao-outbox:sweep-stale-publish-leases';
const SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS = 55_000;
@Injectable()
export class DakaoOutboxSweeperService
@@ -16,7 +19,10 @@ export class DakaoOutboxSweeperService
private readonly logger = new Logger(DakaoOutboxSweeperService.name);
private timer: NodeJS.Timeout | undefined;
constructor(private readonly outboxService: DakaoOutboxService) {}
constructor(
private readonly outboxService: DakaoOutboxService,
private readonly distributedLockService: DistributedLockService,
) {}
onModuleInit() {
this.timer = setInterval(() => {
@@ -31,12 +37,23 @@ export class DakaoOutboxSweeperService
private async sweep(): Promise<void> {
try {
const swept = await this.outboxService.sweepStale({
staleMs: STALE_OUTBOX_MS,
limit: 20,
});
if (swept > 0) {
this.logger.warn(`Re-published ${swept} stale Dakao outbox messages.`);
const swept = await this.distributedLockService.runWithLock(
{
lockKey: SWEEP_LOCK_KEY,
lockExpirationDurationMilliseconds:
SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS,
},
async () =>
this.outboxService.sweepStale({
staleMs: STALE_OUTBOX_MS,
limit: 20,
}),
);
const sweptCount = swept ?? 0;
if (sweptCount > 0) {
this.logger.warn(
`Re-published ${sweptCount} stale Dakao outbox messages.`,
);
}
} catch (err) {
this.logger.error(
@@ -70,44 +70,55 @@ describe('DakaoOutboxService', () => {
process.env.NODE_ENV = 'staging';
try {
const exec = jest.fn().mockResolvedValue([
{
eventId: 'lolympus:chat.send-message:chat-log-1',
type: 'chat.send-message',
eventSource: 'lolympus',
subject: 'staging.dakao.chat.send-message.v1',
correlationId: 'chat-log-1',
payload: {
chatroomId: '18474390111480678',
segments: [{ kind: 'text', text: 'done' }],
},
const staleRow = {
eventId: 'lolympus:chat.send-message:chat-log-1',
type: 'chat.send-message',
eventSource: 'lolympus',
subject: 'staging.dakao.chat.send-message.v1',
correlationId: 'chat-log-1',
payload: {
chatroomId: '18474390111480678',
segments: [{ kind: 'text', text: 'done' }],
},
]);
};
const exec = jest.fn().mockResolvedValue([staleRow]);
const lean = jest.fn().mockReturnValue({ exec });
const limit = jest.fn().mockReturnValue({ lean });
const sort = jest.fn().mockReturnValue({ limit });
const claimExec = jest.fn().mockResolvedValue(staleRow);
const claimLean = jest.fn().mockReturnValue({ exec: claimExec });
const outboxModel = {
find: jest.fn().mockReturnValue({ sort }),
findOneAndUpdate: jest.fn().mockReturnValue({ lean: claimLean }),
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
};
const { service } = makeService(outboxModel);
const enqueue = jest
.spyOn(service, 'enqueueAndPublish')
.mockResolvedValue(undefined);
const { service, natsService } = makeService(outboxModel);
await expect(service.sweepStale({ staleMs: 1, limit: 1 })).resolves.toBe(
1,
);
expect(enqueue).toHaveBeenCalledWith(
expect(outboxModel.findOneAndUpdate).toHaveBeenCalledWith(
expect.objectContaining({
envelope: expect.objectContaining({
resultSubjects: {
succeeded:
'staging.lolympus.dakao.chat.send-message.succeeded.v1',
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
},
eventId: 'lolympus:chat.send-message:chat-log-1',
}),
expect.objectContaining({
$set: expect.objectContaining({
publishLeaseOwnerIdentifier: expect.any(String),
publishLeaseExpiresAt: expect.any(Date),
}),
}),
{ returnDocument: 'after' },
);
expect(natsService.publishJetStream).toHaveBeenCalledWith(
'staging.dakao.chat.send-message.v1',
expect.objectContaining({
resultSubjects: {
succeeded: 'staging.lolympus.dakao.chat.send-message.succeeded.v1',
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
},
}),
{ messageId: 'lolympus:chat.send-message:chat-log-1' },
);
} finally {
process.env.NODE_ENV = previousNodeEnv;
@@ -1,4 +1,5 @@
import { Injectable, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { ConnectionService } from '../global/database/connection.service';
import { NatsService } from '../global/nats/nats.service';
import { resolveDakaoNatsNames } from './dakao.nats-names';
@@ -10,6 +11,7 @@ import {
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
const TERMINAL_STATES = ['sent', 'failed'] as const;
const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000;
@Injectable()
export class DakaoOutboxService {
@@ -50,35 +52,59 @@ export class DakaoOutboxService {
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
).exec();
await this.publishCommand({
subject: params.subject,
eventId: params.eventId,
envelope: params.envelope,
});
}
private async publishCommand<TPayload>(params: {
subject: string;
eventId: string;
envelope: DakaoCommandEnvelope<TPayload>;
publishLeaseOwnerIdentifier?: string;
}): Promise<void> {
const OutboxModel =
await this.connectionService.getModel('dakaoOutboxMessage');
const updateFilter = {
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
...(params.publishLeaseOwnerIdentifier
? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier }
: {}),
};
try {
await this.natsService.publishJetStream(params.subject, params.envelope, {
messageId: params.eventId,
});
await OutboxModel.updateOne(
{
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
},
updateFilter,
{
$set: {
state: 'published',
publishedAt: new Date(),
},
$inc: { publishAttempts: 1 },
$unset: { lastPublishError: '' },
$unset: {
lastPublishError: '',
publishLeaseOwnerIdentifier: '',
publishLeaseExpiresAt: '',
},
},
).exec();
} catch (err) {
await OutboxModel.updateOne(
{
eventId: params.eventId,
state: { $nin: TERMINAL_STATES },
},
updateFilter,
{
$inc: { publishAttempts: 1 },
$set: {
lastPublishError: (err as Error)?.message ?? String(err),
},
$unset: {
publishLeaseOwnerIdentifier: '',
publishLeaseExpiresAt: '',
},
},
).exec();
this.logger.error(
@@ -146,40 +172,77 @@ export class DakaoOutboxService {
public async sweepStale(params?: {
staleMs?: number;
publishLeaseDurationMilliseconds?: number;
limit?: number;
}): Promise<number> {
const OutboxModel =
await this.connectionService.getModel('dakaoOutboxMessage');
const currentTimestamp = new Date();
const staleBefore = new Date(
Date.now() - (params?.staleMs ?? 5 * 60 * 1000),
currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000),
);
const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`;
const publishLeaseExpiresAt = new Date(
currentTimestamp.getTime() +
(params?.publishLeaseDurationMilliseconds ??
DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS),
);
const rows = await OutboxModel.find({
state: { $in: ['pending', 'published'] },
updatedAt: { $lte: staleBefore },
$or: [
{ publishLeaseExpiresAt: { $exists: false } },
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
],
})
.sort({ updatedAt: 1 })
.limit(params?.limit ?? 20)
.lean()
.exec();
let swept = 0;
for (const row of rows) {
await this.enqueueAndPublish({
subject: row.subject,
eventId: row.eventId,
const claimedRow = await OutboxModel.findOneAndUpdate(
{
eventId: row.eventId,
state: { $in: ['pending', 'published'] },
$or: [
{ publishLeaseExpiresAt: { $exists: false } },
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
],
},
{
$set: {
publishLeaseOwnerIdentifier,
publishLeaseExpiresAt,
},
},
{ returnDocument: 'after' },
)
.lean()
.exec();
if (!claimedRow) continue;
await this.publishCommand({
subject: claimedRow.subject,
eventId: claimedRow.eventId,
publishLeaseOwnerIdentifier,
envelope: {
schema: 'dakao.command.v1',
eventId: row.eventId,
type: row.type,
eventSource: row.eventSource,
correlationId: row.correlationId,
eventId: claimedRow.eventId,
type: claimedRow.type,
eventSource: claimedRow.eventSource,
correlationId: claimedRow.correlationId,
resultSubjects:
row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type),
payload: row.payload,
claimedRow.resultSubjects ??
this.resolveDefaultResultSubjects(claimedRow.type),
payload: claimedRow.payload,
},
});
swept += 1;
}
return rows.length;
return swept;
}
private resolveDefaultResultSubjects(
@@ -42,4 +42,21 @@ describe('resolveMongoConnectionOptions', () => {
autoCreate: false,
});
});
it('enables automatic index creation when explicitly requested', () => {
expect(
resolveMongoConnectionOptions(
'lolympus-staging',
'lolympus-api:staging:1',
'true',
),
).toEqual({
dbName: 'lolympus-staging',
appName: 'lolympus-api:staging:1',
minPoolSize: 10,
maxPoolSize: 20,
autoIndex: true,
autoCreate: false,
});
});
});
@@ -19,13 +19,17 @@ export function resolveMongoAppName(
export function resolveMongoConnectionOptions(
dbName: string,
appName: string,
mongoAutoIndex = process.env.MONGODB_AUTO_INDEX,
): mongoose.ConnectOptions {
const shouldEnableMongoAutoIndex =
mongoAutoIndex?.trim().toLowerCase() === 'true';
return {
dbName,
appName,
minPoolSize: 10,
maxPoolSize: 20,
autoIndex: false,
autoIndex: shouldEnableMongoAutoIndex,
autoCreate: false,
};
}
@@ -392,6 +392,8 @@ export interface DakaoOutboxMessage extends MongooseTimestamps {
payload: Record<string, unknown>;
state: DakaoOutboxMessageState;
publishAttempts: number;
publishLeaseOwnerIdentifier?: string;
publishLeaseExpiresAt?: Date;
publishedAt?: Date;
terminalAt?: Date;
resultEventId?: string;
@@ -733,6 +735,7 @@ export interface LegacyImportJob extends MongooseTimestamps {
totalSkippedDuplicate: number;
totalErrors: number;
averageMemberSeconds?: number;
completedMemberJobIds: string[];
errors?: LegacyImportJobError[];
}
@@ -1282,6 +1285,8 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
default: 'pending',
},
publishAttempts: { type: Number, required: true, default: 0 },
publishLeaseOwnerIdentifier: { type: String },
publishLeaseExpiresAt: { type: Date },
publishedAt: { type: Date },
terminalAt: { type: Date },
resultEventId: { type: String },
@@ -1302,6 +1307,11 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
);
DakaoOutboxMessageSchema.index({ state: 1, updatedAt: 1 });
DakaoOutboxMessageSchema.index({
state: 1,
publishLeaseExpiresAt: 1,
updatedAt: 1,
});
DakaoOutboxMessageSchema.index({ correlationId: 1 }, { sparse: true });
export const DakaoSendResultEventSchema = new Schema<DakaoSendResultEvent>(
@@ -1988,6 +1998,11 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
totalSkippedDuplicate: { type: Number, required: true, default: 0 },
totalErrors: { type: Number, required: true, default: 0 },
averageMemberSeconds: { type: Number },
completedMemberJobIds: {
type: [String],
required: true,
default: [],
},
errors: {
type: [LegacyImportJobErrorSubSchema],
required: true,
@@ -2003,6 +2018,7 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
LegacyImportJobSchema.index({ communityId: 1, startedAt: -1 });
LegacyImportJobSchema.index({ communityId: 1, state: 1 });
LegacyImportJobSchema.index({ jobId: 1, completedMemberJobIds: 1 });
const AuctionPlayerSnapshotSubSchema = new Schema<AuctionPlayerSnapshot>(
{
@@ -56,20 +56,25 @@ export interface JetStreamConsumeOptions {
maxMessages?: number;
expiresMs?: number;
nakDelayMs?: number;
restartDelayMilliseconds?: number;
}
const JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS = 1_000;
@Injectable()
export class NatsService implements OnModuleDestroy {
private readonly logger = new Logger(NatsService.name);
private readonly stringCodec = StringCodec();
private readonly subscriptions = new Set<Subscription>();
private readonly jetStreamConsumers = new Set<ConsumerMessages>();
private isDestroying = false;
constructor(
@Inject(NATS_CLIENT) private readonly natsClient: NatsConnection,
) {}
async onModuleDestroy() {
this.isDestroying = true;
for (const consumer of this.jetStreamConsumers) {
try {
await consumer.close();
@@ -199,28 +204,36 @@ export class NatsService implements OnModuleDestroy {
options: JetStreamConsumeOptions,
handler: JetStreamMessageHandler<T>,
): Promise<void> {
try {
const consumer = await this.natsClient
.jetstream()
.consumers.get(options.stream, options.durable);
const messages = await consumer.consume({
max_messages: options.maxMessages ?? 10,
expires: options.expiresMs ?? 30000,
});
this.jetStreamConsumers.add(messages);
while (!this.isDestroying) {
try {
for await (const msg of messages) {
await this.dispatchJetStream(msg, handler, options);
const consumer = await this.natsClient
.jetstream()
.consumers.get(options.stream, options.durable);
const messages = await consumer.consume({
max_messages: options.maxMessages ?? 10,
expires: options.expiresMs ?? 30000,
});
this.jetStreamConsumers.add(messages);
try {
for await (const msg of messages) {
await this.dispatchJetStream(msg, handler, options);
}
} finally {
this.jetStreamConsumers.delete(messages);
}
} finally {
this.jetStreamConsumers.delete(messages);
} catch (err) {
if (this.isDestroying) return;
this.logger.error(
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
(err as Error)?.stack ?? err
}`,
);
}
} catch (err) {
this.logger.error(
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
(err as Error)?.stack ?? err
}`,
await this.sleep(
options.restartDelayMilliseconds ??
JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS,
);
}
}
@@ -313,4 +326,8 @@ export class NatsService implements OnModuleDestroy {
? RetentionPolicy.Workqueue
: RetentionPolicy.Limits;
}
private sleep(durationMilliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
}
}
@@ -0,0 +1,121 @@
import { DistributedLockService } from './distributed-lock.service';
import { RedisService } from './redis.service';
function createService(sendCommand: jest.Mock) {
const redisService = {
raw: jest.fn().mockReturnValue({ sendCommand }),
} as unknown as RedisService;
return new DistributedLockService(redisService);
}
describe('DistributedLockService', () => {
it('acquires a lock with a token and millisecond expiration', async () => {
const sendCommand = jest.fn().mockResolvedValue('OK');
const service = createService(sendCommand);
const lockOwnership = await service.acquireLock({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
});
expect(lockOwnership).toEqual({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
});
expect(sendCommand).toHaveBeenCalledWith([
'SET',
'distributed-lock:test',
'owner-token',
'NX',
'PX',
'30000',
]);
});
it('returns null when another owner already holds the lock', async () => {
const sendCommand = jest.fn().mockResolvedValue(null);
const service = createService(sendCommand);
await expect(
service.acquireLock({
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
}),
).resolves.toBeNull();
});
it('releases a lock only when the ownership token matches', async () => {
const sendCommand = jest.fn().mockResolvedValue(1);
const service = createService(sendCommand);
await expect(
service.releaseLockIfOwned({
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
}),
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'),
'1',
'distributed-lock:test',
'owner-token',
]);
});
it('extends a lock only when the ownership token matches', async () => {
const sendCommand = jest.fn().mockResolvedValue(1);
const service = createService(sendCommand);
await expect(
service.extendLockIfOwned(
{
lockKey: 'distributed-lock:test',
lockOwnershipToken: 'owner-token',
lockExpirationDurationMilliseconds: 30_000,
},
45_000,
),
).resolves.toBe(true);
expect(sendCommand).toHaveBeenCalledWith([
'EVAL',
expect.stringContaining('PEXPIRE'),
'1',
'distributed-lock:test',
'owner-token',
'45000',
]);
});
it('runs the operation and releases the lock afterward', async () => {
const sendCommand = jest
.fn()
.mockResolvedValueOnce('OK')
.mockResolvedValueOnce(1);
const service = createService(sendCommand);
const operation = jest.fn().mockResolvedValue('done');
await expect(
service.runWithLock(
{
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
},
operation,
),
).resolves.toBe('done');
expect(operation).toHaveBeenCalledWith(
expect.objectContaining({
lockKey: 'distributed-lock:test',
lockExpirationDurationMilliseconds: 30_000,
}),
);
expect(sendCommand).toHaveBeenCalledTimes(2);
});
});
@@ -0,0 +1,199 @@
import { Injectable, Logger } from '@nestjs/common';
import { randomUUID } from 'crypto';
import { RedisService } from './redis.service';
const RELEASE_LOCK_IF_OWNED_SCRIPT = `
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("DEL", KEYS[1])
end
return 0
`;
const EXTEND_LOCK_IF_OWNED_SCRIPT = `
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
end
return 0
`;
export interface DistributedLockOwnership {
lockKey: string;
lockOwnershipToken: string;
lockExpirationDurationMilliseconds: number;
}
export interface AcquireDistributedLockOptions {
lockKey: string;
lockExpirationDurationMilliseconds: number;
lockOwnershipToken?: string;
}
export interface RunWithDistributedLockOptions {
lockKey: string;
lockExpirationDurationMilliseconds: number;
lockAcquireTimeoutMilliseconds?: number;
lockRetryDelayMilliseconds?: number;
}
export interface RunWithRenewedDistributedLockOptions
extends RunWithDistributedLockOptions {
lockRenewalIntervalMilliseconds: number;
}
@Injectable()
export class DistributedLockService {
private readonly logger = new Logger(DistributedLockService.name);
constructor(private readonly redisService: RedisService) {}
public async acquireLock(
options: AcquireDistributedLockOptions,
): Promise<DistributedLockOwnership | null> {
this.assertPositiveDuration(
options.lockExpirationDurationMilliseconds,
'lockExpirationDurationMilliseconds',
);
const lockOwnershipToken = options.lockOwnershipToken ?? randomUUID();
const result = await this.redisService.raw().sendCommand([
'SET',
options.lockKey,
lockOwnershipToken,
'NX',
'PX',
String(options.lockExpirationDurationMilliseconds),
]);
if (String(result) !== 'OK') return null;
return {
lockKey: options.lockKey,
lockOwnershipToken,
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
};
}
public async releaseLockIfOwned(
lockOwnership: DistributedLockOwnership,
): Promise<boolean> {
const result = await this.redisService.raw().sendCommand([
'EVAL',
RELEASE_LOCK_IF_OWNED_SCRIPT,
'1',
lockOwnership.lockKey,
lockOwnership.lockOwnershipToken,
]);
return Number(result) === 1;
}
public async extendLockIfOwned(
lockOwnership: DistributedLockOwnership,
lockExpirationDurationMilliseconds =
lockOwnership.lockExpirationDurationMilliseconds,
): Promise<boolean> {
this.assertPositiveDuration(
lockExpirationDurationMilliseconds,
'lockExpirationDurationMilliseconds',
);
const result = await this.redisService.raw().sendCommand([
'EVAL',
EXTEND_LOCK_IF_OWNED_SCRIPT,
'1',
lockOwnership.lockKey,
lockOwnership.lockOwnershipToken,
String(lockExpirationDurationMilliseconds),
]);
return Number(result) === 1;
}
public async runWithLock<TResult>(
options: RunWithDistributedLockOptions,
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
): Promise<TResult | null> {
const lockOwnership = await this.acquireLockWithRetry(options);
if (!lockOwnership) return null;
try {
return await operation(lockOwnership);
} finally {
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
this.logger.warn(
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
error instanceof Error ? error.message : error
}`,
);
});
}
}
public async runWithRenewedLock<TResult>(
options: RunWithRenewedDistributedLockOptions,
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
): Promise<TResult | null> {
this.assertPositiveDuration(
options.lockRenewalIntervalMilliseconds,
'lockRenewalIntervalMilliseconds',
);
const lockOwnership = await this.acquireLockWithRetry(options);
if (!lockOwnership) return null;
let lockRenewalTimer: NodeJS.Timeout | undefined;
lockRenewalTimer = setInterval(() => {
void this.extendLockIfOwned(lockOwnership).then((wasExtended) => {
if (wasExtended) return;
this.logger.warn(
`Lost distributed lock ownership before operation finished: ${lockOwnership.lockKey}`,
);
});
}, options.lockRenewalIntervalMilliseconds);
lockRenewalTimer.unref?.();
try {
return await operation(lockOwnership);
} finally {
if (lockRenewalTimer) clearInterval(lockRenewalTimer);
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
this.logger.warn(
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
error instanceof Error ? error.message : error
}`,
);
});
}
}
private async acquireLockWithRetry(
options: RunWithDistributedLockOptions,
): Promise<DistributedLockOwnership | null> {
const lockAcquireTimeoutMilliseconds =
options.lockAcquireTimeoutMilliseconds ?? 0;
const lockRetryDelayMilliseconds =
options.lockRetryDelayMilliseconds ?? 100;
const acquisitionStartedAtTimestampMilliseconds = Date.now();
while (true) {
const lockOwnership = await this.acquireLock(options);
if (lockOwnership) return lockOwnership;
if (lockAcquireTimeoutMilliseconds <= 0) return null;
const elapsedMilliseconds =
Date.now() - acquisitionStartedAtTimestampMilliseconds;
if (elapsedMilliseconds >= lockAcquireTimeoutMilliseconds) return null;
await this.sleep(
Math.min(lockRetryDelayMilliseconds, lockAcquireTimeoutMilliseconds),
);
}
}
private assertPositiveDuration(value: number, fieldName: string): void {
if (Number.isFinite(value) && value > 0) return;
throw new Error(`${fieldName} must be a positive finite number.`);
}
private sleep(durationMilliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
}
}
@@ -4,6 +4,7 @@ import { SecretModule } from '../secret/secret.module';
import { SecretService } from '../secret/secret.service';
import { REDIS_CLIENT } from './redis.constants';
import { resolveRedisClientOptions } from './redis.config';
import { DistributedLockService } from './distributed-lock.service';
import { RedisService, type RedisClient } from './redis.service';
async function readFirstSecret(
@@ -62,7 +63,8 @@ async function readFirstSecret(
},
},
RedisService,
DistributedLockService,
],
exports: [RedisService, REDIS_CLIENT],
exports: [RedisService, DistributedLockService, REDIS_CLIENT],
})
export class RedisModule {}
@@ -0,0 +1,52 @@
import { RedisService } from './redis.service';
function createRedisService(client: Record<string, unknown>) {
return new RedisService(client as never);
}
describe('RedisService stream pending recovery', () => {
it('normalizes XAUTOCLAIM array replies into stream entries', async () => {
const sendCommand = jest.fn().mockResolvedValue([
'0-0',
[
[
'1700000000000-0',
['job', '{"kind":"poll-tournament-result"}', 'source', 'test'],
],
],
]);
const service = createRedisService({ sendCommand });
await expect(
service.claimStalePendingStreamMessages({
key: 'riot:queue:high',
group: 'riot-refresh-worker',
consumer: 'worker-1',
stalePendingMessageMinimumIdleMilliseconds: 1_800_000,
count: 1,
}),
).resolves.toEqual({
nextStartId: '0-0',
messages: [
{
id: '1700000000000-0',
message: {
job: '{"kind":"poll-tournament-result"}',
source: 'test',
},
},
],
});
expect(sendCommand).toHaveBeenCalledWith([
'XAUTOCLAIM',
'riot:queue:high',
'riot-refresh-worker',
'worker-1',
'1800000',
'0-0',
'COUNT',
'1',
]);
});
});
@@ -16,6 +16,11 @@ export interface StreamReadResult {
name: string;
}
export interface ClaimStalePendingStreamMessagesResult {
messages: StreamEntry[];
nextStartId: string;
}
interface RawStreamMessage {
deliveriesCounter?: number;
id: string;
@@ -29,6 +34,16 @@ interface RawStreamReadResult {
}
function normalizeStreamFields(fields: unknown): Record<string, string> {
if (Array.isArray(fields)) {
const normalized: Record<string, string> = {};
for (let index = 0; index < fields.length; index += 2) {
const key = fields[index];
if (key === undefined) continue;
normalized[String(key)] = String(fields[index + 1] ?? '');
}
return normalized;
}
if (fields instanceof Map) {
return Object.fromEntries(
[...fields.entries()].map(([key, value]) => [String(key), String(value)]),
@@ -188,6 +203,38 @@ export class RedisService implements OnModuleDestroy {
}));
}
public async claimStalePendingStreamMessages(options: {
key: string;
group: string;
consumer: string;
stalePendingMessageMinimumIdleMilliseconds: number;
startId?: string;
count?: number;
}): Promise<ClaimStalePendingStreamMessagesResult> {
const response = (await this.client.sendCommand([
'XAUTOCLAIM',
options.key,
options.group,
options.consumer,
String(options.stalePendingMessageMinimumIdleMilliseconds),
options.startId ?? '0-0',
'COUNT',
String(options.count ?? 1),
])) as unknown;
const responseItems = Array.isArray(response) ? response : [];
const rawMessages = Array.isArray(responseItems[1])
? responseItems[1]
: [];
return {
nextStartId: String(responseItems[0] ?? '0-0'),
messages: rawMessages
.map((rawMessage) => this.normalizeClaimedStreamMessage(rawMessage))
.filter((message): message is StreamEntry => Boolean(message)),
};
}
public async acknowledgeStream(
key: string,
group: string,
@@ -210,4 +257,37 @@ export class RedisService implements OnModuleDestroy {
public async publish(channel: string, message: string): Promise<number> {
return this.client.publish(channel, message);
}
private normalizeClaimedStreamMessage(rawMessage: unknown): StreamEntry | null {
if (Array.isArray(rawMessage)) {
const [id, fields] = rawMessage;
if (id === undefined) return null;
return {
id: String(id),
message: normalizeStreamFields(fields),
};
}
if (
typeof rawMessage === 'object' &&
rawMessage !== null &&
'id' in rawMessage
) {
const message = rawMessage as RawStreamMessage;
return {
deliveriesCounter:
typeof message.deliveriesCounter === 'number'
? message.deliveriesCounter
: undefined,
id: String(message.id),
message: normalizeStreamFields(message.message),
millisElapsedFromDelivery:
typeof message.millisElapsedFromDelivery === 'number'
? message.millisElapsedFromDelivery
: undefined,
};
}
return null;
}
}
@@ -0,0 +1,146 @@
import mongoose from 'mongoose';
import { MatchStatsService } from '../community-stats/match-stats.service';
import { CommunityStatsService } from '../community-stats/community-stats.service';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { RiotMatchService } from '../riot/riot-match.service';
import { LegacyImportService } from './legacy-import.service';
import type { LegacyImportMemberJob } from './legacy-import.types';
function execResult<T>(value: T) {
return { exec: jest.fn().mockResolvedValue(value) };
}
describe('LegacyImportService member job completion', () => {
it('increments aggregate counters only once for the same member job id', async () => {
const legacyImportJobModel = {
findOne: jest
.fn()
.mockReturnValueOnce(
execResult({
jobId: 'legacy-job-1',
state: 'running',
processedMembers: 0,
totalMembers: 1,
averageMemberSeconds: 60,
completedMemberJobIds: [],
}),
)
.mockReturnValueOnce({
select: jest.fn().mockReturnValue({
lean: jest.fn().mockReturnValue(
execResult({
state: 'running',
processedMembers: 1,
totalMembers: 1,
}),
),
}),
})
.mockReturnValueOnce(
execResult({
jobId: 'legacy-job-1',
state: 'completed',
processedMembers: 1,
totalMembers: 1,
averageMemberSeconds: 60,
completedMemberJobIds: ['member-job-1'],
}),
),
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
};
const connectionService = {
getModel: jest.fn().mockResolvedValue(legacyImportJobModel),
} as unknown as jest.Mocked<ConnectionService>;
const distributedLockService = {
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
} as unknown as jest.Mocked<DistributedLockService>;
const communityStatsService = {
invalidateCommunity: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<CommunityStatsService>;
const service = new LegacyImportService(
connectionService,
{} as RedisService,
{} as RiotMatchService,
communityStatsService,
{} as MatchStatsService,
distributedLockService,
);
const job: LegacyImportMemberJob = {
kind: 'legacy-import-member',
jobId: 'member-job-1',
parentJobId: 'legacy-job-1',
communityId: new mongoose.Types.ObjectId().toString(),
userId: new mongoose.Types.ObjectId().toString(),
puuid: 'puuid-1',
startTime: 1,
endTime: 2,
triggeredBy: new mongoose.Types.ObjectId().toString(),
region: 'asia',
};
await (
service as unknown as {
finishMemberJob: (
job: LegacyImportMemberJob,
stats: {
discovered: number;
imported: number;
duplicates: number;
errors: number;
elapsedSeconds: number;
},
) => Promise<void>;
}
).finishMemberJob(job, {
discovered: 3,
imported: 2,
duplicates: 1,
errors: 0,
elapsedSeconds: 30,
});
await (
service as unknown as {
finishMemberJob: (
job: LegacyImportMemberJob,
stats: {
discovered: number;
imported: number;
duplicates: number;
errors: number;
elapsedSeconds: number;
},
) => Promise<void>;
}
).finishMemberJob(job, {
discovered: 3,
imported: 2,
duplicates: 1,
errors: 0,
elapsedSeconds: 30,
});
expect(legacyImportJobModel.updateOne).toHaveBeenNthCalledWith(
1,
{
jobId: 'legacy-job-1',
completedMemberJobIds: { $ne: 'member-job-1' },
state: { $ne: 'cancelled' },
},
expect.objectContaining({
$addToSet: { completedMemberJobIds: 'member-job-1' },
$inc: expect.objectContaining({
processedMembers: 1,
totalDiscoveredMatchIds: 3,
totalImportedMatches: 2,
totalSkippedDuplicate: 1,
totalErrors: 0,
}),
}),
);
expect(legacyImportJobModel.updateOne).toHaveBeenCalledTimes(2);
expect(distributedLockService.releaseLockIfOwned).toHaveBeenCalledTimes(1);
expect(communityStatsService.invalidateCommunity).toHaveBeenCalledTimes(1);
});
});
@@ -10,6 +10,10 @@ import mongoose from 'mongoose';
import { MatchStatsService } from '../community-stats/match-stats.service';
import { CommunityStatsService } from '../community-stats/community-stats.service';
import { ConnectionService } from '../global/database/connection.service';
import {
type DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import type {
LegacyImportJobDocument,
Match,
@@ -57,6 +61,7 @@ export class LegacyImportService {
private readonly riotMatchService: RiotMatchService,
private readonly communityStatsService: CommunityStatsService,
private readonly matchStatsService: MatchStatsService,
private readonly distributedLockService: DistributedLockService,
) {}
public async previewMembers(params: {
@@ -80,12 +85,14 @@ export class LegacyImportService {
}): Promise<StartLegacyImportResponse> {
const lockKey = this.communityLockKey(params.communityId);
const jobId = randomUUID();
const acquired = await this.redisService.setStringIfAbsent(
lockKey,
jobId,
LEGACY_IMPORT_LOCK_TTL_SECONDS,
);
if (!acquired) {
const communityImportLockOwnership =
await this.distributedLockService.acquireLock({
lockKey,
lockOwnershipToken: jobId,
lockExpirationDurationMilliseconds:
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
});
if (!communityImportLockOwnership) {
throw new ConflictException('Legacy import is already running.');
}
@@ -133,6 +140,7 @@ export class LegacyImportService {
totalSkippedDuplicate: 0,
totalErrors: 0,
averageMemberSeconds: LEGACY_IMPORT_MEMBER_AVG_SECONDS,
completedMemberJobIds: [],
});
for (const window of windows) {
@@ -160,7 +168,7 @@ export class LegacyImportService {
{ jobId },
{ $set: { state: 'completed', completedAt: new Date() } },
).exec();
await this.redisService.delete(lockKey);
await this.releaseDistributedLockSafely(communityImportLockOwnership);
}
return {
@@ -169,7 +177,7 @@ export class LegacyImportService {
etaSeconds: windows.length * LEGACY_IMPORT_MEMBER_AVG_SECONDS,
};
} catch (error) {
await this.redisService.delete(lockKey);
await this.releaseDistributedLockSafely(communityImportLockOwnership);
throw error;
}
}
@@ -220,7 +228,11 @@ export class LegacyImportService {
).exec();
if (!job)
throw new NotFoundException('Running legacy import job not found.');
await this.redisService.delete(this.communityLockKey(params.communityId));
await this.distributedLockService.releaseLockIfOwned({
lockKey: this.communityLockKey(params.communityId),
lockOwnershipToken: job.jobId,
lockExpirationDurationMilliseconds: LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
});
return job;
}
@@ -228,6 +240,7 @@ export class LegacyImportService {
const started = Date.now();
const legacyJob = await this.getRunnableJob(job);
if (!legacyJob) return;
if (legacyJob.completedMemberJobIds?.includes(job.jobId)) return;
let discovered = 0;
let imported = 0;
@@ -500,75 +513,81 @@ export class LegacyImportService {
userByPuuid: UserByPuuidMap,
): Promise<ImportOutcome> {
const lockKey = `legacy-import:match:${riotMatchId}`;
const acquired = await this.redisService.setStringIfAbsent(
lockKey,
job.parentJobId,
MATCH_IMPORT_LOCK_TTL_SECONDS,
);
if (!acquired) return 'duplicate';
try {
const MatchModel = await this.connectionService.getModel('match');
const existing = await MatchModel.exists({
'result.games.riotMatchId': riotMatchId,
});
if (existing) return 'duplicate';
const importOutcome = await this.distributedLockService.runWithLock(
{
lockKey,
lockExpirationDurationMilliseconds:
MATCH_IMPORT_LOCK_TTL_SECONDS * 1000,
},
async () => {
const MatchModel = await this.connectionService.getModel('match');
const existing = await MatchModel.exists({
'result.games.riotMatchId': riotMatchId,
});
if (existing) return 'duplicate';
const detail = await this.ensureRiotMatchPersisted(
riotMatchId,
job.region,
);
if (!detail.info.tournamentCode) return 'skipped';
const counts = countKnownParticipants(detail, userByPuuid);
if (counts.total === 0) return 'skipped';
const detail = await this.ensureRiotMatchPersisted(
riotMatchId,
job.region,
);
if (!detail.info.tournamentCode) return 'skipped';
const counts = countKnownParticipants(detail, userByPuuid);
if (counts.total === 0) return 'skipped';
const gameResult = buildGameResult(detail, userByPuuid, 1);
if (!gameResult || gameResult.participants.length === 0) return 'skipped';
const gameResult = buildGameResult(detail, userByPuuid, 1);
if (!gameResult || gameResult.participants.length === 0) {
return 'skipped';
}
const winnerTeamIndex = gameResult.winnerTeamIndex;
const matchPayload: Partial<Match> = {
communityId: new mongoose.Types.ObjectId(job.communityId),
title: this.formatLegacyDate(detail.info.gameCreation),
description: 'Legacy import (자동 생성)',
mapId: detail.info.mapId ?? 11,
status: 'finished',
scheduledAt: new Date(detail.info.gameCreation),
maxParticipants: 10,
gameCount: 1,
gameMode: 'pair',
tournamentCodeIds: [],
teamBuiltAt: new Date(detail.info.gameCreation),
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
participants: buildParticipantsSnapshot(detail, userByPuuid),
teams: buildTeams(detail, userByPuuid),
result: {
winnerTeamIndex,
riotMatchIds: [riotMatchId],
games: [gameResult],
seriesResults: [
{
teamAIndex: 1,
teamBIndex: 2,
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
const winnerTeamIndex = gameResult.winnerTeamIndex;
const matchPayload: Partial<Match> = {
communityId: new mongoose.Types.ObjectId(job.communityId),
title: this.formatLegacyDate(detail.info.gameCreation),
description: 'Legacy import (자동 생성)',
mapId: detail.info.mapId ?? 11,
status: 'finished',
scheduledAt: new Date(detail.info.gameCreation),
maxParticipants: 10,
gameCount: 1,
gameMode: 'pair',
tournamentCodeIds: [],
teamBuiltAt: new Date(detail.info.gameCreation),
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
participants: buildParticipantsSnapshot(detail, userByPuuid),
teams: buildTeams(detail, userByPuuid),
result: {
winnerTeamIndex,
riotMatchIds: [riotMatchId],
games: [gameResult],
seriesResults: [
{
teamAIndex: 1,
teamBIndex: 2,
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
winnerTeamIndex,
},
],
},
],
legacyImported: true,
legacyImportSource: {
importedAt: new Date(),
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
triggeringPuuid: job.puuid,
jobId: job.parentJobId,
},
prizes: [],
notices: [],
schedule: [],
};
const match = await MatchModel.create(matchPayload);
await this.matchStatsService.buildAndUpsert(match._id);
return 'imported';
},
legacyImported: true,
legacyImportSource: {
importedAt: new Date(),
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
triggeringPuuid: job.puuid,
jobId: job.parentJobId,
},
prizes: [],
notices: [],
schedule: [],
};
const match = await MatchModel.create(matchPayload);
await this.matchStatsService.buildAndUpsert(match._id);
return 'imported';
);
if (importOutcome === null) return 'duplicate';
return importOutcome;
} catch (error) {
await this.recordError(job.parentJobId, {
puuid: job.puuid,
@@ -576,8 +595,6 @@ export class LegacyImportService {
reason: error instanceof Error ? error.message : String(error),
});
return 'error';
} finally {
await this.redisService.delete(lockKey);
}
}
@@ -626,6 +643,7 @@ export class LegacyImportService {
jobId: job.parentJobId,
}).exec();
if (!current || current.state === 'cancelled') return;
if (current.completedMemberJobIds?.includes(job.jobId)) return;
const average =
current.averageMemberSeconds === undefined
@@ -634,14 +652,14 @@ export class LegacyImportService {
current.averageMemberSeconds * 0.7 + stats.elapsedSeconds * 0.3,
);
const nextProcessed = current.processedMembers + 1;
const completed = nextProcessed >= current.totalMembers;
await LegacyImportJobModel.updateOne(
{ jobId: job.parentJobId },
const updateResult = await LegacyImportJobModel.updateOne(
{
jobId: job.parentJobId,
completedMemberJobIds: { $ne: job.jobId },
state: { $ne: 'cancelled' },
},
{
$set: {
state: completed ? 'completed' : current.state,
completedAt: completed ? new Date() : current.completedAt,
averageMemberSeconds: average,
},
$inc: {
@@ -651,14 +669,49 @@ export class LegacyImportService {
totalSkippedDuplicate: stats.duplicates,
totalErrors: stats.errors,
},
$addToSet: {
completedMemberJobIds: job.jobId,
},
},
).exec();
if (updateResult.modifiedCount === 0) return;
const updated = await LegacyImportJobModel.findOne({
jobId: job.parentJobId,
})
.select({ processedMembers: 1, totalMembers: 1, state: 1, completedAt: 1 })
.lean()
.exec();
const completed = Boolean(
updated &&
updated.state !== 'cancelled' &&
updated.processedMembers >= updated.totalMembers,
);
if (completed) {
await LegacyImportJobModel.updateOne(
{
jobId: job.parentJobId,
state: { $ne: 'cancelled' },
},
{
$set: {
state: 'completed',
completedAt: updated?.completedAt ?? new Date(),
},
},
).exec();
}
if (completed) {
await Promise.all([
this.redisService.delete(
this.communityLockKey(new mongoose.Types.ObjectId(job.communityId)),
),
this.distributedLockService.releaseLockIfOwned({
lockKey: this.communityLockKey(
new mongoose.Types.ObjectId(job.communityId),
),
lockOwnershipToken: job.parentJobId,
lockExpirationDurationMilliseconds:
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
}),
this.communityStatsService.invalidateCommunity(
new mongoose.Types.ObjectId(job.communityId),
),
@@ -680,6 +733,20 @@ export class LegacyImportService {
return legacyJob;
}
private async releaseDistributedLockSafely(
lockOwnership: DistributedLockOwnership,
): Promise<void> {
await this.distributedLockService
.releaseLockIfOwned(lockOwnership)
.catch((error: unknown) => {
this.logger.warn(
`Failed to release legacy import lock ${lockOwnership.lockKey}: ${
error instanceof Error ? error.message : error
}`,
);
});
}
private async isCancelled(parentJobId: string): Promise<boolean> {
const LegacyImportJobModel =
await this.connectionService.getModel('legacyImportJob');
@@ -1,4 +1,5 @@
import mongoose from 'mongoose';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
@@ -35,11 +36,18 @@ function makeScheduler() {
};
const redis = {
getString: jest.fn().mockResolvedValue(null),
setString: jest.fn().mockResolvedValue(undefined),
setStringIfAbsent: jest.fn().mockResolvedValue(true),
delete: jest.fn().mockResolvedValue(1),
raw: jest.fn().mockReturnValue(rawClient),
} as unknown as jest.Mocked<RedisService> & { raw: jest.Mock };
const distributedLockService = {
acquireLock: jest.fn().mockResolvedValue({
lockKey: 'distributed-lock:riot-background-refresh:leader',
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds: 90_000,
}),
extendLockIfOwned: jest.fn().mockResolvedValue(true),
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
} as unknown as jest.Mocked<DistributedLockService>;
const rateLimiter = {
isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
} as unknown as jest.Mocked<RiotRateLimiterService>;
@@ -52,6 +60,7 @@ function makeScheduler() {
} as unknown as jest.Mocked<StaleAccountRepository>;
const scheduler = new BackgroundRefreshScheduler(
redis,
distributedLockService,
rateLimiter,
enrichmentService,
staleAccountRepository,
@@ -60,6 +69,7 @@ function makeScheduler() {
return {
scheduler,
redis,
distributedLockService,
rawClient,
rateLimiter,
enrichmentService,
@@ -115,8 +125,9 @@ describe('BackgroundRefreshScheduler', () => {
it('does not query stale accounts when another leader holds the lock', async () => {
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
const { scheduler, redis, staleAccountRepository } = makeScheduler();
redis.getString.mockResolvedValue('another-consumer');
const { scheduler, distributedLockService, staleAccountRepository } =
makeScheduler();
distributedLockService.acquireLock.mockResolvedValueOnce(null);
await scheduler.tick();
@@ -9,13 +9,17 @@ import {
RiotAccountEnrichmentService,
} from '../riot/riot-account-enrichment.service';
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
import {
type DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { isInKstBackgroundWindow } from './kst-window.util';
import { StaleAccountRepository } from './stale-account.repository';
const TICK_INTERVAL_MS = 60_000;
const LEADER_KEY = 'riot:bg-refresh:leader';
const LEADER_TTL_SECONDS = 90;
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
const LEADER_RENEW_INTERVAL_MS = 30_000;
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
const PER_TICK_CANDIDATE_LIMIT = 20;
@@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler
private tickTimer?: NodeJS.Timeout;
private leaderRenewTimer?: NodeJS.Timeout;
private tickInProgress = false;
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private leaderLockOwnership: DistributedLockOwnership | undefined;
constructor(
private readonly redis: RedisService,
private readonly distributedLockService: DistributedLockService,
private readonly rateLimiter: RiotRateLimiterService,
private readonly enrichmentService: RiotAccountEnrichmentService,
private readonly staleAccountRepository: StaleAccountRepository,
@@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler
void this.safeTick();
}
public onModuleDestroy(): void {
public async onModuleDestroy(): Promise<void> {
if (this.tickTimer) clearInterval(this.tickTimer);
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
.catch((err: unknown) => {
this.logger.warn(
`Background refresh leader release failed: ${
err instanceof Error ? err.message : err
}`,
);
});
this.leaderLockOwnership = undefined;
}
private async safeTick(): Promise<void> {
@@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler
}
private async tryBecomeLeader(): Promise<boolean> {
const current = await this.redis.getString(LEADER_KEY);
if (current === this.consumer) {
await this.redis.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
return true;
if (this.leaderLockOwnership) {
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return true;
this.leaderLockOwnership = undefined;
}
if (current) return false;
return this.redis.setStringIfAbsent(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
const leaderLockOwnership = await this.distributedLockService.acquireLock({
lockKey: LEADER_LOCK_KEY,
lockExpirationDurationMilliseconds:
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
});
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
return Boolean(leaderLockOwnership);
}
private async renewLeaderIfHeld(): Promise<void> {
try {
const current = await this.redis.getString(LEADER_KEY);
if (current !== this.consumer) return;
await this.redis.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
if (!this.leaderLockOwnership) return;
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return;
this.leaderLockOwnership = undefined;
} catch (err) {
this.logger.warn(
`Background refresh leader renew failed: ${
@@ -1,5 +1,6 @@
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { RedisService } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
@@ -19,6 +20,16 @@ function makeWorker() {
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RedisService>;
const distributedLockService = {
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
} as unknown as jest.Mocked<DistributedLockService>;
const jobQueueService = {
markProcessing: jest.fn().mockResolvedValue(undefined),
markCompleted: jest.fn().mockResolvedValue({
@@ -54,6 +65,7 @@ function makeWorker() {
const worker = new RiotJobWorkerService(
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
@@ -66,6 +78,7 @@ function makeWorker() {
return {
worker,
redisService,
distributedLockService,
jobQueueService,
recentMatchesService,
connectionService,
@@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => {
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
);
});
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
const { worker, distributedLockService, redisService } = makeWorker();
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
const userId = new mongoose.Types.ObjectId().toString();
const job: RiotRefreshSubJob = {
jobId: 'job-1',
parentJobId: 'parent-1',
userId,
puuid: 'puuid-1',
region: 'asia',
platform: 'kr',
queueGroup: 'summoners-rift',
queueIds: [420],
requestedAt: Date.now(),
priority: 'high',
};
await (
worker as unknown as {
processEntry: (
stream: string,
entry: { id: string; message: Record<string, string> },
) => Promise<void>;
}
).processEntry('riot:queue:high', {
id: '1-0',
message: { job: JSON.stringify(job) },
});
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
});
});
@@ -6,6 +6,10 @@ import {
} from '@nestjs/common';
import mongoose from 'mongoose';
import { ConnectionService } from '../global/database/connection.service';
import {
DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import { RedisService, StreamEntry } from '../global/redis/redis.service';
import { LegacyImportService } from '../legacy-import/legacy-import.service';
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
@@ -23,17 +27,24 @@ import {
import { RiotJobQueueService } from './riot-job-queue.service';
import { RiotRefreshGateway } from './riot-refresh.gateway';
const LEADER_KEY = 'riot:worker:leader';
const LEADER_TTL_SECONDS = 60;
const RIOT_JOB_WORKER_LEADER_LOCK_KEY =
'distributed-lock:riot-job-worker:leader';
const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000;
const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000;
const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000;
const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1;
@Injectable()
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(RiotJobWorkerService.name);
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`;
private leaderLockOwnership: DistributedLockOwnership | undefined;
private stopped = false;
constructor(
private readonly redisService: RedisService,
private readonly distributedLockService: DistributedLockService,
private readonly jobQueueService: RiotJobQueueService,
private readonly recentMatchesService: RiotRecentMatchesService,
private readonly connectionService: ConnectionService,
@@ -48,8 +59,19 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
void this.loop();
}
public onModuleDestroy() {
public async onModuleDestroy() {
this.stopped = true;
if (!this.leaderLockOwnership) return;
await this.distributedLockService
.releaseLockIfOwned(this.leaderLockOwnership)
.catch((error: unknown) => {
this.logger.warn(
`Failed to release Riot job worker leader lock: ${
error instanceof Error ? error.message : error
}`,
);
});
this.leaderLockOwnership = undefined;
}
private async loop(): Promise<void> {
@@ -63,7 +85,9 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
await this.jobQueueService.promoteDueScheduledJobs();
const entry =
(await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ??
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
(await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ??
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
if (!entry) continue;
@@ -89,6 +113,31 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
return;
}
const didProcessJob = await this.distributedLockService.runWithRenewedLock(
{
lockKey: this.jobProcessingLockKey(job),
lockExpirationDurationMilliseconds:
RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.processJob(job);
return true;
},
);
if (!didProcessJob) {
this.logger.debug(
`Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`,
);
return;
}
await this.acknowledgeAndDelete(stream, entry.id);
}
private async processJob(job: RiotJobPayload): Promise<void> {
try {
if (this.isTournamentResultJob(job)) {
await this.tournamentResultService.processPollJob(job);
@@ -136,8 +185,6 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
error instanceof Error ? error.message : error
}`,
);
} finally {
await this.acknowledgeAndDelete(stream, entry.id);
}
}
@@ -228,15 +275,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.readGroup(
stream,
RIOT_JOB_GROUP,
this.consumer,
'>',
{ count: 1, blockMs },
RIOT_JOB_GROUP,
this.consumerIdentifier,
'>',
{ count: 1, blockMs },
);
const message = result?.[0]?.messages[0];
return message ? { stream, message } : null;
}
private async readOneStalePendingMessage(
stream: string,
): Promise<{ stream: string; message: StreamEntry } | null> {
const result = await this.redisService.claimStalePendingStreamMessages({
key: stream,
group: RIOT_JOB_GROUP,
consumer: this.consumerIdentifier,
stalePendingMessageMinimumIdleMilliseconds:
STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
count: STALE_PENDING_MESSAGE_CLAIM_COUNT,
});
const message = result.messages[0];
return message ? { stream, message } : null;
}
private async ensureConsumerGroups(): Promise<void> {
await Promise.all(
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
@@ -252,21 +314,26 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
}
private async tryBecomeLeader(): Promise<boolean> {
const current = await this.redisService.getString(LEADER_KEY);
if (current === this.consumer) {
await this.redisService.setString(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
return true;
if (this.leaderLockOwnership) {
const didExtendLeaderLock =
await this.distributedLockService.extendLockIfOwned(
this.leaderLockOwnership,
);
if (didExtendLeaderLock) return true;
this.leaderLockOwnership = undefined;
}
if (current) return false;
return this.redisService.setStringIfAbsent(
LEADER_KEY,
this.consumer,
LEADER_TTL_SECONDS,
);
const leaderLockOwnership = await this.distributedLockService.acquireLock({
lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY,
lockExpirationDurationMilliseconds:
RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
});
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
return Boolean(leaderLockOwnership);
}
private jobProcessingLockKey(job: RiotJobPayload): string {
return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`;
}
private sleep(ms: number): Promise<void> {
@@ -4,14 +4,14 @@ import type {
RiotMatch as StoredRiotMatch,
} from '../global/database/schema';
import { ConnectionService } from '../global/database/connection.service';
import { RedisService } from '../global/redis/redis.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import type { RiotRegionalRoute } from './riot.types';
import { RiotMatchService } from './riot-match.service';
export const RIOT_RECENT_MATCH_LIMIT = 20;
export const RIOT_SOLO_QUEUE_ID = 420;
const MATCH_ENRICHMENT_LOCK_TTL_SECONDS = 60;
const MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
const MATCH_DETAIL_CONCURRENCY = 2;
const DEFAULT_REGION: RiotRegionalRoute = 'asia';
@@ -22,7 +22,7 @@ export class RiotRecentMatchesService {
constructor(
private readonly connectionService: ConnectionService,
private readonly matchService: RiotMatchService,
private readonly redisService: RedisService,
private readonly distributedLockService: DistributedLockService,
) {}
public async syncRecentMatches(params: {
@@ -34,83 +34,84 @@ export class RiotRecentMatchesService {
const region = params.region ?? DEFAULT_REGION;
const limit = params.limit ?? RIOT_RECENT_MATCH_LIMIT;
const queueId = params.queueId ?? RIOT_SOLO_QUEUE_ID;
const lockKey = `riot:enrich:matches:${params.puuid}`;
const lockValue = `${process.pid}:${Date.now()}`;
const acquired = await this.redisService.setStringIfAbsent(
lockKey,
lockValue,
MATCH_ENRICHMENT_LOCK_TTL_SECONDS,
);
if (!acquired) {
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
try {
const matchIds = await this.matchService.getMatchIdsByPuuid(params.puuid, {
region,
queue: queueId,
count: limit,
});
if (matchIds.length === 0) {
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
const RiotMatchModel = await this.connectionService.getModel('riotMatch');
const existingMatches = await RiotMatchModel.find({
matchId: { $in: matchIds },
})
.select({ matchId: 1 })
.lean()
.exec();
const existingMatchIds = new Set(existingMatches.map((match) => match.matchId));
const missingMatchIds = matchIds.filter((matchId) => !existingMatchIds.has(matchId));
await runWithConcurrency(missingMatchIds, MATCH_DETAIL_CONCURRENCY, async (matchId) => {
try {
const dto = await this.matchService.getMatch(matchId, { region });
if (dto.info.queueId !== queueId) {
this.logger.warn(
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
);
return;
}
const payload: Partial<StoredRiotMatch> = {
matchId: dto.metadata.matchId,
gameCreation: dto.info.gameCreation,
gameDuration: dto.info.gameDuration,
gameVersion: dto.info.gameVersion,
queueId: dto.info.queueId,
participantPuuids: dto.metadata.participants,
raw: dto as unknown as Record<string, unknown>,
fetchedAt: new Date(),
};
await RiotMatchModel.updateOne(
{ matchId: dto.metadata.matchId },
{ $setOnInsert: payload },
{ upsert: true },
).exec();
} catch (error: unknown) {
this.logger.warn(
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
error instanceof Error ? error.message : error
}`,
);
}
});
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
} finally {
await this.redisService.delete(lockKey).catch((error: unknown) => {
this.logger.warn(
`Failed to release recent matches lock for ${params.puuid}: ${
(error as Error)?.message ?? error
}`,
const result = await this.distributedLockService.runWithLock(
{
lockKey: `distributed-lock:riot-recent-matches:sync:puuid:${params.puuid}`,
lockExpirationDurationMilliseconds:
MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS,
},
async () => {
const matchIds = await this.matchService.getMatchIdsByPuuid(
params.puuid,
{
region,
queue: queueId,
count: limit,
},
);
});
}
if (matchIds.length === 0) {
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
const RiotMatchModel =
await this.connectionService.getModel('riotMatch');
const existingMatches = await RiotMatchModel.find({
matchId: { $in: matchIds },
})
.select({ matchId: 1 })
.lean()
.exec();
const existingMatchIds = new Set(
existingMatches.map((match) => match.matchId),
);
const missingMatchIds = matchIds.filter(
(matchId) => !existingMatchIds.has(matchId),
);
await runWithConcurrency(
missingMatchIds,
MATCH_DETAIL_CONCURRENCY,
async (matchId) => {
try {
const dto = await this.matchService.getMatch(matchId, { region });
if (dto.info.queueId !== queueId) {
this.logger.warn(
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
);
return;
}
const payload: Partial<StoredRiotMatch> = {
matchId: dto.metadata.matchId,
gameCreation: dto.info.gameCreation,
gameDuration: dto.info.gameDuration,
gameVersion: dto.info.gameVersion,
queueId: dto.info.queueId,
participantPuuids: dto.metadata.participants,
raw: dto as unknown as Record<string, unknown>,
fetchedAt: new Date(),
};
await RiotMatchModel.updateOne(
{ matchId: dto.metadata.matchId },
{ $setOnInsert: payload },
{ upsert: true },
).exec();
} catch (error: unknown) {
this.logger.warn(
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
error instanceof Error ? error.message : error
}`,
);
}
},
);
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
},
);
return result ?? this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
public async ensureFreshForQueue(params: {
@@ -1,3 +1,4 @@
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { TournamentCodeScheduler } from './tournament-code.scheduler';
describe('TournamentCodeScheduler', () => {
@@ -18,16 +19,28 @@ describe('TournamentCodeScheduler', () => {
scheduleInitialPoll:
params?.scheduleInitialPoll ?? jest.fn().mockResolvedValue(null),
};
const distributedLockService = {
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
} as unknown as jest.Mocked<DistributedLockService>;
const scheduler = new TournamentCodeScheduler(
tournamentCodeService as never,
autoProvisionService as never,
tournamentResultService as never,
distributedLockService as never,
) as unknown as { attachPending(): Promise<void> };
return {
scheduler,
tournamentCodeService,
tournamentResultService,
distributedLockService,
};
}
@@ -59,4 +72,16 @@ describe('TournamentCodeScheduler', () => {
expect(tournamentResultService.scheduleInitialPoll).not.toHaveBeenCalled();
});
it('does not attach pending tournament codes when another instance owns the scheduler lock', async () => {
const { scheduler, tournamentCodeService, distributedLockService } =
createScheduler();
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
await scheduler.attachPending();
expect(
tournamentCodeService.attachPendingMatchDocuments,
).not.toHaveBeenCalled();
});
});
@@ -1,8 +1,18 @@
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import { TournamentResultService } from '../tournament-result/tournament-result.service';
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
import { TournamentCodeService } from './tournament-code.service';
const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches';
const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes';
const ENSURE_VENDOR_CODE_POOL_LOCK_KEY =
'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool';
const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000;
const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000;
@Injectable()
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(TournamentCodeScheduler.name);
@@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private readonly tournamentCodeService: TournamentCodeService,
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
private readonly tournamentResultService: TournamentResultService,
private readonly distributedLockService: DistributedLockService,
) {}
public onModuleInit() {
@@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async attachPending() {
try {
const matches =
await this.tournamentCodeService.attachPendingMatchDocuments();
if (matches.length > 0) {
this.logger.debug(
`Attached tournament codes for ${matches.length} matches.`,
);
for (const match of matches) {
await this.tournamentResultService.scheduleInitialPoll(match);
}
}
await this.distributedLockService.runWithRenewedLock(
{
lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
const matches =
await this.tournamentCodeService.attachPendingMatchDocuments();
if (matches.length > 0) {
this.logger.debug(
`Attached tournament codes for ${matches.length} matches.`,
);
for (const match of matches) {
await this.tournamentResultService.scheduleInitialPoll(match);
}
}
},
);
} catch (error) {
this.logger.warn(
`Failed to attach pending tournament codes: ${
@@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async markExpired() {
try {
await this.tournamentCodeService.markExpired();
await this.distributedLockService.runWithRenewedLock(
{
lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.tournamentCodeService.markExpired();
},
);
} catch (error) {
this.logger.warn(
`Failed to mark expired tournament codes: ${
@@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
private async ensureVendorPool() {
try {
await this.autoProvisionService.ensurePool();
await this.distributedLockService.runWithRenewedLock(
{
lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY,
lockExpirationDurationMilliseconds:
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.autoProvisionService.ensurePool();
},
);
} catch (error) {
this.logger.warn(
`Failed to auto-provision tournament codes: ${
@@ -1,5 +1,6 @@
import mongoose from 'mongoose';
import type { MatchDocument } from '../global/database/schema';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import type { RiotMatchDto } from '../riot/riot.types';
import { TournamentResultService } from './tournament-result.service';
@@ -16,15 +17,38 @@ type TournamentResultServiceInternals = TournamentResultService & {
};
function createService() {
const distributedLockService = createDistributedLockService();
return new TournamentResultService(
{} as never,
{} as never,
{} as never,
{} as never,
{} as never,
distributedLockService as never,
) as unknown as TournamentResultServiceInternals;
}
function createDistributedLockService() {
return {
runWithLock: jest.fn().mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
operation({
lockKey: options.lockKey,
lockOwnershipToken: 'lock-owner',
lockExpirationDurationMilliseconds:
options.lockExpirationDurationMilliseconds,
}),
),
} as unknown as jest.Mocked<DistributedLockService>;
}
interface MatchOverrides {
gameCount?: number;
gameMode?: 'pair' | 'bracket' | 'round-robin';
@@ -281,12 +305,14 @@ describe('TournamentResultService tournament code polling', () => {
const riotJobQueueService = {
enqueueTournamentResultPoll: jest.fn().mockResolvedValue(undefined),
};
const distributedLockService = createDistributedLockService();
const service = new TournamentResultService(
connectionService as never,
riotMatchService as never,
riotJobQueueService as never,
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
distributedLockService as never,
);
return { service, match, code, riotMatchService };
@@ -455,12 +481,14 @@ describe('TournamentResultService manual result collection', () => {
getMatchIdsByPuuid: jest.fn().mockResolvedValue(['KR_1234567890']),
getMatch: jest.fn().mockResolvedValue(detail),
};
const distributedLockService = createDistributedLockService();
const service = new TournamentResultService(
connectionService as never,
riotMatchService as never,
{} as never,
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
distributedLockService as never,
);
return {
@@ -11,6 +11,7 @@ import {
import { MatchStatsService } from '../community-stats/match-stats.service';
import { CommunityStatsService } from '../community-stats/community-stats.service';
import { ConnectionService } from '../global/database/connection.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import type {
MatchDocument,
MatchGameResult,
@@ -46,6 +47,12 @@ const POLL_EXPIRY_MS = 24 * 60 * 60_000;
const EXPAND_AFTER_MS = 60 * 60_000;
const EXPANDED_MEMBER_LIMIT_PER_TEAM = 2;
const TOURNAMENT_LOOKUP_START_MARGIN_SECONDS = 60;
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS =
30 * 60_000;
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS =
5 * 60_000;
const TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS =
5 * 60_000;
interface ManualGameAssignment {
gameNumber: number;
@@ -64,11 +71,29 @@ export class TournamentResultService {
private readonly riotJobQueueService: RiotJobQueueService,
private readonly communityStatsService: CommunityStatsService,
private readonly matchStatsService: MatchStatsService,
private readonly distributedLockService: DistributedLockService,
) {}
public async scheduleInitialPoll(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
priority: RiotJobPriority = 'high',
): Promise<EnqueueTournamentResultPollResult | null> {
const matchIdentifier = this.resolveMatchIdentifier(matchOrId);
return this.distributedLockService.runWithLock(
{
lockKey: this.tournamentResultPollProcessingLockKey(matchIdentifier),
lockExpirationDurationMilliseconds:
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockAcquireTimeoutMilliseconds: 1_000,
lockRetryDelayMilliseconds: 100,
},
async () => this.scheduleInitialPollWhileLockOwned(matchOrId, priority),
);
}
private async scheduleInitialPollWhileLockOwned(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
priority: RiotJobPriority,
): Promise<EnqueueTournamentResultPollResult | null> {
const match = await this.loadMatch(matchOrId);
if (match.status === 'finished') return null;
@@ -195,6 +220,29 @@ export class TournamentResultService {
}
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
const didProcessPollJob = await this.distributedLockService.runWithRenewedLock(
{
lockKey: this.tournamentResultPollProcessingLockKey(job.matchId),
lockExpirationDurationMilliseconds:
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
lockRenewalIntervalMilliseconds:
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
},
async () => {
await this.processPollJobWhileLockOwned(job);
return true;
},
);
if (didProcessPollJob) return;
this.logger.debug(
`Skipped tournament result poll job ${job.jobId}; another worker is processing match ${job.matchId}.`,
);
}
private async processPollJobWhileLockOwned(
job: PollTournamentResultJob,
): Promise<void> {
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
@@ -818,12 +866,28 @@ export class TournamentResultService {
private async reserveReadyDoubleEliminationCodes(
match: MatchDocument,
): Promise<void> {
await this.distributedLockService.runWithLock(
{
lockKey: this.tournamentResultReadyCodeReservationLockKey(
match._id.toString(),
),
lockExpirationDurationMilliseconds:
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
},
async () => this.reserveReadyDoubleEliminationCodesWhileLockOwned(match),
);
}
private async reserveReadyDoubleEliminationCodesWhileLockOwned(
match: MatchDocument,
): Promise<void> {
if ((match.gameMode ?? 'pair') !== 'double-elimination') return;
const readyBracketMatches = (match.doubleElimination?.bracketMatches ?? [])
.filter((bracketMatch) => bracketMatch.state === 'ready');
if (readyBracketMatches.length === 0) return;
const MatchModel = await this.connectionService.getModel('match');
const TournamentCodeModel =
await this.connectionService.getModel('tournamentCode');
const existingCodes =
@@ -871,8 +935,19 @@ export class TournamentResultService {
}
if (reservedIds.length === 0) return;
match.tournamentCodeIds.push(...reservedIds);
await match.save();
await MatchModel.updateOne(
{ _id: match._id },
{ $addToSet: { tournamentCodeIds: { $each: reservedIds } } },
).exec();
const existingTournamentCodeIds = new Set(
match.tournamentCodeIds.map((tournamentCodeId) =>
tournamentCodeId.toString(),
),
);
for (const reservedId of reservedIds) {
if (existingTournamentCodeIds.has(reservedId.toString())) continue;
match.tournamentCodeIds.push(reservedId);
}
}
private async markTournamentCodeUsedForManualResult(
@@ -1126,6 +1201,24 @@ export class TournamentResultService {
if (!match) throw new NotFoundException('Match not found.');
return match;
}
private resolveMatchIdentifier(
matchOrId: MatchDocument | mongoose.Types.ObjectId,
): string {
return matchOrId instanceof mongoose.Types.ObjectId
? matchOrId.toString()
: matchOrId._id.toString();
}
private tournamentResultPollProcessingLockKey(matchIdentifier: string): string {
return `distributed-lock:tournament-result:process-poll-job:match:${matchIdentifier}`;
}
private tournamentResultReadyCodeReservationLockKey(
matchIdentifier: string,
): string {
return `distributed-lock:tournament-result:reserve-ready-double-elimination-codes:match:${matchIdentifier}`;
}
}
function isRiotMatchDto(value: unknown): value is RiotMatchDto {