From 60a006008add29e72400de941858e941e8b47649 Mon Sep 17 00:00:00 2001 From: dahyeon Date: Sat, 27 Jun 2026 13:19:43 -0700 Subject: [PATCH] fix: harden multi-instance scheduler and worker execution --- .../dakao/dakao-outbox-sweeper.service.ts | 31 ++- .../dakao/dakao-outbox.service.spec.ts | 57 +++-- .../src/modules/dakao/dakao-outbox.service.ts | 103 ++++++-- .../database/connection.service.spec.ts | 17 ++ .../global/database/connection.service.ts | 6 +- .../api/src/modules/global/database/schema.ts | 16 ++ .../src/modules/global/nats/nats.service.ts | 55 +++-- .../redis/distributed-lock.service.spec.ts | 121 +++++++++ .../global/redis/distributed-lock.service.ts | 199 +++++++++++++++ .../src/modules/global/redis/redis.module.ts | 4 +- .../global/redis/redis.service.spec.ts | 52 ++++ .../src/modules/global/redis/redis.service.ts | 80 ++++++ .../legacy-import.service.spec.ts | 146 +++++++++++ .../legacy-import/legacy-import.service.ts | 231 +++++++++++------- .../background-refresh.scheduler.spec.ts | 19 +- .../background-refresh.scheduler.ts | 66 +++-- .../riot-job-worker.service.spec.ts | 46 ++++ .../riot-job-queue/riot-job-worker.service.ts | 115 +++++++-- .../riot/riot-recent-matches.service.ts | 159 ++++++------ .../tournament-code.scheduler.spec.ts | 25 ++ .../tournament-code.scheduler.ts | 68 +++++- .../tournament-result.service.spec.ts | 28 +++ .../tournament-result.service.ts | 97 +++++++- 23 files changed, 1442 insertions(+), 299 deletions(-) create mode 100644 apps/api/src/modules/global/redis/distributed-lock.service.spec.ts create mode 100644 apps/api/src/modules/global/redis/distributed-lock.service.ts create mode 100644 apps/api/src/modules/global/redis/redis.service.spec.ts create mode 100644 apps/api/src/modules/legacy-import/legacy-import.service.spec.ts diff --git a/apps/api/src/modules/dakao/dakao-outbox-sweeper.service.ts b/apps/api/src/modules/dakao/dakao-outbox-sweeper.service.ts index ab2c3ebb..e060242f 100644 --- a/apps/api/src/modules/dakao/dakao-outbox-sweeper.service.ts +++ b/apps/api/src/modules/dakao/dakao-outbox-sweeper.service.ts @@ -4,10 +4,13 @@ import { OnModuleDestroy, OnModuleInit, } from '@nestjs/common'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import { DakaoOutboxService } from './dakao-outbox.service'; const SWEEP_INTERVAL_MS = 60 * 1000; const STALE_OUTBOX_MS = 5 * 60 * 1000; +const SWEEP_LOCK_KEY = 'distributed-lock:dakao-outbox:sweep-stale-publish-leases'; +const SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS = 55_000; @Injectable() export class DakaoOutboxSweeperService @@ -16,7 +19,10 @@ export class DakaoOutboxSweeperService private readonly logger = new Logger(DakaoOutboxSweeperService.name); private timer: NodeJS.Timeout | undefined; - constructor(private readonly outboxService: DakaoOutboxService) {} + constructor( + private readonly outboxService: DakaoOutboxService, + private readonly distributedLockService: DistributedLockService, + ) {} onModuleInit() { this.timer = setInterval(() => { @@ -31,12 +37,23 @@ export class DakaoOutboxSweeperService private async sweep(): Promise { try { - const swept = await this.outboxService.sweepStale({ - staleMs: STALE_OUTBOX_MS, - limit: 20, - }); - if (swept > 0) { - this.logger.warn(`Re-published ${swept} stale Dakao outbox messages.`); + const swept = await this.distributedLockService.runWithLock( + { + lockKey: SWEEP_LOCK_KEY, + lockExpirationDurationMilliseconds: + SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS, + }, + async () => + this.outboxService.sweepStale({ + staleMs: STALE_OUTBOX_MS, + limit: 20, + }), + ); + const sweptCount = swept ?? 0; + if (sweptCount > 0) { + this.logger.warn( + `Re-published ${sweptCount} stale Dakao outbox messages.`, + ); } } catch (err) { this.logger.error( diff --git a/apps/api/src/modules/dakao/dakao-outbox.service.spec.ts b/apps/api/src/modules/dakao/dakao-outbox.service.spec.ts index 97b6c264..fc07ea5f 100644 --- a/apps/api/src/modules/dakao/dakao-outbox.service.spec.ts +++ b/apps/api/src/modules/dakao/dakao-outbox.service.spec.ts @@ -70,44 +70,55 @@ describe('DakaoOutboxService', () => { process.env.NODE_ENV = 'staging'; try { - const exec = jest.fn().mockResolvedValue([ - { - eventId: 'lolympus:chat.send-message:chat-log-1', - type: 'chat.send-message', - eventSource: 'lolympus', - subject: 'staging.dakao.chat.send-message.v1', - correlationId: 'chat-log-1', - payload: { - chatroomId: '18474390111480678', - segments: [{ kind: 'text', text: 'done' }], - }, + const staleRow = { + eventId: 'lolympus:chat.send-message:chat-log-1', + type: 'chat.send-message', + eventSource: 'lolympus', + subject: 'staging.dakao.chat.send-message.v1', + correlationId: 'chat-log-1', + payload: { + chatroomId: '18474390111480678', + segments: [{ kind: 'text', text: 'done' }], }, - ]); + }; + const exec = jest.fn().mockResolvedValue([staleRow]); const lean = jest.fn().mockReturnValue({ exec }); const limit = jest.fn().mockReturnValue({ lean }); const sort = jest.fn().mockReturnValue({ limit }); + const claimExec = jest.fn().mockResolvedValue(staleRow); + const claimLean = jest.fn().mockReturnValue({ exec: claimExec }); const outboxModel = { find: jest.fn().mockReturnValue({ sort }), + findOneAndUpdate: jest.fn().mockReturnValue({ lean: claimLean }), + updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })), }; - const { service } = makeService(outboxModel); - const enqueue = jest - .spyOn(service, 'enqueueAndPublish') - .mockResolvedValue(undefined); + const { service, natsService } = makeService(outboxModel); await expect(service.sweepStale({ staleMs: 1, limit: 1 })).resolves.toBe( 1, ); - expect(enqueue).toHaveBeenCalledWith( + expect(outboxModel.findOneAndUpdate).toHaveBeenCalledWith( expect.objectContaining({ - envelope: expect.objectContaining({ - resultSubjects: { - succeeded: - 'staging.lolympus.dakao.chat.send-message.succeeded.v1', - failed: 'staging.lolympus.dakao.chat.send-message.failed.v1', - }, + eventId: 'lolympus:chat.send-message:chat-log-1', + }), + expect.objectContaining({ + $set: expect.objectContaining({ + publishLeaseOwnerIdentifier: expect.any(String), + publishLeaseExpiresAt: expect.any(Date), }), }), + { returnDocument: 'after' }, + ); + expect(natsService.publishJetStream).toHaveBeenCalledWith( + 'staging.dakao.chat.send-message.v1', + expect.objectContaining({ + resultSubjects: { + succeeded: 'staging.lolympus.dakao.chat.send-message.succeeded.v1', + failed: 'staging.lolympus.dakao.chat.send-message.failed.v1', + }, + }), + { messageId: 'lolympus:chat.send-message:chat-log-1' }, ); } finally { process.env.NODE_ENV = previousNodeEnv; diff --git a/apps/api/src/modules/dakao/dakao-outbox.service.ts b/apps/api/src/modules/dakao/dakao-outbox.service.ts index 21fb7cda..1662c6a6 100644 --- a/apps/api/src/modules/dakao/dakao-outbox.service.ts +++ b/apps/api/src/modules/dakao/dakao-outbox.service.ts @@ -1,4 +1,5 @@ import { Injectable, Logger } from '@nestjs/common'; +import { randomUUID } from 'crypto'; import { ConnectionService } from '../global/database/connection.service'; import { NatsService } from '../global/nats/nats.service'; import { resolveDakaoNatsNames } from './dakao.nats-names'; @@ -10,6 +11,7 @@ import { const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message'; const TERMINAL_STATES = ['sent', 'failed'] as const; +const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000; @Injectable() export class DakaoOutboxService { @@ -50,35 +52,59 @@ export class DakaoOutboxService { { upsert: true, returnDocument: 'after', setDefaultsOnInsert: true }, ).exec(); + await this.publishCommand({ + subject: params.subject, + eventId: params.eventId, + envelope: params.envelope, + }); + } + + private async publishCommand(params: { + subject: string; + eventId: string; + envelope: DakaoCommandEnvelope; + publishLeaseOwnerIdentifier?: string; + }): Promise { + const OutboxModel = + await this.connectionService.getModel('dakaoOutboxMessage'); + const updateFilter = { + eventId: params.eventId, + state: { $nin: TERMINAL_STATES }, + ...(params.publishLeaseOwnerIdentifier + ? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier } + : {}), + }; try { await this.natsService.publishJetStream(params.subject, params.envelope, { messageId: params.eventId, }); await OutboxModel.updateOne( - { - eventId: params.eventId, - state: { $nin: TERMINAL_STATES }, - }, + updateFilter, { $set: { state: 'published', publishedAt: new Date(), }, $inc: { publishAttempts: 1 }, - $unset: { lastPublishError: '' }, + $unset: { + lastPublishError: '', + publishLeaseOwnerIdentifier: '', + publishLeaseExpiresAt: '', + }, }, ).exec(); } catch (err) { await OutboxModel.updateOne( - { - eventId: params.eventId, - state: { $nin: TERMINAL_STATES }, - }, + updateFilter, { $inc: { publishAttempts: 1 }, $set: { lastPublishError: (err as Error)?.message ?? String(err), }, + $unset: { + publishLeaseOwnerIdentifier: '', + publishLeaseExpiresAt: '', + }, }, ).exec(); this.logger.error( @@ -146,40 +172,77 @@ export class DakaoOutboxService { public async sweepStale(params?: { staleMs?: number; + publishLeaseDurationMilliseconds?: number; limit?: number; }): Promise { const OutboxModel = await this.connectionService.getModel('dakaoOutboxMessage'); + const currentTimestamp = new Date(); const staleBefore = new Date( - Date.now() - (params?.staleMs ?? 5 * 60 * 1000), + currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000), + ); + const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`; + const publishLeaseExpiresAt = new Date( + currentTimestamp.getTime() + + (params?.publishLeaseDurationMilliseconds ?? + DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS), ); const rows = await OutboxModel.find({ state: { $in: ['pending', 'published'] }, updatedAt: { $lte: staleBefore }, + $or: [ + { publishLeaseExpiresAt: { $exists: false } }, + { publishLeaseExpiresAt: { $lte: currentTimestamp } }, + ], }) .sort({ updatedAt: 1 }) .limit(params?.limit ?? 20) .lean() .exec(); + let swept = 0; for (const row of rows) { - await this.enqueueAndPublish({ - subject: row.subject, - eventId: row.eventId, + const claimedRow = await OutboxModel.findOneAndUpdate( + { + eventId: row.eventId, + state: { $in: ['pending', 'published'] }, + $or: [ + { publishLeaseExpiresAt: { $exists: false } }, + { publishLeaseExpiresAt: { $lte: currentTimestamp } }, + ], + }, + { + $set: { + publishLeaseOwnerIdentifier, + publishLeaseExpiresAt, + }, + }, + { returnDocument: 'after' }, + ) + .lean() + .exec(); + if (!claimedRow) continue; + + await this.publishCommand({ + subject: claimedRow.subject, + eventId: claimedRow.eventId, + publishLeaseOwnerIdentifier, envelope: { schema: 'dakao.command.v1', - eventId: row.eventId, - type: row.type, - eventSource: row.eventSource, - correlationId: row.correlationId, + eventId: claimedRow.eventId, + type: claimedRow.type, + eventSource: claimedRow.eventSource, + correlationId: claimedRow.correlationId, resultSubjects: - row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type), - payload: row.payload, + claimedRow.resultSubjects ?? + this.resolveDefaultResultSubjects(claimedRow.type), + payload: claimedRow.payload, }, }); + swept += 1; } - return rows.length; + return swept; } private resolveDefaultResultSubjects( diff --git a/apps/api/src/modules/global/database/connection.service.spec.ts b/apps/api/src/modules/global/database/connection.service.spec.ts index c0a3ec83..110d9220 100644 --- a/apps/api/src/modules/global/database/connection.service.spec.ts +++ b/apps/api/src/modules/global/database/connection.service.spec.ts @@ -42,4 +42,21 @@ describe('resolveMongoConnectionOptions', () => { autoCreate: false, }); }); + + it('enables automatic index creation when explicitly requested', () => { + expect( + resolveMongoConnectionOptions( + 'lolympus-staging', + 'lolympus-api:staging:1', + 'true', + ), + ).toEqual({ + dbName: 'lolympus-staging', + appName: 'lolympus-api:staging:1', + minPoolSize: 10, + maxPoolSize: 20, + autoIndex: true, + autoCreate: false, + }); + }); }); diff --git a/apps/api/src/modules/global/database/connection.service.ts b/apps/api/src/modules/global/database/connection.service.ts index 6f184d42..2ef95351 100644 --- a/apps/api/src/modules/global/database/connection.service.ts +++ b/apps/api/src/modules/global/database/connection.service.ts @@ -19,13 +19,17 @@ export function resolveMongoAppName( export function resolveMongoConnectionOptions( dbName: string, appName: string, + mongoAutoIndex = process.env.MONGODB_AUTO_INDEX, ): mongoose.ConnectOptions { + const shouldEnableMongoAutoIndex = + mongoAutoIndex?.trim().toLowerCase() === 'true'; + return { dbName, appName, minPoolSize: 10, maxPoolSize: 20, - autoIndex: false, + autoIndex: shouldEnableMongoAutoIndex, autoCreate: false, }; } diff --git a/apps/api/src/modules/global/database/schema.ts b/apps/api/src/modules/global/database/schema.ts index 77405a87..c14485ab 100644 --- a/apps/api/src/modules/global/database/schema.ts +++ b/apps/api/src/modules/global/database/schema.ts @@ -392,6 +392,8 @@ export interface DakaoOutboxMessage extends MongooseTimestamps { payload: Record; state: DakaoOutboxMessageState; publishAttempts: number; + publishLeaseOwnerIdentifier?: string; + publishLeaseExpiresAt?: Date; publishedAt?: Date; terminalAt?: Date; resultEventId?: string; @@ -733,6 +735,7 @@ export interface LegacyImportJob extends MongooseTimestamps { totalSkippedDuplicate: number; totalErrors: number; averageMemberSeconds?: number; + completedMemberJobIds: string[]; errors?: LegacyImportJobError[]; } @@ -1282,6 +1285,8 @@ export const DakaoOutboxMessageSchema = new Schema( default: 'pending', }, publishAttempts: { type: Number, required: true, default: 0 }, + publishLeaseOwnerIdentifier: { type: String }, + publishLeaseExpiresAt: { type: Date }, publishedAt: { type: Date }, terminalAt: { type: Date }, resultEventId: { type: String }, @@ -1302,6 +1307,11 @@ export const DakaoOutboxMessageSchema = new Schema( ); DakaoOutboxMessageSchema.index({ state: 1, updatedAt: 1 }); +DakaoOutboxMessageSchema.index({ + state: 1, + publishLeaseExpiresAt: 1, + updatedAt: 1, +}); DakaoOutboxMessageSchema.index({ correlationId: 1 }, { sparse: true }); export const DakaoSendResultEventSchema = new Schema( @@ -1988,6 +1998,11 @@ export const LegacyImportJobSchema = new Schema( totalSkippedDuplicate: { type: Number, required: true, default: 0 }, totalErrors: { type: Number, required: true, default: 0 }, averageMemberSeconds: { type: Number }, + completedMemberJobIds: { + type: [String], + required: true, + default: [], + }, errors: { type: [LegacyImportJobErrorSubSchema], required: true, @@ -2003,6 +2018,7 @@ export const LegacyImportJobSchema = new Schema( LegacyImportJobSchema.index({ communityId: 1, startedAt: -1 }); LegacyImportJobSchema.index({ communityId: 1, state: 1 }); +LegacyImportJobSchema.index({ jobId: 1, completedMemberJobIds: 1 }); const AuctionPlayerSnapshotSubSchema = new Schema( { diff --git a/apps/api/src/modules/global/nats/nats.service.ts b/apps/api/src/modules/global/nats/nats.service.ts index acbc2b3b..5b58c6ef 100644 --- a/apps/api/src/modules/global/nats/nats.service.ts +++ b/apps/api/src/modules/global/nats/nats.service.ts @@ -56,20 +56,25 @@ export interface JetStreamConsumeOptions { maxMessages?: number; expiresMs?: number; nakDelayMs?: number; + restartDelayMilliseconds?: number; } +const JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS = 1_000; + @Injectable() export class NatsService implements OnModuleDestroy { private readonly logger = new Logger(NatsService.name); private readonly stringCodec = StringCodec(); private readonly subscriptions = new Set(); private readonly jetStreamConsumers = new Set(); + private isDestroying = false; constructor( @Inject(NATS_CLIENT) private readonly natsClient: NatsConnection, ) {} async onModuleDestroy() { + this.isDestroying = true; for (const consumer of this.jetStreamConsumers) { try { await consumer.close(); @@ -199,28 +204,36 @@ export class NatsService implements OnModuleDestroy { options: JetStreamConsumeOptions, handler: JetStreamMessageHandler, ): Promise { - try { - const consumer = await this.natsClient - .jetstream() - .consumers.get(options.stream, options.durable); - const messages = await consumer.consume({ - max_messages: options.maxMessages ?? 10, - expires: options.expiresMs ?? 30000, - }); - this.jetStreamConsumers.add(messages); - + while (!this.isDestroying) { try { - for await (const msg of messages) { - await this.dispatchJetStream(msg, handler, options); + const consumer = await this.natsClient + .jetstream() + .consumers.get(options.stream, options.durable); + const messages = await consumer.consume({ + max_messages: options.maxMessages ?? 10, + expires: options.expiresMs ?? 30000, + }); + this.jetStreamConsumers.add(messages); + + try { + for await (const msg of messages) { + await this.dispatchJetStream(msg, handler, options); + } + } finally { + this.jetStreamConsumers.delete(messages); } - } finally { - this.jetStreamConsumers.delete(messages); + } catch (err) { + if (this.isDestroying) return; + this.logger.error( + `JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${ + (err as Error)?.stack ?? err + }`, + ); } - } catch (err) { - this.logger.error( - `JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${ - (err as Error)?.stack ?? err - }`, + + await this.sleep( + options.restartDelayMilliseconds ?? + JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS, ); } } @@ -313,4 +326,8 @@ export class NatsService implements OnModuleDestroy { ? RetentionPolicy.Workqueue : RetentionPolicy.Limits; } + + private sleep(durationMilliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, durationMilliseconds)); + } } diff --git a/apps/api/src/modules/global/redis/distributed-lock.service.spec.ts b/apps/api/src/modules/global/redis/distributed-lock.service.spec.ts new file mode 100644 index 00000000..1b99bb42 --- /dev/null +++ b/apps/api/src/modules/global/redis/distributed-lock.service.spec.ts @@ -0,0 +1,121 @@ +import { DistributedLockService } from './distributed-lock.service'; +import { RedisService } from './redis.service'; + +function createService(sendCommand: jest.Mock) { + const redisService = { + raw: jest.fn().mockReturnValue({ sendCommand }), + } as unknown as RedisService; + return new DistributedLockService(redisService); +} + +describe('DistributedLockService', () => { + it('acquires a lock with a token and millisecond expiration', async () => { + const sendCommand = jest.fn().mockResolvedValue('OK'); + const service = createService(sendCommand); + + const lockOwnership = await service.acquireLock({ + lockKey: 'distributed-lock:test', + lockOwnershipToken: 'owner-token', + lockExpirationDurationMilliseconds: 30_000, + }); + + expect(lockOwnership).toEqual({ + lockKey: 'distributed-lock:test', + lockOwnershipToken: 'owner-token', + lockExpirationDurationMilliseconds: 30_000, + }); + expect(sendCommand).toHaveBeenCalledWith([ + 'SET', + 'distributed-lock:test', + 'owner-token', + 'NX', + 'PX', + '30000', + ]); + }); + + it('returns null when another owner already holds the lock', async () => { + const sendCommand = jest.fn().mockResolvedValue(null); + const service = createService(sendCommand); + + await expect( + service.acquireLock({ + lockKey: 'distributed-lock:test', + lockExpirationDurationMilliseconds: 30_000, + }), + ).resolves.toBeNull(); + }); + + it('releases a lock only when the ownership token matches', async () => { + const sendCommand = jest.fn().mockResolvedValue(1); + const service = createService(sendCommand); + + await expect( + service.releaseLockIfOwned({ + lockKey: 'distributed-lock:test', + lockOwnershipToken: 'owner-token', + lockExpirationDurationMilliseconds: 30_000, + }), + ).resolves.toBe(true); + + expect(sendCommand).toHaveBeenCalledWith([ + 'EVAL', + expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), + '1', + 'distributed-lock:test', + 'owner-token', + ]); + }); + + it('extends a lock only when the ownership token matches', async () => { + const sendCommand = jest.fn().mockResolvedValue(1); + const service = createService(sendCommand); + + await expect( + service.extendLockIfOwned( + { + lockKey: 'distributed-lock:test', + lockOwnershipToken: 'owner-token', + lockExpirationDurationMilliseconds: 30_000, + }, + 45_000, + ), + ).resolves.toBe(true); + + expect(sendCommand).toHaveBeenCalledWith([ + 'EVAL', + expect.stringContaining('PEXPIRE'), + '1', + 'distributed-lock:test', + 'owner-token', + '45000', + ]); + }); + + it('runs the operation and releases the lock afterward', async () => { + const sendCommand = jest + .fn() + .mockResolvedValueOnce('OK') + .mockResolvedValueOnce(1); + const service = createService(sendCommand); + const operation = jest.fn().mockResolvedValue('done'); + + await expect( + service.runWithLock( + { + lockKey: 'distributed-lock:test', + lockExpirationDurationMilliseconds: 30_000, + }, + operation, + ), + ).resolves.toBe('done'); + + expect(operation).toHaveBeenCalledWith( + expect.objectContaining({ + lockKey: 'distributed-lock:test', + lockExpirationDurationMilliseconds: 30_000, + }), + ); + expect(sendCommand).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/api/src/modules/global/redis/distributed-lock.service.ts b/apps/api/src/modules/global/redis/distributed-lock.service.ts new file mode 100644 index 00000000..fbaa2ccd --- /dev/null +++ b/apps/api/src/modules/global/redis/distributed-lock.service.ts @@ -0,0 +1,199 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import { RedisService } from './redis.service'; + +const RELEASE_LOCK_IF_OWNED_SCRIPT = ` +if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("DEL", KEYS[1]) +end +return 0 +`; + +const EXTEND_LOCK_IF_OWNED_SCRIPT = ` +if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("PEXPIRE", KEYS[1], ARGV[2]) +end +return 0 +`; + +export interface DistributedLockOwnership { + lockKey: string; + lockOwnershipToken: string; + lockExpirationDurationMilliseconds: number; +} + +export interface AcquireDistributedLockOptions { + lockKey: string; + lockExpirationDurationMilliseconds: number; + lockOwnershipToken?: string; +} + +export interface RunWithDistributedLockOptions { + lockKey: string; + lockExpirationDurationMilliseconds: number; + lockAcquireTimeoutMilliseconds?: number; + lockRetryDelayMilliseconds?: number; +} + +export interface RunWithRenewedDistributedLockOptions + extends RunWithDistributedLockOptions { + lockRenewalIntervalMilliseconds: number; +} + +@Injectable() +export class DistributedLockService { + private readonly logger = new Logger(DistributedLockService.name); + + constructor(private readonly redisService: RedisService) {} + + public async acquireLock( + options: AcquireDistributedLockOptions, + ): Promise { + this.assertPositiveDuration( + options.lockExpirationDurationMilliseconds, + 'lockExpirationDurationMilliseconds', + ); + + const lockOwnershipToken = options.lockOwnershipToken ?? randomUUID(); + const result = await this.redisService.raw().sendCommand([ + 'SET', + options.lockKey, + lockOwnershipToken, + 'NX', + 'PX', + String(options.lockExpirationDurationMilliseconds), + ]); + + if (String(result) !== 'OK') return null; + return { + lockKey: options.lockKey, + lockOwnershipToken, + lockExpirationDurationMilliseconds: + options.lockExpirationDurationMilliseconds, + }; + } + + public async releaseLockIfOwned( + lockOwnership: DistributedLockOwnership, + ): Promise { + const result = await this.redisService.raw().sendCommand([ + 'EVAL', + RELEASE_LOCK_IF_OWNED_SCRIPT, + '1', + lockOwnership.lockKey, + lockOwnership.lockOwnershipToken, + ]); + return Number(result) === 1; + } + + public async extendLockIfOwned( + lockOwnership: DistributedLockOwnership, + lockExpirationDurationMilliseconds = + lockOwnership.lockExpirationDurationMilliseconds, + ): Promise { + this.assertPositiveDuration( + lockExpirationDurationMilliseconds, + 'lockExpirationDurationMilliseconds', + ); + + const result = await this.redisService.raw().sendCommand([ + 'EVAL', + EXTEND_LOCK_IF_OWNED_SCRIPT, + '1', + lockOwnership.lockKey, + lockOwnership.lockOwnershipToken, + String(lockExpirationDurationMilliseconds), + ]); + return Number(result) === 1; + } + + public async runWithLock( + options: RunWithDistributedLockOptions, + operation: (lockOwnership: DistributedLockOwnership) => Promise, + ): Promise { + const lockOwnership = await this.acquireLockWithRetry(options); + if (!lockOwnership) return null; + + try { + return await operation(lockOwnership); + } finally { + await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => { + this.logger.warn( + `Failed to release distributed lock ${lockOwnership.lockKey}: ${ + error instanceof Error ? error.message : error + }`, + ); + }); + } + } + + public async runWithRenewedLock( + options: RunWithRenewedDistributedLockOptions, + operation: (lockOwnership: DistributedLockOwnership) => Promise, + ): Promise { + this.assertPositiveDuration( + options.lockRenewalIntervalMilliseconds, + 'lockRenewalIntervalMilliseconds', + ); + + const lockOwnership = await this.acquireLockWithRetry(options); + if (!lockOwnership) return null; + + let lockRenewalTimer: NodeJS.Timeout | undefined; + lockRenewalTimer = setInterval(() => { + void this.extendLockIfOwned(lockOwnership).then((wasExtended) => { + if (wasExtended) return; + this.logger.warn( + `Lost distributed lock ownership before operation finished: ${lockOwnership.lockKey}`, + ); + }); + }, options.lockRenewalIntervalMilliseconds); + lockRenewalTimer.unref?.(); + + try { + return await operation(lockOwnership); + } finally { + if (lockRenewalTimer) clearInterval(lockRenewalTimer); + await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => { + this.logger.warn( + `Failed to release distributed lock ${lockOwnership.lockKey}: ${ + error instanceof Error ? error.message : error + }`, + ); + }); + } + } + + private async acquireLockWithRetry( + options: RunWithDistributedLockOptions, + ): Promise { + const lockAcquireTimeoutMilliseconds = + options.lockAcquireTimeoutMilliseconds ?? 0; + const lockRetryDelayMilliseconds = + options.lockRetryDelayMilliseconds ?? 100; + const acquisitionStartedAtTimestampMilliseconds = Date.now(); + + while (true) { + const lockOwnership = await this.acquireLock(options); + if (lockOwnership) return lockOwnership; + + if (lockAcquireTimeoutMilliseconds <= 0) return null; + const elapsedMilliseconds = + Date.now() - acquisitionStartedAtTimestampMilliseconds; + if (elapsedMilliseconds >= lockAcquireTimeoutMilliseconds) return null; + + await this.sleep( + Math.min(lockRetryDelayMilliseconds, lockAcquireTimeoutMilliseconds), + ); + } + } + + private assertPositiveDuration(value: number, fieldName: string): void { + if (Number.isFinite(value) && value > 0) return; + throw new Error(`${fieldName} must be a positive finite number.`); + } + + private sleep(durationMilliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, durationMilliseconds)); + } +} diff --git a/apps/api/src/modules/global/redis/redis.module.ts b/apps/api/src/modules/global/redis/redis.module.ts index 18ba4316..205dd4e7 100644 --- a/apps/api/src/modules/global/redis/redis.module.ts +++ b/apps/api/src/modules/global/redis/redis.module.ts @@ -4,6 +4,7 @@ import { SecretModule } from '../secret/secret.module'; import { SecretService } from '../secret/secret.service'; import { REDIS_CLIENT } from './redis.constants'; import { resolveRedisClientOptions } from './redis.config'; +import { DistributedLockService } from './distributed-lock.service'; import { RedisService, type RedisClient } from './redis.service'; async function readFirstSecret( @@ -62,7 +63,8 @@ async function readFirstSecret( }, }, RedisService, + DistributedLockService, ], - exports: [RedisService, REDIS_CLIENT], + exports: [RedisService, DistributedLockService, REDIS_CLIENT], }) export class RedisModule {} diff --git a/apps/api/src/modules/global/redis/redis.service.spec.ts b/apps/api/src/modules/global/redis/redis.service.spec.ts new file mode 100644 index 00000000..46aa02cf --- /dev/null +++ b/apps/api/src/modules/global/redis/redis.service.spec.ts @@ -0,0 +1,52 @@ +import { RedisService } from './redis.service'; + +function createRedisService(client: Record) { + return new RedisService(client as never); +} + +describe('RedisService stream pending recovery', () => { + it('normalizes XAUTOCLAIM array replies into stream entries', async () => { + const sendCommand = jest.fn().mockResolvedValue([ + '0-0', + [ + [ + '1700000000000-0', + ['job', '{"kind":"poll-tournament-result"}', 'source', 'test'], + ], + ], + ]); + const service = createRedisService({ sendCommand }); + + await expect( + service.claimStalePendingStreamMessages({ + key: 'riot:queue:high', + group: 'riot-refresh-worker', + consumer: 'worker-1', + stalePendingMessageMinimumIdleMilliseconds: 1_800_000, + count: 1, + }), + ).resolves.toEqual({ + nextStartId: '0-0', + messages: [ + { + id: '1700000000000-0', + message: { + job: '{"kind":"poll-tournament-result"}', + source: 'test', + }, + }, + ], + }); + + expect(sendCommand).toHaveBeenCalledWith([ + 'XAUTOCLAIM', + 'riot:queue:high', + 'riot-refresh-worker', + 'worker-1', + '1800000', + '0-0', + 'COUNT', + '1', + ]); + }); +}); diff --git a/apps/api/src/modules/global/redis/redis.service.ts b/apps/api/src/modules/global/redis/redis.service.ts index 084c18de..71f1169a 100644 --- a/apps/api/src/modules/global/redis/redis.service.ts +++ b/apps/api/src/modules/global/redis/redis.service.ts @@ -16,6 +16,11 @@ export interface StreamReadResult { name: string; } +export interface ClaimStalePendingStreamMessagesResult { + messages: StreamEntry[]; + nextStartId: string; +} + interface RawStreamMessage { deliveriesCounter?: number; id: string; @@ -29,6 +34,16 @@ interface RawStreamReadResult { } function normalizeStreamFields(fields: unknown): Record { + if (Array.isArray(fields)) { + const normalized: Record = {}; + for (let index = 0; index < fields.length; index += 2) { + const key = fields[index]; + if (key === undefined) continue; + normalized[String(key)] = String(fields[index + 1] ?? ''); + } + return normalized; + } + if (fields instanceof Map) { return Object.fromEntries( [...fields.entries()].map(([key, value]) => [String(key), String(value)]), @@ -188,6 +203,38 @@ export class RedisService implements OnModuleDestroy { })); } + public async claimStalePendingStreamMessages(options: { + key: string; + group: string; + consumer: string; + stalePendingMessageMinimumIdleMilliseconds: number; + startId?: string; + count?: number; + }): Promise { + const response = (await this.client.sendCommand([ + 'XAUTOCLAIM', + options.key, + options.group, + options.consumer, + String(options.stalePendingMessageMinimumIdleMilliseconds), + options.startId ?? '0-0', + 'COUNT', + String(options.count ?? 1), + ])) as unknown; + + const responseItems = Array.isArray(response) ? response : []; + const rawMessages = Array.isArray(responseItems[1]) + ? responseItems[1] + : []; + + return { + nextStartId: String(responseItems[0] ?? '0-0'), + messages: rawMessages + .map((rawMessage) => this.normalizeClaimedStreamMessage(rawMessage)) + .filter((message): message is StreamEntry => Boolean(message)), + }; + } + public async acknowledgeStream( key: string, group: string, @@ -210,4 +257,37 @@ export class RedisService implements OnModuleDestroy { public async publish(channel: string, message: string): Promise { return this.client.publish(channel, message); } + + private normalizeClaimedStreamMessage(rawMessage: unknown): StreamEntry | null { + if (Array.isArray(rawMessage)) { + const [id, fields] = rawMessage; + if (id === undefined) return null; + return { + id: String(id), + message: normalizeStreamFields(fields), + }; + } + + if ( + typeof rawMessage === 'object' && + rawMessage !== null && + 'id' in rawMessage + ) { + const message = rawMessage as RawStreamMessage; + return { + deliveriesCounter: + typeof message.deliveriesCounter === 'number' + ? message.deliveriesCounter + : undefined, + id: String(message.id), + message: normalizeStreamFields(message.message), + millisElapsedFromDelivery: + typeof message.millisElapsedFromDelivery === 'number' + ? message.millisElapsedFromDelivery + : undefined, + }; + } + + return null; + } } diff --git a/apps/api/src/modules/legacy-import/legacy-import.service.spec.ts b/apps/api/src/modules/legacy-import/legacy-import.service.spec.ts new file mode 100644 index 00000000..6c17a48c --- /dev/null +++ b/apps/api/src/modules/legacy-import/legacy-import.service.spec.ts @@ -0,0 +1,146 @@ +import mongoose from 'mongoose'; +import { MatchStatsService } from '../community-stats/match-stats.service'; +import { CommunityStatsService } from '../community-stats/community-stats.service'; +import { ConnectionService } from '../global/database/connection.service'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; +import { RedisService } from '../global/redis/redis.service'; +import { RiotMatchService } from '../riot/riot-match.service'; +import { LegacyImportService } from './legacy-import.service'; +import type { LegacyImportMemberJob } from './legacy-import.types'; + +function execResult(value: T) { + return { exec: jest.fn().mockResolvedValue(value) }; +} + +describe('LegacyImportService member job completion', () => { + it('increments aggregate counters only once for the same member job id', async () => { + const legacyImportJobModel = { + findOne: jest + .fn() + .mockReturnValueOnce( + execResult({ + jobId: 'legacy-job-1', + state: 'running', + processedMembers: 0, + totalMembers: 1, + averageMemberSeconds: 60, + completedMemberJobIds: [], + }), + ) + .mockReturnValueOnce({ + select: jest.fn().mockReturnValue({ + lean: jest.fn().mockReturnValue( + execResult({ + state: 'running', + processedMembers: 1, + totalMembers: 1, + }), + ), + }), + }) + .mockReturnValueOnce( + execResult({ + jobId: 'legacy-job-1', + state: 'completed', + processedMembers: 1, + totalMembers: 1, + averageMemberSeconds: 60, + completedMemberJobIds: ['member-job-1'], + }), + ), + updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })), + }; + const connectionService = { + getModel: jest.fn().mockResolvedValue(legacyImportJobModel), + } as unknown as jest.Mocked; + const distributedLockService = { + releaseLockIfOwned: jest.fn().mockResolvedValue(true), + } as unknown as jest.Mocked; + const communityStatsService = { + invalidateCommunity: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + const service = new LegacyImportService( + connectionService, + {} as RedisService, + {} as RiotMatchService, + communityStatsService, + {} as MatchStatsService, + distributedLockService, + ); + const job: LegacyImportMemberJob = { + kind: 'legacy-import-member', + jobId: 'member-job-1', + parentJobId: 'legacy-job-1', + communityId: new mongoose.Types.ObjectId().toString(), + userId: new mongoose.Types.ObjectId().toString(), + puuid: 'puuid-1', + startTime: 1, + endTime: 2, + triggeredBy: new mongoose.Types.ObjectId().toString(), + region: 'asia', + }; + + await ( + service as unknown as { + finishMemberJob: ( + job: LegacyImportMemberJob, + stats: { + discovered: number; + imported: number; + duplicates: number; + errors: number; + elapsedSeconds: number; + }, + ) => Promise; + } + ).finishMemberJob(job, { + discovered: 3, + imported: 2, + duplicates: 1, + errors: 0, + elapsedSeconds: 30, + }); + await ( + service as unknown as { + finishMemberJob: ( + job: LegacyImportMemberJob, + stats: { + discovered: number; + imported: number; + duplicates: number; + errors: number; + elapsedSeconds: number; + }, + ) => Promise; + } + ).finishMemberJob(job, { + discovered: 3, + imported: 2, + duplicates: 1, + errors: 0, + elapsedSeconds: 30, + }); + + expect(legacyImportJobModel.updateOne).toHaveBeenNthCalledWith( + 1, + { + jobId: 'legacy-job-1', + completedMemberJobIds: { $ne: 'member-job-1' }, + state: { $ne: 'cancelled' }, + }, + expect.objectContaining({ + $addToSet: { completedMemberJobIds: 'member-job-1' }, + $inc: expect.objectContaining({ + processedMembers: 1, + totalDiscoveredMatchIds: 3, + totalImportedMatches: 2, + totalSkippedDuplicate: 1, + totalErrors: 0, + }), + }), + ); + expect(legacyImportJobModel.updateOne).toHaveBeenCalledTimes(2); + expect(distributedLockService.releaseLockIfOwned).toHaveBeenCalledTimes(1); + expect(communityStatsService.invalidateCommunity).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/api/src/modules/legacy-import/legacy-import.service.ts b/apps/api/src/modules/legacy-import/legacy-import.service.ts index cb3522e0..caad90e5 100644 --- a/apps/api/src/modules/legacy-import/legacy-import.service.ts +++ b/apps/api/src/modules/legacy-import/legacy-import.service.ts @@ -10,6 +10,10 @@ import mongoose from 'mongoose'; import { MatchStatsService } from '../community-stats/match-stats.service'; import { CommunityStatsService } from '../community-stats/community-stats.service'; import { ConnectionService } from '../global/database/connection.service'; +import { + type DistributedLockOwnership, + DistributedLockService, +} from '../global/redis/distributed-lock.service'; import type { LegacyImportJobDocument, Match, @@ -57,6 +61,7 @@ export class LegacyImportService { private readonly riotMatchService: RiotMatchService, private readonly communityStatsService: CommunityStatsService, private readonly matchStatsService: MatchStatsService, + private readonly distributedLockService: DistributedLockService, ) {} public async previewMembers(params: { @@ -80,12 +85,14 @@ export class LegacyImportService { }): Promise { const lockKey = this.communityLockKey(params.communityId); const jobId = randomUUID(); - const acquired = await this.redisService.setStringIfAbsent( - lockKey, - jobId, - LEGACY_IMPORT_LOCK_TTL_SECONDS, - ); - if (!acquired) { + const communityImportLockOwnership = + await this.distributedLockService.acquireLock({ + lockKey, + lockOwnershipToken: jobId, + lockExpirationDurationMilliseconds: + LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000, + }); + if (!communityImportLockOwnership) { throw new ConflictException('Legacy import is already running.'); } @@ -133,6 +140,7 @@ export class LegacyImportService { totalSkippedDuplicate: 0, totalErrors: 0, averageMemberSeconds: LEGACY_IMPORT_MEMBER_AVG_SECONDS, + completedMemberJobIds: [], }); for (const window of windows) { @@ -160,7 +168,7 @@ export class LegacyImportService { { jobId }, { $set: { state: 'completed', completedAt: new Date() } }, ).exec(); - await this.redisService.delete(lockKey); + await this.releaseDistributedLockSafely(communityImportLockOwnership); } return { @@ -169,7 +177,7 @@ export class LegacyImportService { etaSeconds: windows.length * LEGACY_IMPORT_MEMBER_AVG_SECONDS, }; } catch (error) { - await this.redisService.delete(lockKey); + await this.releaseDistributedLockSafely(communityImportLockOwnership); throw error; } } @@ -220,7 +228,11 @@ export class LegacyImportService { ).exec(); if (!job) throw new NotFoundException('Running legacy import job not found.'); - await this.redisService.delete(this.communityLockKey(params.communityId)); + await this.distributedLockService.releaseLockIfOwned({ + lockKey: this.communityLockKey(params.communityId), + lockOwnershipToken: job.jobId, + lockExpirationDurationMilliseconds: LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000, + }); return job; } @@ -228,6 +240,7 @@ export class LegacyImportService { const started = Date.now(); const legacyJob = await this.getRunnableJob(job); if (!legacyJob) return; + if (legacyJob.completedMemberJobIds?.includes(job.jobId)) return; let discovered = 0; let imported = 0; @@ -500,75 +513,81 @@ export class LegacyImportService { userByPuuid: UserByPuuidMap, ): Promise { const lockKey = `legacy-import:match:${riotMatchId}`; - const acquired = await this.redisService.setStringIfAbsent( - lockKey, - job.parentJobId, - MATCH_IMPORT_LOCK_TTL_SECONDS, - ); - if (!acquired) return 'duplicate'; - try { - const MatchModel = await this.connectionService.getModel('match'); - const existing = await MatchModel.exists({ - 'result.games.riotMatchId': riotMatchId, - }); - if (existing) return 'duplicate'; + const importOutcome = await this.distributedLockService.runWithLock( + { + lockKey, + lockExpirationDurationMilliseconds: + MATCH_IMPORT_LOCK_TTL_SECONDS * 1000, + }, + async () => { + const MatchModel = await this.connectionService.getModel('match'); + const existing = await MatchModel.exists({ + 'result.games.riotMatchId': riotMatchId, + }); + if (existing) return 'duplicate'; - const detail = await this.ensureRiotMatchPersisted( - riotMatchId, - job.region, - ); - if (!detail.info.tournamentCode) return 'skipped'; - const counts = countKnownParticipants(detail, userByPuuid); - if (counts.total === 0) return 'skipped'; + const detail = await this.ensureRiotMatchPersisted( + riotMatchId, + job.region, + ); + if (!detail.info.tournamentCode) return 'skipped'; + const counts = countKnownParticipants(detail, userByPuuid); + if (counts.total === 0) return 'skipped'; - const gameResult = buildGameResult(detail, userByPuuid, 1); - if (!gameResult || gameResult.participants.length === 0) return 'skipped'; + const gameResult = buildGameResult(detail, userByPuuid, 1); + if (!gameResult || gameResult.participants.length === 0) { + return 'skipped'; + } - const winnerTeamIndex = gameResult.winnerTeamIndex; - const matchPayload: Partial = { - communityId: new mongoose.Types.ObjectId(job.communityId), - title: this.formatLegacyDate(detail.info.gameCreation), - description: 'Legacy import (자동 생성)', - mapId: detail.info.mapId ?? 11, - status: 'finished', - scheduledAt: new Date(detail.info.gameCreation), - maxParticipants: 10, - gameCount: 1, - gameMode: 'pair', - tournamentCodeIds: [], - teamBuiltAt: new Date(detail.info.gameCreation), - hostUserId: new mongoose.Types.ObjectId(job.triggeredBy), - participants: buildParticipantsSnapshot(detail, userByPuuid), - teams: buildTeams(detail, userByPuuid), - result: { - winnerTeamIndex, - riotMatchIds: [riotMatchId], - games: [gameResult], - seriesResults: [ - { - teamAIndex: 1, - teamBIndex: 2, - teamAWins: winnerTeamIndex === 1 ? 1 : 0, - teamBWins: winnerTeamIndex === 2 ? 1 : 0, + const winnerTeamIndex = gameResult.winnerTeamIndex; + const matchPayload: Partial = { + communityId: new mongoose.Types.ObjectId(job.communityId), + title: this.formatLegacyDate(detail.info.gameCreation), + description: 'Legacy import (자동 생성)', + mapId: detail.info.mapId ?? 11, + status: 'finished', + scheduledAt: new Date(detail.info.gameCreation), + maxParticipants: 10, + gameCount: 1, + gameMode: 'pair', + tournamentCodeIds: [], + teamBuiltAt: new Date(detail.info.gameCreation), + hostUserId: new mongoose.Types.ObjectId(job.triggeredBy), + participants: buildParticipantsSnapshot(detail, userByPuuid), + teams: buildTeams(detail, userByPuuid), + result: { winnerTeamIndex, + riotMatchIds: [riotMatchId], + games: [gameResult], + seriesResults: [ + { + teamAIndex: 1, + teamBIndex: 2, + teamAWins: winnerTeamIndex === 1 ? 1 : 0, + teamBWins: winnerTeamIndex === 2 ? 1 : 0, + winnerTeamIndex, + }, + ], }, - ], + legacyImported: true, + legacyImportSource: { + importedAt: new Date(), + importedBy: new mongoose.Types.ObjectId(job.triggeredBy), + triggeringPuuid: job.puuid, + jobId: job.parentJobId, + }, + prizes: [], + notices: [], + schedule: [], + }; + const match = await MatchModel.create(matchPayload); + await this.matchStatsService.buildAndUpsert(match._id); + return 'imported'; }, - legacyImported: true, - legacyImportSource: { - importedAt: new Date(), - importedBy: new mongoose.Types.ObjectId(job.triggeredBy), - triggeringPuuid: job.puuid, - jobId: job.parentJobId, - }, - prizes: [], - notices: [], - schedule: [], - }; - const match = await MatchModel.create(matchPayload); - await this.matchStatsService.buildAndUpsert(match._id); - return 'imported'; + ); + if (importOutcome === null) return 'duplicate'; + return importOutcome; } catch (error) { await this.recordError(job.parentJobId, { puuid: job.puuid, @@ -576,8 +595,6 @@ export class LegacyImportService { reason: error instanceof Error ? error.message : String(error), }); return 'error'; - } finally { - await this.redisService.delete(lockKey); } } @@ -626,6 +643,7 @@ export class LegacyImportService { jobId: job.parentJobId, }).exec(); if (!current || current.state === 'cancelled') return; + if (current.completedMemberJobIds?.includes(job.jobId)) return; const average = current.averageMemberSeconds === undefined @@ -634,14 +652,14 @@ export class LegacyImportService { current.averageMemberSeconds * 0.7 + stats.elapsedSeconds * 0.3, ); - const nextProcessed = current.processedMembers + 1; - const completed = nextProcessed >= current.totalMembers; - await LegacyImportJobModel.updateOne( - { jobId: job.parentJobId }, + const updateResult = await LegacyImportJobModel.updateOne( + { + jobId: job.parentJobId, + completedMemberJobIds: { $ne: job.jobId }, + state: { $ne: 'cancelled' }, + }, { $set: { - state: completed ? 'completed' : current.state, - completedAt: completed ? new Date() : current.completedAt, averageMemberSeconds: average, }, $inc: { @@ -651,14 +669,49 @@ export class LegacyImportService { totalSkippedDuplicate: stats.duplicates, totalErrors: stats.errors, }, + $addToSet: { + completedMemberJobIds: job.jobId, + }, }, ).exec(); + if (updateResult.modifiedCount === 0) return; + + const updated = await LegacyImportJobModel.findOne({ + jobId: job.parentJobId, + }) + .select({ processedMembers: 1, totalMembers: 1, state: 1, completedAt: 1 }) + .lean() + .exec(); + const completed = Boolean( + updated && + updated.state !== 'cancelled' && + updated.processedMembers >= updated.totalMembers, + ); + if (completed) { + await LegacyImportJobModel.updateOne( + { + jobId: job.parentJobId, + state: { $ne: 'cancelled' }, + }, + { + $set: { + state: 'completed', + completedAt: updated?.completedAt ?? new Date(), + }, + }, + ).exec(); + } if (completed) { await Promise.all([ - this.redisService.delete( - this.communityLockKey(new mongoose.Types.ObjectId(job.communityId)), - ), + this.distributedLockService.releaseLockIfOwned({ + lockKey: this.communityLockKey( + new mongoose.Types.ObjectId(job.communityId), + ), + lockOwnershipToken: job.parentJobId, + lockExpirationDurationMilliseconds: + LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000, + }), this.communityStatsService.invalidateCommunity( new mongoose.Types.ObjectId(job.communityId), ), @@ -680,6 +733,20 @@ export class LegacyImportService { return legacyJob; } + private async releaseDistributedLockSafely( + lockOwnership: DistributedLockOwnership, + ): Promise { + await this.distributedLockService + .releaseLockIfOwned(lockOwnership) + .catch((error: unknown) => { + this.logger.warn( + `Failed to release legacy import lock ${lockOwnership.lockKey}: ${ + error instanceof Error ? error.message : error + }`, + ); + }); + } + private async isCancelled(parentJobId: string): Promise { const LegacyImportJobModel = await this.connectionService.getModel('legacyImportJob'); diff --git a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts index 3cb32bc5..9c642af0 100644 --- a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts +++ b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.spec.ts @@ -1,4 +1,5 @@ import mongoose from 'mongoose'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import { RedisService } from '../global/redis/redis.service'; import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service'; import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service'; @@ -35,11 +36,18 @@ function makeScheduler() { }; const redis = { getString: jest.fn().mockResolvedValue(null), - setString: jest.fn().mockResolvedValue(undefined), - setStringIfAbsent: jest.fn().mockResolvedValue(true), delete: jest.fn().mockResolvedValue(1), raw: jest.fn().mockReturnValue(rawClient), } as unknown as jest.Mocked & { raw: jest.Mock }; + const distributedLockService = { + acquireLock: jest.fn().mockResolvedValue({ + lockKey: 'distributed-lock:riot-background-refresh:leader', + lockOwnershipToken: 'lock-owner', + lockExpirationDurationMilliseconds: 90_000, + }), + extendLockIfOwned: jest.fn().mockResolvedValue(true), + releaseLockIfOwned: jest.fn().mockResolvedValue(true), + } as unknown as jest.Mocked; const rateLimiter = { isRefreshWindowOpen: jest.fn().mockResolvedValue(true), } as unknown as jest.Mocked; @@ -52,6 +60,7 @@ function makeScheduler() { } as unknown as jest.Mocked; const scheduler = new BackgroundRefreshScheduler( redis, + distributedLockService, rateLimiter, enrichmentService, staleAccountRepository, @@ -60,6 +69,7 @@ function makeScheduler() { return { scheduler, redis, + distributedLockService, rawClient, rateLimiter, enrichmentService, @@ -115,8 +125,9 @@ describe('BackgroundRefreshScheduler', () => { it('does not query stale accounts when another leader holds the lock', async () => { jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z')); - const { scheduler, redis, staleAccountRepository } = makeScheduler(); - redis.getString.mockResolvedValue('another-consumer'); + const { scheduler, distributedLockService, staleAccountRepository } = + makeScheduler(); + distributedLockService.acquireLock.mockResolvedValueOnce(null); await scheduler.tick(); diff --git a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts index f7c81fd7..bbb9a940 100644 --- a/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts +++ b/apps/api/src/modules/riot-background-refresh/background-refresh.scheduler.ts @@ -9,13 +9,17 @@ import { RiotAccountEnrichmentService, } from '../riot/riot-account-enrichment.service'; import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service'; +import { + type DistributedLockOwnership, + DistributedLockService, +} from '../global/redis/distributed-lock.service'; import { RedisService } from '../global/redis/redis.service'; import { isInKstBackgroundWindow } from './kst-window.util'; import { StaleAccountRepository } from './stale-account.repository'; const TICK_INTERVAL_MS = 60_000; -const LEADER_KEY = 'riot:bg-refresh:leader'; -const LEADER_TTL_SECONDS = 90; +const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader'; +const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000; const LEADER_RENEW_INTERVAL_MS = 30_000; const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000; const PER_TICK_CANDIDATE_LIMIT = 20; @@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler private tickTimer?: NodeJS.Timeout; private leaderRenewTimer?: NodeJS.Timeout; private tickInProgress = false; - private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`; + private leaderLockOwnership: DistributedLockOwnership | undefined; constructor( private readonly redis: RedisService, + private readonly distributedLockService: DistributedLockService, private readonly rateLimiter: RiotRateLimiterService, private readonly enrichmentService: RiotAccountEnrichmentService, private readonly staleAccountRepository: StaleAccountRepository, @@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler void this.safeTick(); } - public onModuleDestroy(): void { + public async onModuleDestroy(): Promise { if (this.tickTimer) clearInterval(this.tickTimer); if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer); + if (!this.leaderLockOwnership) return; + await this.distributedLockService + .releaseLockIfOwned(this.leaderLockOwnership) + .catch((err: unknown) => { + this.logger.warn( + `Background refresh leader release failed: ${ + err instanceof Error ? err.message : err + }`, + ); + }); + this.leaderLockOwnership = undefined; } private async safeTick(): Promise { @@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler } private async tryBecomeLeader(): Promise { - const current = await this.redis.getString(LEADER_KEY); - if (current === this.consumer) { - await this.redis.setString( - LEADER_KEY, - this.consumer, - LEADER_TTL_SECONDS, - ); - return true; + if (this.leaderLockOwnership) { + const didExtendLeaderLock = + await this.distributedLockService.extendLockIfOwned( + this.leaderLockOwnership, + ); + if (didExtendLeaderLock) return true; + this.leaderLockOwnership = undefined; } - if (current) return false; - return this.redis.setStringIfAbsent( - LEADER_KEY, - this.consumer, - LEADER_TTL_SECONDS, - ); + const leaderLockOwnership = await this.distributedLockService.acquireLock({ + lockKey: LEADER_LOCK_KEY, + lockExpirationDurationMilliseconds: + LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS, + }); + this.leaderLockOwnership = leaderLockOwnership ?? undefined; + return Boolean(leaderLockOwnership); } private async renewLeaderIfHeld(): Promise { try { - const current = await this.redis.getString(LEADER_KEY); - if (current !== this.consumer) return; - await this.redis.setString( - LEADER_KEY, - this.consumer, - LEADER_TTL_SECONDS, - ); + if (!this.leaderLockOwnership) return; + const didExtendLeaderLock = + await this.distributedLockService.extendLockIfOwned( + this.leaderLockOwnership, + ); + if (didExtendLeaderLock) return; + this.leaderLockOwnership = undefined; } catch (err) { this.logger.warn( `Background refresh leader renew failed: ${ diff --git a/apps/api/src/modules/riot-job-queue/riot-job-worker.service.spec.ts b/apps/api/src/modules/riot-job-queue/riot-job-worker.service.spec.ts index def3f7bf..9f9c866b 100644 --- a/apps/api/src/modules/riot-job-queue/riot-job-worker.service.spec.ts +++ b/apps/api/src/modules/riot-job-queue/riot-job-worker.service.spec.ts @@ -1,5 +1,6 @@ import mongoose from 'mongoose'; import { ConnectionService } from '../global/database/connection.service'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import { RedisService } from '../global/redis/redis.service'; import { LegacyImportService } from '../legacy-import/legacy-import.service'; import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service'; @@ -19,6 +20,16 @@ function makeWorker() { acknowledgeStream: jest.fn().mockResolvedValue(undefined), deleteStreamEntries: jest.fn().mockResolvedValue(undefined), } as unknown as jest.Mocked; + const distributedLockService = { + runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) => + operation({ + lockKey: options.lockKey, + lockOwnershipToken: 'lock-owner', + lockExpirationDurationMilliseconds: + options.lockExpirationDurationMilliseconds, + }), + ), + } as unknown as jest.Mocked; const jobQueueService = { markProcessing: jest.fn().mockResolvedValue(undefined), markCompleted: jest.fn().mockResolvedValue({ @@ -54,6 +65,7 @@ function makeWorker() { const worker = new RiotJobWorkerService( redisService, + distributedLockService, jobQueueService, recentMatchesService, connectionService, @@ -66,6 +78,7 @@ function makeWorker() { return { worker, redisService, + distributedLockService, jobQueueService, recentMatchesService, connectionService, @@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => { redisService.deleteStreamEntries.mock.invocationCallOrder[0], ); }); + + it('leaves a stream entry pending when another worker owns the job processing lock', async () => { + const { worker, distributedLockService, redisService } = makeWorker(); + distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null); + const userId = new mongoose.Types.ObjectId().toString(); + const job: RiotRefreshSubJob = { + jobId: 'job-1', + parentJobId: 'parent-1', + userId, + puuid: 'puuid-1', + region: 'asia', + platform: 'kr', + queueGroup: 'summoners-rift', + queueIds: [420], + requestedAt: Date.now(), + priority: 'high', + }; + + await ( + worker as unknown as { + processEntry: ( + stream: string, + entry: { id: string; message: Record }, + ) => Promise; + } + ).processEntry('riot:queue:high', { + id: '1-0', + message: { job: JSON.stringify(job) }, + }); + + expect(redisService.acknowledgeStream).not.toHaveBeenCalled(); + expect(redisService.deleteStreamEntries).not.toHaveBeenCalled(); + }); }); diff --git a/apps/api/src/modules/riot-job-queue/riot-job-worker.service.ts b/apps/api/src/modules/riot-job-queue/riot-job-worker.service.ts index 48aad1f1..02185c56 100644 --- a/apps/api/src/modules/riot-job-queue/riot-job-worker.service.ts +++ b/apps/api/src/modules/riot-job-queue/riot-job-worker.service.ts @@ -6,6 +6,10 @@ import { } from '@nestjs/common'; import mongoose from 'mongoose'; import { ConnectionService } from '../global/database/connection.service'; +import { + DistributedLockOwnership, + DistributedLockService, +} from '../global/redis/distributed-lock.service'; import { RedisService, StreamEntry } from '../global/redis/redis.service'; import { LegacyImportService } from '../legacy-import/legacy-import.service'; import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types'; @@ -23,17 +27,24 @@ import { import { RiotJobQueueService } from './riot-job-queue.service'; import { RiotRefreshGateway } from './riot-refresh.gateway'; -const LEADER_KEY = 'riot:worker:leader'; -const LEADER_TTL_SECONDS = 60; +const RIOT_JOB_WORKER_LEADER_LOCK_KEY = + 'distributed-lock:riot-job-worker:leader'; +const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000; +const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000; +const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000; +const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000; +const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1; @Injectable() export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(RiotJobWorkerService.name); - private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`; + private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`; + private leaderLockOwnership: DistributedLockOwnership | undefined; private stopped = false; constructor( private readonly redisService: RedisService, + private readonly distributedLockService: DistributedLockService, private readonly jobQueueService: RiotJobQueueService, private readonly recentMatchesService: RiotRecentMatchesService, private readonly connectionService: ConnectionService, @@ -48,8 +59,19 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { void this.loop(); } - public onModuleDestroy() { + public async onModuleDestroy() { this.stopped = true; + if (!this.leaderLockOwnership) return; + await this.distributedLockService + .releaseLockIfOwned(this.leaderLockOwnership) + .catch((error: unknown) => { + this.logger.warn( + `Failed to release Riot job worker leader lock: ${ + error instanceof Error ? error.message : error + }`, + ); + }); + this.leaderLockOwnership = undefined; } private async loop(): Promise { @@ -63,7 +85,9 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { await this.jobQueueService.promoteDueScheduledJobs(); const entry = + (await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ?? (await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ?? + (await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ?? (await this.readOne(RIOT_JOB_LOW_STREAM, 1_000)); if (!entry) continue; @@ -89,6 +113,31 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { return; } + const didProcessJob = await this.distributedLockService.runWithRenewedLock( + { + lockKey: this.jobProcessingLockKey(job), + lockExpirationDurationMilliseconds: + RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS, + lockRenewalIntervalMilliseconds: + RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS, + }, + async () => { + await this.processJob(job); + return true; + }, + ); + + if (!didProcessJob) { + this.logger.debug( + `Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`, + ); + return; + } + + await this.acknowledgeAndDelete(stream, entry.id); + } + + private async processJob(job: RiotJobPayload): Promise { try { if (this.isTournamentResultJob(job)) { await this.tournamentResultService.processPollJob(job); @@ -136,8 +185,6 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { error instanceof Error ? error.message : error }`, ); - } finally { - await this.acknowledgeAndDelete(stream, entry.id); } } @@ -228,15 +275,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { ): Promise<{ stream: string; message: StreamEntry } | null> { const result = await this.redisService.readGroup( stream, - RIOT_JOB_GROUP, - this.consumer, - '>', - { count: 1, blockMs }, + RIOT_JOB_GROUP, + this.consumerIdentifier, + '>', + { count: 1, blockMs }, ); const message = result?.[0]?.messages[0]; return message ? { stream, message } : null; } + private async readOneStalePendingMessage( + stream: string, + ): Promise<{ stream: string; message: StreamEntry } | null> { + const result = await this.redisService.claimStalePendingStreamMessages({ + key: stream, + group: RIOT_JOB_GROUP, + consumer: this.consumerIdentifier, + stalePendingMessageMinimumIdleMilliseconds: + STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS, + count: STALE_PENDING_MESSAGE_CLAIM_COUNT, + }); + const message = result.messages[0]; + return message ? { stream, message } : null; + } + private async ensureConsumerGroups(): Promise { await Promise.all( [RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => { @@ -252,21 +314,26 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { } private async tryBecomeLeader(): Promise { - const current = await this.redisService.getString(LEADER_KEY); - if (current === this.consumer) { - await this.redisService.setString( - LEADER_KEY, - this.consumer, - LEADER_TTL_SECONDS, - ); - return true; + if (this.leaderLockOwnership) { + const didExtendLeaderLock = + await this.distributedLockService.extendLockIfOwned( + this.leaderLockOwnership, + ); + if (didExtendLeaderLock) return true; + this.leaderLockOwnership = undefined; } - if (current) return false; - return this.redisService.setStringIfAbsent( - LEADER_KEY, - this.consumer, - LEADER_TTL_SECONDS, - ); + + const leaderLockOwnership = await this.distributedLockService.acquireLock({ + lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY, + lockExpirationDurationMilliseconds: + RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS, + }); + this.leaderLockOwnership = leaderLockOwnership ?? undefined; + return Boolean(leaderLockOwnership); + } + + private jobProcessingLockKey(job: RiotJobPayload): string { + return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`; } private sleep(ms: number): Promise { diff --git a/apps/api/src/modules/riot/riot-recent-matches.service.ts b/apps/api/src/modules/riot/riot-recent-matches.service.ts index 2d96434a..9dd6112e 100644 --- a/apps/api/src/modules/riot/riot-recent-matches.service.ts +++ b/apps/api/src/modules/riot/riot-recent-matches.service.ts @@ -4,14 +4,14 @@ import type { RiotMatch as StoredRiotMatch, } from '../global/database/schema'; import { ConnectionService } from '../global/database/connection.service'; -import { RedisService } from '../global/redis/redis.service'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import type { RiotRegionalRoute } from './riot.types'; import { RiotMatchService } from './riot-match.service'; export const RIOT_RECENT_MATCH_LIMIT = 20; export const RIOT_SOLO_QUEUE_ID = 420; -const MATCH_ENRICHMENT_LOCK_TTL_SECONDS = 60; +const MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000; const MATCH_DETAIL_CONCURRENCY = 2; const DEFAULT_REGION: RiotRegionalRoute = 'asia'; @@ -22,7 +22,7 @@ export class RiotRecentMatchesService { constructor( private readonly connectionService: ConnectionService, private readonly matchService: RiotMatchService, - private readonly redisService: RedisService, + private readonly distributedLockService: DistributedLockService, ) {} public async syncRecentMatches(params: { @@ -34,83 +34,84 @@ export class RiotRecentMatchesService { const region = params.region ?? DEFAULT_REGION; const limit = params.limit ?? RIOT_RECENT_MATCH_LIMIT; const queueId = params.queueId ?? RIOT_SOLO_QUEUE_ID; - const lockKey = `riot:enrich:matches:${params.puuid}`; - const lockValue = `${process.pid}:${Date.now()}`; - - const acquired = await this.redisService.setStringIfAbsent( - lockKey, - lockValue, - MATCH_ENRICHMENT_LOCK_TTL_SECONDS, - ); - if (!acquired) { - return this.loadFromDb({ puuid: params.puuid, limit, queueId }); - } - - try { - const matchIds = await this.matchService.getMatchIdsByPuuid(params.puuid, { - region, - queue: queueId, - count: limit, - }); - if (matchIds.length === 0) { - return this.loadFromDb({ puuid: params.puuid, limit, queueId }); - } - - const RiotMatchModel = await this.connectionService.getModel('riotMatch'); - const existingMatches = await RiotMatchModel.find({ - matchId: { $in: matchIds }, - }) - .select({ matchId: 1 }) - .lean() - .exec(); - const existingMatchIds = new Set(existingMatches.map((match) => match.matchId)); - const missingMatchIds = matchIds.filter((matchId) => !existingMatchIds.has(matchId)); - - await runWithConcurrency(missingMatchIds, MATCH_DETAIL_CONCURRENCY, async (matchId) => { - try { - const dto = await this.matchService.getMatch(matchId, { region }); - if (dto.info.queueId !== queueId) { - this.logger.warn( - `Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`, - ); - return; - } - - const payload: Partial = { - matchId: dto.metadata.matchId, - gameCreation: dto.info.gameCreation, - gameDuration: dto.info.gameDuration, - gameVersion: dto.info.gameVersion, - queueId: dto.info.queueId, - participantPuuids: dto.metadata.participants, - raw: dto as unknown as Record, - fetchedAt: new Date(), - }; - - await RiotMatchModel.updateOne( - { matchId: dto.metadata.matchId }, - { $setOnInsert: payload }, - { upsert: true }, - ).exec(); - } catch (error: unknown) { - this.logger.warn( - `Skipping Riot match ${matchId} while syncing ${params.puuid}: ${ - error instanceof Error ? error.message : error - }`, - ); - } - }); - - return this.loadFromDb({ puuid: params.puuid, limit, queueId }); - } finally { - await this.redisService.delete(lockKey).catch((error: unknown) => { - this.logger.warn( - `Failed to release recent matches lock for ${params.puuid}: ${ - (error as Error)?.message ?? error - }`, + const result = await this.distributedLockService.runWithLock( + { + lockKey: `distributed-lock:riot-recent-matches:sync:puuid:${params.puuid}`, + lockExpirationDurationMilliseconds: + MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS, + }, + async () => { + const matchIds = await this.matchService.getMatchIdsByPuuid( + params.puuid, + { + region, + queue: queueId, + count: limit, + }, ); - }); - } + if (matchIds.length === 0) { + return this.loadFromDb({ puuid: params.puuid, limit, queueId }); + } + + const RiotMatchModel = + await this.connectionService.getModel('riotMatch'); + const existingMatches = await RiotMatchModel.find({ + matchId: { $in: matchIds }, + }) + .select({ matchId: 1 }) + .lean() + .exec(); + const existingMatchIds = new Set( + existingMatches.map((match) => match.matchId), + ); + const missingMatchIds = matchIds.filter( + (matchId) => !existingMatchIds.has(matchId), + ); + + await runWithConcurrency( + missingMatchIds, + MATCH_DETAIL_CONCURRENCY, + async (matchId) => { + try { + const dto = await this.matchService.getMatch(matchId, { region }); + if (dto.info.queueId !== queueId) { + this.logger.warn( + `Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`, + ); + return; + } + + const payload: Partial = { + matchId: dto.metadata.matchId, + gameCreation: dto.info.gameCreation, + gameDuration: dto.info.gameDuration, + gameVersion: dto.info.gameVersion, + queueId: dto.info.queueId, + participantPuuids: dto.metadata.participants, + raw: dto as unknown as Record, + fetchedAt: new Date(), + }; + + await RiotMatchModel.updateOne( + { matchId: dto.metadata.matchId }, + { $setOnInsert: payload }, + { upsert: true }, + ).exec(); + } catch (error: unknown) { + this.logger.warn( + `Skipping Riot match ${matchId} while syncing ${params.puuid}: ${ + error instanceof Error ? error.message : error + }`, + ); + } + }, + ); + + return this.loadFromDb({ puuid: params.puuid, limit, queueId }); + }, + ); + + return result ?? this.loadFromDb({ puuid: params.puuid, limit, queueId }); } public async ensureFreshForQueue(params: { diff --git a/apps/api/src/modules/tournament-code/tournament-code.scheduler.spec.ts b/apps/api/src/modules/tournament-code/tournament-code.scheduler.spec.ts index 56e87f8c..c570fd1e 100644 --- a/apps/api/src/modules/tournament-code/tournament-code.scheduler.spec.ts +++ b/apps/api/src/modules/tournament-code/tournament-code.scheduler.spec.ts @@ -1,3 +1,4 @@ +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import { TournamentCodeScheduler } from './tournament-code.scheduler'; describe('TournamentCodeScheduler', () => { @@ -18,16 +19,28 @@ describe('TournamentCodeScheduler', () => { scheduleInitialPoll: params?.scheduleInitialPoll ?? jest.fn().mockResolvedValue(null), }; + const distributedLockService = { + runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) => + operation({ + lockKey: options.lockKey, + lockOwnershipToken: 'lock-owner', + lockExpirationDurationMilliseconds: + options.lockExpirationDurationMilliseconds, + }), + ), + } as unknown as jest.Mocked; const scheduler = new TournamentCodeScheduler( tournamentCodeService as never, autoProvisionService as never, tournamentResultService as never, + distributedLockService as never, ) as unknown as { attachPending(): Promise }; return { scheduler, tournamentCodeService, tournamentResultService, + distributedLockService, }; } @@ -59,4 +72,16 @@ describe('TournamentCodeScheduler', () => { expect(tournamentResultService.scheduleInitialPoll).not.toHaveBeenCalled(); }); + + it('does not attach pending tournament codes when another instance owns the scheduler lock', async () => { + const { scheduler, tournamentCodeService, distributedLockService } = + createScheduler(); + distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null); + + await scheduler.attachPending(); + + expect( + tournamentCodeService.attachPendingMatchDocuments, + ).not.toHaveBeenCalled(); + }); }); diff --git a/apps/api/src/modules/tournament-code/tournament-code.scheduler.ts b/apps/api/src/modules/tournament-code/tournament-code.scheduler.ts index 47846abb..3429c19d 100644 --- a/apps/api/src/modules/tournament-code/tournament-code.scheduler.ts +++ b/apps/api/src/modules/tournament-code/tournament-code.scheduler.ts @@ -1,8 +1,18 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import { TournamentResultService } from '../tournament-result/tournament-result.service'; import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service'; import { TournamentCodeService } from './tournament-code.service'; +const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY = + 'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches'; +const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY = + 'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes'; +const ENSURE_VENDOR_CODE_POOL_LOCK_KEY = + 'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool'; +const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000; +const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000; + @Injectable() export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(TournamentCodeScheduler.name); @@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy { private readonly tournamentCodeService: TournamentCodeService, private readonly autoProvisionService: TournamentCodeAutoProvisionService, private readonly tournamentResultService: TournamentResultService, + private readonly distributedLockService: DistributedLockService, ) {} public onModuleInit() { @@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy { private async attachPending() { try { - const matches = - await this.tournamentCodeService.attachPendingMatchDocuments(); - if (matches.length > 0) { - this.logger.debug( - `Attached tournament codes for ${matches.length} matches.`, - ); - for (const match of matches) { - await this.tournamentResultService.scheduleInitialPoll(match); - } - } + await this.distributedLockService.runWithRenewedLock( + { + lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY, + lockExpirationDurationMilliseconds: + SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS, + lockRenewalIntervalMilliseconds: + SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS, + }, + async () => { + const matches = + await this.tournamentCodeService.attachPendingMatchDocuments(); + if (matches.length > 0) { + this.logger.debug( + `Attached tournament codes for ${matches.length} matches.`, + ); + for (const match of matches) { + await this.tournamentResultService.scheduleInitialPoll(match); + } + } + }, + ); } catch (error) { this.logger.warn( `Failed to attach pending tournament codes: ${ @@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy { private async markExpired() { try { - await this.tournamentCodeService.markExpired(); + await this.distributedLockService.runWithRenewedLock( + { + lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY, + lockExpirationDurationMilliseconds: + SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS, + lockRenewalIntervalMilliseconds: + SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS, + }, + async () => { + await this.tournamentCodeService.markExpired(); + }, + ); } catch (error) { this.logger.warn( `Failed to mark expired tournament codes: ${ @@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy { private async ensureVendorPool() { try { - await this.autoProvisionService.ensurePool(); + await this.distributedLockService.runWithRenewedLock( + { + lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY, + lockExpirationDurationMilliseconds: + SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS, + lockRenewalIntervalMilliseconds: + SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS, + }, + async () => { + await this.autoProvisionService.ensurePool(); + }, + ); } catch (error) { this.logger.warn( `Failed to auto-provision tournament codes: ${ diff --git a/apps/api/src/modules/tournament-result/tournament-result.service.spec.ts b/apps/api/src/modules/tournament-result/tournament-result.service.spec.ts index 2d969164..7bebe1f0 100644 --- a/apps/api/src/modules/tournament-result/tournament-result.service.spec.ts +++ b/apps/api/src/modules/tournament-result/tournament-result.service.spec.ts @@ -1,5 +1,6 @@ import mongoose from 'mongoose'; import type { MatchDocument } from '../global/database/schema'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import type { RiotMatchDto } from '../riot/riot.types'; import { TournamentResultService } from './tournament-result.service'; @@ -16,15 +17,38 @@ type TournamentResultServiceInternals = TournamentResultService & { }; function createService() { + const distributedLockService = createDistributedLockService(); return new TournamentResultService( {} as never, {} as never, {} as never, {} as never, {} as never, + distributedLockService as never, ) as unknown as TournamentResultServiceInternals; } +function createDistributedLockService() { + return { + runWithLock: jest.fn().mockImplementation(async (options, operation) => + operation({ + lockKey: options.lockKey, + lockOwnershipToken: 'lock-owner', + lockExpirationDurationMilliseconds: + options.lockExpirationDurationMilliseconds, + }), + ), + runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) => + operation({ + lockKey: options.lockKey, + lockOwnershipToken: 'lock-owner', + lockExpirationDurationMilliseconds: + options.lockExpirationDurationMilliseconds, + }), + ), + } as unknown as jest.Mocked; +} + interface MatchOverrides { gameCount?: number; gameMode?: 'pair' | 'bracket' | 'round-robin'; @@ -281,12 +305,14 @@ describe('TournamentResultService tournament code polling', () => { const riotJobQueueService = { enqueueTournamentResultPoll: jest.fn().mockResolvedValue(undefined), }; + const distributedLockService = createDistributedLockService(); const service = new TournamentResultService( connectionService as never, riotMatchService as never, riotJobQueueService as never, { invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never, { buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never, + distributedLockService as never, ); return { service, match, code, riotMatchService }; @@ -455,12 +481,14 @@ describe('TournamentResultService manual result collection', () => { getMatchIdsByPuuid: jest.fn().mockResolvedValue(['KR_1234567890']), getMatch: jest.fn().mockResolvedValue(detail), }; + const distributedLockService = createDistributedLockService(); const service = new TournamentResultService( connectionService as never, riotMatchService as never, {} as never, { invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never, { buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never, + distributedLockService as never, ); return { diff --git a/apps/api/src/modules/tournament-result/tournament-result.service.ts b/apps/api/src/modules/tournament-result/tournament-result.service.ts index 85f97040..789acb1b 100644 --- a/apps/api/src/modules/tournament-result/tournament-result.service.ts +++ b/apps/api/src/modules/tournament-result/tournament-result.service.ts @@ -11,6 +11,7 @@ import { import { MatchStatsService } from '../community-stats/match-stats.service'; import { CommunityStatsService } from '../community-stats/community-stats.service'; import { ConnectionService } from '../global/database/connection.service'; +import { DistributedLockService } from '../global/redis/distributed-lock.service'; import type { MatchDocument, MatchGameResult, @@ -46,6 +47,12 @@ const POLL_EXPIRY_MS = 24 * 60 * 60_000; const EXPAND_AFTER_MS = 60 * 60_000; const EXPANDED_MEMBER_LIMIT_PER_TEAM = 2; const TOURNAMENT_LOOKUP_START_MARGIN_SECONDS = 60; +const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = + 30 * 60_000; +const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = + 5 * 60_000; +const TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS = + 5 * 60_000; interface ManualGameAssignment { gameNumber: number; @@ -64,11 +71,29 @@ export class TournamentResultService { private readonly riotJobQueueService: RiotJobQueueService, private readonly communityStatsService: CommunityStatsService, private readonly matchStatsService: MatchStatsService, + private readonly distributedLockService: DistributedLockService, ) {} public async scheduleInitialPoll( matchOrId: MatchDocument | mongoose.Types.ObjectId, priority: RiotJobPriority = 'high', + ): Promise { + const matchIdentifier = this.resolveMatchIdentifier(matchOrId); + return this.distributedLockService.runWithLock( + { + lockKey: this.tournamentResultPollProcessingLockKey(matchIdentifier), + lockExpirationDurationMilliseconds: + TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS, + lockAcquireTimeoutMilliseconds: 1_000, + lockRetryDelayMilliseconds: 100, + }, + async () => this.scheduleInitialPollWhileLockOwned(matchOrId, priority), + ); + } + + private async scheduleInitialPollWhileLockOwned( + matchOrId: MatchDocument | mongoose.Types.ObjectId, + priority: RiotJobPriority, ): Promise { const match = await this.loadMatch(matchOrId); if (match.status === 'finished') return null; @@ -195,6 +220,29 @@ export class TournamentResultService { } public async processPollJob(job: PollTournamentResultJob): Promise { + const didProcessPollJob = await this.distributedLockService.runWithRenewedLock( + { + lockKey: this.tournamentResultPollProcessingLockKey(job.matchId), + lockExpirationDurationMilliseconds: + TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS, + lockRenewalIntervalMilliseconds: + TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS, + }, + async () => { + await this.processPollJobWhileLockOwned(job); + return true; + }, + ); + + if (didProcessPollJob) return; + this.logger.debug( + `Skipped tournament result poll job ${job.jobId}; another worker is processing match ${job.matchId}.`, + ); + } + + private async processPollJobWhileLockOwned( + job: PollTournamentResultJob, + ): Promise { const MatchModel = await this.connectionService.getModel('match'); const TournamentCodeModel = await this.connectionService.getModel('tournamentCode'); @@ -818,12 +866,28 @@ export class TournamentResultService { private async reserveReadyDoubleEliminationCodes( match: MatchDocument, + ): Promise { + await this.distributedLockService.runWithLock( + { + lockKey: this.tournamentResultReadyCodeReservationLockKey( + match._id.toString(), + ), + lockExpirationDurationMilliseconds: + TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS, + }, + async () => this.reserveReadyDoubleEliminationCodesWhileLockOwned(match), + ); + } + + private async reserveReadyDoubleEliminationCodesWhileLockOwned( + match: MatchDocument, ): Promise { if ((match.gameMode ?? 'pair') !== 'double-elimination') return; const readyBracketMatches = (match.doubleElimination?.bracketMatches ?? []) .filter((bracketMatch) => bracketMatch.state === 'ready'); if (readyBracketMatches.length === 0) return; + const MatchModel = await this.connectionService.getModel('match'); const TournamentCodeModel = await this.connectionService.getModel('tournamentCode'); const existingCodes = @@ -871,8 +935,19 @@ export class TournamentResultService { } if (reservedIds.length === 0) return; - match.tournamentCodeIds.push(...reservedIds); - await match.save(); + await MatchModel.updateOne( + { _id: match._id }, + { $addToSet: { tournamentCodeIds: { $each: reservedIds } } }, + ).exec(); + const existingTournamentCodeIds = new Set( + match.tournamentCodeIds.map((tournamentCodeId) => + tournamentCodeId.toString(), + ), + ); + for (const reservedId of reservedIds) { + if (existingTournamentCodeIds.has(reservedId.toString())) continue; + match.tournamentCodeIds.push(reservedId); + } } private async markTournamentCodeUsedForManualResult( @@ -1126,6 +1201,24 @@ export class TournamentResultService { if (!match) throw new NotFoundException('Match not found.'); return match; } + + private resolveMatchIdentifier( + matchOrId: MatchDocument | mongoose.Types.ObjectId, + ): string { + return matchOrId instanceof mongoose.Types.ObjectId + ? matchOrId.toString() + : matchOrId._id.toString(); + } + + private tournamentResultPollProcessingLockKey(matchIdentifier: string): string { + return `distributed-lock:tournament-result:process-poll-job:match:${matchIdentifier}`; + } + + private tournamentResultReadyCodeReservationLockKey( + matchIdentifier: string, + ): string { + return `distributed-lock:tournament-result:reserve-ready-double-elimination-codes:match:${matchIdentifier}`; + } } function isRiotMatchDto(value: unknown): value is RiotMatchDto {