import { Injectable, Logger, OnModuleDestroy, OnModuleInit, } from '@nestjs/common'; import mongoose from 'mongoose'; import { ConnectionService } from '../global/database/connection.service'; import { DistributedLockOwnership, DistributedLockService, } from '../global/redis/distributed-lock.service'; import { RedisService, StreamEntry } from '../global/redis/redis.service'; import { LegacyImportService } from '../legacy-import/legacy-import.service'; import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types'; import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service'; import { RiotRecentMatchesService } from '../riot/riot-recent-matches.service'; import { TournamentResultService } from '../tournament-result/tournament-result.service'; import type { PollTournamentResultJob } from '../tournament-result/tournament-result.types'; import { RIOT_JOB_GROUP, RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM, RiotJobPayload, RiotRefreshSubJob, } from './riot-job-queue.types'; import { RiotJobQueueService } from './riot-job-queue.service'; import { RiotRefreshGateway } from './riot-refresh.gateway'; const RIOT_JOB_WORKER_LEADER_LOCK_KEY = 'distributed-lock:riot-job-worker:leader'; const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000; const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000; const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000; const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000; const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1; @Injectable() export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(RiotJobWorkerService.name); private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`; private leaderLockOwnership: DistributedLockOwnership | undefined; private stopped = false; constructor( private readonly redisService: RedisService, private readonly distributedLockService: DistributedLockService, private readonly jobQueueService: RiotJobQueueService, private readonly recentMatchesService: RiotRecentMatchesService, private readonly connectionService: ConnectionService, private readonly refreshGateway: RiotRefreshGateway, private readonly tournamentResultService: TournamentResultService, private readonly legacyImportService: LegacyImportService, private readonly enrichmentService: RiotAccountEnrichmentService, ) {} public async onModuleInit() { await this.ensureConsumerGroups(); void this.loop(); } public async onModuleDestroy() { this.stopped = true; if (!this.leaderLockOwnership) return; await this.distributedLockService .releaseLockIfOwned(this.leaderLockOwnership) .catch((error: unknown) => { this.logger.warn( `Failed to release Riot job worker leader lock: ${ error instanceof Error ? error.message : String(error) }`, ); }); this.leaderLockOwnership = undefined; } private async loop(): Promise { while (!this.stopped) { try { const isLeader = await this.tryBecomeLeader(); if (!isLeader) { await this.sleep(1_000); continue; } await this.jobQueueService.promoteDueScheduledJobs(); const entry = (await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ?? (await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ?? (await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ?? (await this.readOne(RIOT_JOB_LOW_STREAM, 1_000)); if (!entry) continue; await this.processEntry(entry.stream, entry.message); } catch (error) { this.logger.warn( `Riot job worker loop failed: ${ error instanceof Error ? error.message : error }`, ); await this.sleep(1_000); } } } private async processEntry( stream: string, entry: StreamEntry, ): Promise { const job = this.parseJob(entry); if (!job) { await this.acknowledgeAndDelete(stream, entry.id); return; } const didProcessJob = await this.distributedLockService.runWithRenewedLock( { lockKey: this.jobProcessingLockKey(job), lockExpirationDurationMilliseconds: RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS, lockRenewalIntervalMilliseconds: RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS, }, async () => { await this.processJob(job); return true; }, ); if (!didProcessJob) { this.logger.debug( `Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`, ); return; } await this.acknowledgeAndDelete(stream, entry.id); } private async processJob(job: RiotJobPayload): Promise { try { if (this.isTournamentResultJob(job)) { await this.tournamentResultService.processPollJob(job); return; } if (this.isLegacyImportJob(job)) { await this.legacyImportService.processMemberJob(job); return; } await this.jobQueueService.markProcessing(job); for (const queueId of job.queueIds) { await this.recentMatchesService.ensureFreshForQueue({ puuid: job.puuid, region: job.region, queueId, }); } await this.refreshRiotIdentity(job); await this.refreshLeagueEntries(job); await this.markUserFetchedAt(job); const event = await this.jobQueueService.markCompleted(job); this.refreshGateway.broadcastRefreshEvent(event); } catch (error) { if (this.isTournamentResultJob(job)) { this.logger.warn( `Tournament result poll job failed ${job.jobId}: ${ error instanceof Error ? error.message : error }`, ); return; } if (this.isLegacyImportJob(job)) { this.logger.warn( `Legacy import member job failed ${job.jobId}: ${ error instanceof Error ? error.message : error }`, ); return; } const event = await this.jobQueueService.markFailed(job, error); this.refreshGateway.broadcastRefreshEvent(event); this.logger.warn( `Riot refresh job failed ${job.jobId}: ${ error instanceof Error ? error.message : error }`, ); } } private async acknowledgeAndDelete( stream: string, id: string, ): Promise { await this.redisService.acknowledgeStream(stream, RIOT_JOB_GROUP, id); await this.deleteAcknowledgedEntry(stream, id); } private async deleteAcknowledgedEntry( stream: string, id: string, ): Promise { try { await this.redisService.deleteStreamEntries(stream, id); } catch (error) { this.logger.warn( `Failed to delete acknowledged Riot job stream entry ${stream} ${id}: ${ error instanceof Error ? error.message : error }`, ); } } private async refreshRiotIdentity(job: RiotRefreshSubJob): Promise { try { const result = await this.enrichmentService.refreshRiotIdentity({ userId: new mongoose.Types.ObjectId(job.userId), puuid: job.puuid, region: job.region, }); if (result !== 'failed') return; this.logger.warn( `Riot identity refresh failed ${job.jobId}: puuid=${job.puuid}`, ); } catch (error) { this.logger.warn( `Riot identity refresh failed ${job.jobId}: ${ error instanceof Error ? error.message : error }`, ); } } private async refreshLeagueEntries(job: RiotRefreshSubJob): Promise { try { const result = await this.enrichmentService.refreshLeagueEntries({ userId: new mongoose.Types.ObjectId(job.userId), puuid: job.puuid, platform: job.platform, }); if (result !== 'failed') return; this.logger.warn( `Riot league refresh failed ${job.jobId}: puuid=${job.puuid}`, ); } catch (error) { this.logger.warn( `Riot league refresh failed ${job.jobId}: ${ error instanceof Error ? error.message : error }`, ); } } private async markUserFetchedAt(job: RiotRefreshSubJob): Promise { const UserModel = await this.connectionService.getModel('user'); const now = new Date(); const $set: Record = { 'riotAccounts.$.matchIdsFetchedAt': now, }; for (const queueId of job.queueIds) { $set[`riotAccounts.$.matchIdsByQueueFetchedAt.${queueId}`] = now; } await UserModel.updateOne( { _id: new mongoose.Types.ObjectId(job.userId), 'riotAccounts.puuid': job.puuid, }, { $set }, ).exec(); } private parseJob(entry: StreamEntry): RiotJobPayload | null { const raw = entry.message.job; if (!raw) return null; try { return JSON.parse(raw) as RiotJobPayload; } catch { return null; } } private isTournamentResultJob( job: RiotJobPayload, ): job is PollTournamentResultJob { return job.kind === 'poll-tournament-result'; } private isLegacyImportJob(job: RiotJobPayload): job is LegacyImportMemberJob { return job.kind === 'legacy-import-member'; } private async readOne( stream: string, blockMs: number, ): Promise<{ stream: string; message: StreamEntry } | null> { const result = await this.redisService.readGroup( stream, RIOT_JOB_GROUP, this.consumerIdentifier, '>', { count: 1, blockMs }, ); const message = result?.[0]?.messages[0]; return message ? { stream, message } : null; } private async readOneStalePendingMessage( stream: string, ): Promise<{ stream: string; message: StreamEntry } | null> { const result = await this.redisService.claimStalePendingStreamMessages({ key: stream, group: RIOT_JOB_GROUP, consumer: this.consumerIdentifier, stalePendingMessageMinimumIdleMilliseconds: STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS, count: STALE_PENDING_MESSAGE_CLAIM_COUNT, }); const message = result.messages[0]; return message ? { stream, message } : null; } private async ensureConsumerGroups(): Promise { await Promise.all( [RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => { try { await this.redisService.createConsumerGroup(stream, RIOT_JOB_GROUP); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!message.includes('BUSYGROUP')) throw error; } }), ); } private async tryBecomeLeader(): Promise { if (this.leaderLockOwnership) { const didExtendLeaderLock = await this.distributedLockService.extendLockIfOwned( this.leaderLockOwnership, ); if (didExtendLeaderLock) return true; this.leaderLockOwnership = undefined; } const leaderLockOwnership = await this.distributedLockService.acquireLock({ lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY, lockExpirationDurationMilliseconds: RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS, }); this.leaderLockOwnership = leaderLockOwnership ?? undefined; return Boolean(leaderLockOwnership); } private jobProcessingLockKey(job: RiotJobPayload): string { return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`; } private sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } }