feat(api): add Riot refresh queue
This commit is contained in:
1 parent
c212505ffe
commit
4bc2df7bcc
6 files changed
+915
No files matched your search
@@ -0,0 +1,24 @@
|
|||||||
|
import { Controller, Get, NotFoundException, Param } from '@nestjs/common';
|
||||||
|
import { AllowPublicAccess } from '../auth/allow-public-access.decorator';
|
||||||
|
import { RiotJobQueueService } from './riot-job-queue.service';
|
||||||
|
|
||||||
|
@Controller('riot/jobs')
|
||||||
|
export class RiotJobQueueController {
|
||||||
|
constructor(private readonly jobQueueService: RiotJobQueueService) {}
|
||||||
|
|
||||||
|
@Get('parents/:parentJobId')
|
||||||
|
@AllowPublicAccess()
|
||||||
|
public async getParent(@Param('parentJobId') parentJobId: string) {
|
||||||
|
const status = await this.jobQueueService.getParentStatus(parentJobId);
|
||||||
|
if (!status) throw new NotFoundException('Riot parent job not found.');
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get(':jobId')
|
||||||
|
@AllowPublicAccess()
|
||||||
|
public async getJob(@Param('jobId') jobId: string) {
|
||||||
|
const status = await this.jobQueueService.getStatus(jobId);
|
||||||
|
if (!status) throw new NotFoundException('Riot job not found.');
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { forwardRef, Module } from '@nestjs/common';
|
||||||
|
import { LegacyImportModule } from '../legacy-import/legacy-import.module';
|
||||||
|
import { RiotModule } from '../riot/riot.module';
|
||||||
|
import { TournamentResultModule } from '../tournament-result/tournament-result.module';
|
||||||
|
import { RiotJobQueueController } from './riot-job-queue.controller';
|
||||||
|
import { RiotJobQueueService } from './riot-job-queue.service';
|
||||||
|
import { RiotJobWorkerService } from './riot-job-worker.service';
|
||||||
|
import { RiotRefreshGateway } from './riot-refresh.gateway';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [RiotModule, LegacyImportModule, forwardRef(() => TournamentResultModule)],
|
||||||
|
controllers: [RiotJobQueueController],
|
||||||
|
providers: [RiotJobQueueService, RiotJobWorkerService, RiotRefreshGateway],
|
||||||
|
exports: [RiotJobQueueService],
|
||||||
|
})
|
||||||
|
export class RiotJobQueueModule {}
|
||||||
@@ -0,0 +1,482 @@
|
|||||||
|
import {
|
||||||
|
BadRequestException,
|
||||||
|
HttpException,
|
||||||
|
HttpStatus,
|
||||||
|
Injectable,
|
||||||
|
NotFoundException,
|
||||||
|
} from '@nestjs/common';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import type {
|
||||||
|
CommunityDocument,
|
||||||
|
RiotAccount,
|
||||||
|
UserDocument,
|
||||||
|
} from '../global/database/schema';
|
||||||
|
import { GarnetService } from '../global/garnet/garnet.service';
|
||||||
|
import {
|
||||||
|
isRiotQueueGroup,
|
||||||
|
resolveRiotQueueIds,
|
||||||
|
type RiotQueueGroup,
|
||||||
|
} from '../riot/riot-queue.constants';
|
||||||
|
import {
|
||||||
|
EnqueueRefreshResult,
|
||||||
|
RIOT_JOB_AVG_SECONDS,
|
||||||
|
RIOT_JOB_DEDUP_TTL_SECONDS,
|
||||||
|
RIOT_JOB_HIGH_COOLDOWN_SECONDS,
|
||||||
|
RIOT_JOB_HIGH_STREAM,
|
||||||
|
RIOT_JOB_LOW_COOLDOWN_SECONDS,
|
||||||
|
RIOT_JOB_LOW_STREAM,
|
||||||
|
RIOT_JOB_SCHEDULED_ZSET,
|
||||||
|
RIOT_JOB_STATUS_TTL_SECONDS,
|
||||||
|
RiotJobPriority,
|
||||||
|
RiotJobState,
|
||||||
|
RiotJobStatus,
|
||||||
|
RiotParentJobStatus,
|
||||||
|
RiotRefreshEvent,
|
||||||
|
RiotRefreshSubJob,
|
||||||
|
} from './riot-job-queue.types';
|
||||||
|
import type {
|
||||||
|
EnqueueTournamentResultPollResult,
|
||||||
|
PollTournamentResultJob,
|
||||||
|
} from '../tournament-result/tournament-result.types';
|
||||||
|
|
||||||
|
interface RefreshContext {
|
||||||
|
community: CommunityDocument;
|
||||||
|
user: UserDocument;
|
||||||
|
account: RiotAccount;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class RiotJobQueueService {
|
||||||
|
constructor(
|
||||||
|
private readonly connectionService: ConnectionService,
|
||||||
|
private readonly garnetService: GarnetService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public async enqueueRefresh(params: {
|
||||||
|
communityId: mongoose.Types.ObjectId;
|
||||||
|
userId: mongoose.Types.ObjectId;
|
||||||
|
queueGroups: string[];
|
||||||
|
priority: RiotJobPriority;
|
||||||
|
requestedBy?: string;
|
||||||
|
}): Promise<EnqueueRefreshResult> {
|
||||||
|
const context = await this.resolveContext(params);
|
||||||
|
const queueGroups = this.normalizeQueueGroups(params.queueGroups);
|
||||||
|
const stream = this.resolveStream(params.priority);
|
||||||
|
const streamLength = await this.garnetService.streamLength(stream);
|
||||||
|
const dedupKeys = queueGroups.map((queueGroup) =>
|
||||||
|
this.dedupKey(context.account.puuid, queueGroup),
|
||||||
|
);
|
||||||
|
const existingJobIds = await Promise.all(
|
||||||
|
dedupKeys.map((key) => this.garnetService.getString(key)),
|
||||||
|
);
|
||||||
|
const needsNewJob = existingJobIds.some((jobId) => !jobId);
|
||||||
|
|
||||||
|
if (params.priority === 'high' && needsNewJob) {
|
||||||
|
this.assertHighQueueAccepting(streamLength);
|
||||||
|
this.assertRefreshBudget(context.account.lastRefreshRequestedAt, 'high');
|
||||||
|
}
|
||||||
|
if (params.priority === 'low' && needsNewJob) {
|
||||||
|
this.assertRefreshBudget(context.account.lastRefreshRequestedAt, 'low');
|
||||||
|
}
|
||||||
|
|
||||||
|
const parentJobId = randomUUID();
|
||||||
|
const subJobs: EnqueueRefreshResult['subJobs'] = [];
|
||||||
|
|
||||||
|
for (let index = 0; index < queueGroups.length; index += 1) {
|
||||||
|
const queueGroup = queueGroups[index];
|
||||||
|
const existingJobId = existingJobIds[index];
|
||||||
|
const queuePosition = streamLength + subJobs.length + 1;
|
||||||
|
|
||||||
|
if (existingJobId) {
|
||||||
|
const existingStatus = await this.getStatus(existingJobId);
|
||||||
|
subJobs.push({
|
||||||
|
jobId: existingJobId,
|
||||||
|
queueGroup,
|
||||||
|
alreadyQueued: true,
|
||||||
|
etaSeconds: existingStatus?.etaSeconds ?? queuePosition * RIOT_JOB_AVG_SECONDS,
|
||||||
|
queuePosition: existingStatus?.queuePosition ?? queuePosition,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const job: RiotRefreshSubJob = {
|
||||||
|
jobId: randomUUID(),
|
||||||
|
parentJobId,
|
||||||
|
userId: context.user._id.toString(),
|
||||||
|
puuid: context.account.puuid,
|
||||||
|
region: context.community.riotRegion ?? 'asia',
|
||||||
|
platform: context.community.riotPlatform ?? 'kr',
|
||||||
|
queueGroup,
|
||||||
|
queueIds: resolveRiotQueueIds(queueGroup, 'all'),
|
||||||
|
requestedAt: Date.now(),
|
||||||
|
requestedBy: params.requestedBy,
|
||||||
|
priority: params.priority,
|
||||||
|
};
|
||||||
|
const acquired = await this.garnetService.setStringIfAbsent(
|
||||||
|
dedupKeys[index],
|
||||||
|
job.jobId,
|
||||||
|
RIOT_JOB_DEDUP_TTL_SECONDS,
|
||||||
|
);
|
||||||
|
if (!acquired) {
|
||||||
|
const jobId = await this.garnetService.getString(dedupKeys[index]);
|
||||||
|
if (!jobId) continue;
|
||||||
|
const status = await this.getStatus(jobId);
|
||||||
|
subJobs.push({
|
||||||
|
jobId,
|
||||||
|
queueGroup,
|
||||||
|
alreadyQueued: true,
|
||||||
|
etaSeconds: status?.etaSeconds ?? queuePosition * RIOT_JOB_AVG_SECONDS,
|
||||||
|
queuePosition: status?.queuePosition ?? queuePosition,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const status: RiotJobStatus = {
|
||||||
|
jobId: job.jobId,
|
||||||
|
parentJobId,
|
||||||
|
state: 'queued',
|
||||||
|
priority: params.priority,
|
||||||
|
queueGroup,
|
||||||
|
queueIds: job.queueIds,
|
||||||
|
puuid: job.puuid,
|
||||||
|
userId: job.userId,
|
||||||
|
enqueuedAt: new Date(job.requestedAt).toISOString(),
|
||||||
|
etaSeconds: queuePosition * RIOT_JOB_AVG_SECONDS,
|
||||||
|
queuePosition,
|
||||||
|
};
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
this.setStatus(status),
|
||||||
|
this.garnetService.appendStream(
|
||||||
|
stream,
|
||||||
|
{ job: JSON.stringify(job) },
|
||||||
|
{ maxLen: 10_000 },
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
subJobs.push({
|
||||||
|
jobId: job.jobId,
|
||||||
|
queueGroup,
|
||||||
|
alreadyQueued: false,
|
||||||
|
etaSeconds: status.etaSeconds,
|
||||||
|
queuePosition,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.setParent(parentJobId, subJobs.map((job) => job.jobId));
|
||||||
|
if (params.priority === 'high' && needsNewJob) {
|
||||||
|
await this.markRefreshRequested(context.user._id, context.account.puuid);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
parentJobId,
|
||||||
|
subJobs,
|
||||||
|
totalEtaSeconds: subJobs.reduce((sum, job) => sum + job.etaSeconds, 0),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async enqueueTournamentResultPoll(params: {
|
||||||
|
matchId: string;
|
||||||
|
attempt: number;
|
||||||
|
expandedToAllPuuids: boolean;
|
||||||
|
priority: RiotJobPriority;
|
||||||
|
deferUntilMs?: number;
|
||||||
|
}): Promise<EnqueueTournamentResultPollResult> {
|
||||||
|
const job: PollTournamentResultJob = {
|
||||||
|
kind: 'poll-tournament-result',
|
||||||
|
jobId: randomUUID(),
|
||||||
|
matchId: params.matchId,
|
||||||
|
attempt: params.attempt,
|
||||||
|
expandedToAllPuuids: params.expandedToAllPuuids,
|
||||||
|
enqueuedAt: Date.now(),
|
||||||
|
priority: params.priority,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (params.deferUntilMs && params.deferUntilMs > Date.now()) {
|
||||||
|
await this.scheduleJob(job, params.deferUntilMs);
|
||||||
|
} else {
|
||||||
|
await this.appendJob(job);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
jobId: job.jobId,
|
||||||
|
matchId: job.matchId,
|
||||||
|
priority: job.priority,
|
||||||
|
attempt: job.attempt,
|
||||||
|
expandedToAllPuuids: job.expandedToAllPuuids,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async promoteDueScheduledJobs(limit = 100): Promise<number> {
|
||||||
|
const client = this.garnetService.raw();
|
||||||
|
const raw = (await client.sendCommand([
|
||||||
|
'ZRANGEBYSCORE',
|
||||||
|
RIOT_JOB_SCHEDULED_ZSET,
|
||||||
|
'-inf',
|
||||||
|
String(Date.now()),
|
||||||
|
'LIMIT',
|
||||||
|
'0',
|
||||||
|
String(limit),
|
||||||
|
])) as string[] | null;
|
||||||
|
const entries = raw ?? [];
|
||||||
|
let promoted = 0;
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
const removed = (await client.sendCommand([
|
||||||
|
'ZREM',
|
||||||
|
RIOT_JOB_SCHEDULED_ZSET,
|
||||||
|
entry,
|
||||||
|
])) as number;
|
||||||
|
if (removed !== 1) continue;
|
||||||
|
const job = this.parseScheduledJob(entry);
|
||||||
|
if (!job) continue;
|
||||||
|
await this.appendJob(job);
|
||||||
|
promoted += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return promoted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getStatus(jobId: string): Promise<RiotJobStatus | null> {
|
||||||
|
return this.garnetService.getJson<RiotJobStatus>(this.statusKey(jobId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getParentStatus(
|
||||||
|
parentJobId: string,
|
||||||
|
): Promise<RiotParentJobStatus | null> {
|
||||||
|
const jobIds = await this.garnetService.getJson<string[]>(
|
||||||
|
this.parentKey(parentJobId),
|
||||||
|
);
|
||||||
|
if (!jobIds) return null;
|
||||||
|
|
||||||
|
const subJobs = (
|
||||||
|
await Promise.all(jobIds.map((jobId) => this.getStatus(jobId)))
|
||||||
|
).filter((status): status is RiotJobStatus => Boolean(status));
|
||||||
|
const done = subJobs.filter((job) =>
|
||||||
|
job.state === 'completed' || job.state === 'failed',
|
||||||
|
).length;
|
||||||
|
const state = this.resolveParentState(subJobs);
|
||||||
|
|
||||||
|
return {
|
||||||
|
parentJobId,
|
||||||
|
state,
|
||||||
|
done,
|
||||||
|
total: subJobs.length,
|
||||||
|
etaSeconds: subJobs
|
||||||
|
.filter((job) => job.state === 'queued' || job.state === 'processing')
|
||||||
|
.reduce((sum, job) => sum + job.etaSeconds, 0),
|
||||||
|
subJobs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async markProcessing(job: RiotRefreshSubJob): Promise<RiotJobStatus> {
|
||||||
|
const status = await this.requireStatus(job.jobId);
|
||||||
|
const next: RiotJobStatus = {
|
||||||
|
...status,
|
||||||
|
state: 'processing',
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
etaSeconds: RIOT_JOB_AVG_SECONDS,
|
||||||
|
queuePosition: 0,
|
||||||
|
};
|
||||||
|
await this.setStatus(next);
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async markCompleted(
|
||||||
|
job: RiotRefreshSubJob,
|
||||||
|
): Promise<RiotRefreshEvent> {
|
||||||
|
const status = await this.requireStatus(job.jobId);
|
||||||
|
const next: RiotJobStatus = {
|
||||||
|
...status,
|
||||||
|
state: 'completed',
|
||||||
|
finishedAt: new Date().toISOString(),
|
||||||
|
etaSeconds: 0,
|
||||||
|
queuePosition: 0,
|
||||||
|
};
|
||||||
|
await Promise.all([
|
||||||
|
this.setStatus(next),
|
||||||
|
this.garnetService.delete(this.dedupKey(job.puuid, job.queueGroup)),
|
||||||
|
]);
|
||||||
|
return this.toEvent(next);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async markFailed(
|
||||||
|
job: RiotRefreshSubJob,
|
||||||
|
error: unknown,
|
||||||
|
): Promise<RiotRefreshEvent> {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
const status = await this.requireStatus(job.jobId);
|
||||||
|
const next: RiotJobStatus = {
|
||||||
|
...status,
|
||||||
|
state: 'failed',
|
||||||
|
finishedAt: new Date().toISOString(),
|
||||||
|
error: message,
|
||||||
|
etaSeconds: 0,
|
||||||
|
queuePosition: 0,
|
||||||
|
};
|
||||||
|
await Promise.all([
|
||||||
|
this.setStatus(next),
|
||||||
|
this.garnetService.delete(this.dedupKey(job.puuid, job.queueGroup)),
|
||||||
|
]);
|
||||||
|
return this.toEvent(next);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveContext(params: {
|
||||||
|
communityId: mongoose.Types.ObjectId;
|
||||||
|
userId: mongoose.Types.ObjectId;
|
||||||
|
}): Promise<RefreshContext> {
|
||||||
|
const UserModel = await this.connectionService.getModel('user');
|
||||||
|
const CommunityModel = await this.connectionService.getModel('community');
|
||||||
|
const [user, community] = await Promise.all([
|
||||||
|
UserModel.findById(params.userId).exec(),
|
||||||
|
CommunityModel.findById(params.communityId).exec(),
|
||||||
|
]);
|
||||||
|
if (!community) throw new NotFoundException('Community not found.');
|
||||||
|
if (!user || !user.communityId.equals(params.communityId)) {
|
||||||
|
throw new NotFoundException('User not found.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const account =
|
||||||
|
user.riotAccounts.find((riotAccount) => riotAccount.isPrimary) ??
|
||||||
|
user.riotAccounts[0];
|
||||||
|
if (!account) throw new NotFoundException('Riot account not found.');
|
||||||
|
return { community, user, account };
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeQueueGroups(queueGroups: string[]): RiotQueueGroup[] {
|
||||||
|
const values = queueGroups.length > 0 ? queueGroups : ['summoners-rift'];
|
||||||
|
const unique = [...new Set(values)];
|
||||||
|
if (!unique.every(isRiotQueueGroup)) {
|
||||||
|
throw new BadRequestException('Invalid queueGroups.');
|
||||||
|
}
|
||||||
|
return unique;
|
||||||
|
}
|
||||||
|
|
||||||
|
private assertHighQueueAccepting(streamLength: number): void {
|
||||||
|
if (streamLength <= 50) return;
|
||||||
|
throw new HttpException(
|
||||||
|
{
|
||||||
|
reason: 'queue-full',
|
||||||
|
retryAfterSeconds: RIOT_JOB_AVG_SECONDS * streamLength,
|
||||||
|
},
|
||||||
|
HttpStatus.TOO_MANY_REQUESTS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private assertRefreshBudget(
|
||||||
|
lastRequestedAt: Date | undefined,
|
||||||
|
priority: RiotJobPriority,
|
||||||
|
): void {
|
||||||
|
if (!lastRequestedAt) return;
|
||||||
|
const cooldownSeconds =
|
||||||
|
priority === 'high'
|
||||||
|
? RIOT_JOB_HIGH_COOLDOWN_SECONDS
|
||||||
|
: RIOT_JOB_LOW_COOLDOWN_SECONDS;
|
||||||
|
const elapsedSeconds = (Date.now() - lastRequestedAt.getTime()) / 1000;
|
||||||
|
if (elapsedSeconds >= cooldownSeconds) return;
|
||||||
|
throw new HttpException(
|
||||||
|
{
|
||||||
|
reason: 'cooldown',
|
||||||
|
retryAfterSeconds: Math.ceil(cooldownSeconds - elapsedSeconds),
|
||||||
|
},
|
||||||
|
HttpStatus.TOO_MANY_REQUESTS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async markRefreshRequested(
|
||||||
|
userId: mongoose.Types.ObjectId,
|
||||||
|
puuid: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const UserModel = await this.connectionService.getModel('user');
|
||||||
|
await UserModel.updateOne(
|
||||||
|
{ _id: userId, 'riotAccounts.puuid': puuid },
|
||||||
|
{ $set: { 'riotAccounts.$.lastRefreshRequestedAt': new Date() } },
|
||||||
|
).exec();
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveStream(priority: RiotJobPriority): string {
|
||||||
|
return priority === 'high' ? RIOT_JOB_HIGH_STREAM : RIOT_JOB_LOW_STREAM;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async appendJob(job: RiotRefreshSubJob | PollTournamentResultJob) {
|
||||||
|
await this.garnetService.appendStream(
|
||||||
|
this.resolveStream(job.priority),
|
||||||
|
{ job: JSON.stringify(job) },
|
||||||
|
{ maxLen: 10_000 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async scheduleJob(
|
||||||
|
job: PollTournamentResultJob,
|
||||||
|
deferUntilMs: number,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.garnetService.raw().sendCommand([
|
||||||
|
'ZADD',
|
||||||
|
RIOT_JOB_SCHEDULED_ZSET,
|
||||||
|
String(deferUntilMs),
|
||||||
|
JSON.stringify(job),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseScheduledJob(value: string): PollTournamentResultJob | null {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(value) as PollTournamentResultJob;
|
||||||
|
return parsed.kind === 'poll-tournament-result' ? parsed : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveParentState(subJobs: RiotJobStatus[]): RiotJobState {
|
||||||
|
if (subJobs.some((job) => job.state === 'processing')) return 'processing';
|
||||||
|
if (subJobs.some((job) => job.state === 'queued')) return 'queued';
|
||||||
|
if (subJobs.some((job) => job.state === 'failed')) return 'failed';
|
||||||
|
return 'completed';
|
||||||
|
}
|
||||||
|
|
||||||
|
private async requireStatus(jobId: string): Promise<RiotJobStatus> {
|
||||||
|
const status = await this.getStatus(jobId);
|
||||||
|
if (!status) throw new Error(`Missing riot job status: ${jobId}`);
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async setStatus(status: RiotJobStatus): Promise<void> {
|
||||||
|
await this.garnetService.setJson(
|
||||||
|
this.statusKey(status.jobId),
|
||||||
|
status,
|
||||||
|
RIOT_JOB_STATUS_TTL_SECONDS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async setParent(parentJobId: string, jobIds: string[]): Promise<void> {
|
||||||
|
await this.garnetService.setJson(
|
||||||
|
this.parentKey(parentJobId),
|
||||||
|
jobIds,
|
||||||
|
RIOT_JOB_STATUS_TTL_SECONDS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private toEvent(status: RiotJobStatus): RiotRefreshEvent {
|
||||||
|
return {
|
||||||
|
parentJobId: status.parentJobId,
|
||||||
|
jobId: status.jobId,
|
||||||
|
queueGroup: status.queueGroup,
|
||||||
|
puuid: status.puuid,
|
||||||
|
state: status.state,
|
||||||
|
error: status.error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private statusKey(jobId: string): string {
|
||||||
|
return `riot:job:status:${jobId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private parentKey(parentJobId: string): string {
|
||||||
|
return `riot:job:parent:${parentJobId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private dedupKey(puuid: string, queueGroup: RiotQueueGroup): string {
|
||||||
|
return `riot:job:dedup:${puuid}:${queueGroup}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import type {
|
||||||
|
RiotPlatformRoute,
|
||||||
|
RiotRegionalRoute,
|
||||||
|
} from '../riot/riot.types';
|
||||||
|
import type { RiotQueueGroup } from '../riot/riot-queue.constants';
|
||||||
|
import type { PollTournamentResultJob } from '../tournament-result/tournament-result.types';
|
||||||
|
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
|
||||||
|
|
||||||
|
export const RIOT_JOB_HIGH_STREAM = 'riot:queue:high';
|
||||||
|
export const RIOT_JOB_LOW_STREAM = 'riot:queue:low';
|
||||||
|
export const RIOT_JOB_SCHEDULED_ZSET = 'riot:queue:scheduled';
|
||||||
|
export const RIOT_JOB_GROUP = 'riot-refresh-worker';
|
||||||
|
export const RIOT_JOB_STATUS_TTL_SECONDS = 24 * 60 * 60;
|
||||||
|
export const RIOT_JOB_DEDUP_TTL_SECONDS = 5 * 60;
|
||||||
|
export const RIOT_JOB_HIGH_COOLDOWN_SECONDS = 5 * 60;
|
||||||
|
export const RIOT_JOB_LOW_COOLDOWN_SECONDS = 60 * 60;
|
||||||
|
export const RIOT_JOB_AVG_SECONDS = 20;
|
||||||
|
|
||||||
|
export type RiotJobPriority = 'high' | 'low';
|
||||||
|
export type RiotJobState = 'queued' | 'processing' | 'completed' | 'failed';
|
||||||
|
|
||||||
|
export interface RiotRefreshSubJob {
|
||||||
|
kind?: 'refresh-account';
|
||||||
|
jobId: string;
|
||||||
|
parentJobId: string;
|
||||||
|
userId: string;
|
||||||
|
puuid: string;
|
||||||
|
region: RiotRegionalRoute;
|
||||||
|
platform: RiotPlatformRoute;
|
||||||
|
queueGroup: RiotQueueGroup;
|
||||||
|
queueIds: number[];
|
||||||
|
requestedAt: number;
|
||||||
|
requestedBy?: string;
|
||||||
|
priority: RiotJobPriority;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RiotJobPayload =
|
||||||
|
| RiotRefreshSubJob
|
||||||
|
| PollTournamentResultJob
|
||||||
|
| LegacyImportMemberJob;
|
||||||
|
|
||||||
|
export interface RiotJobStatus {
|
||||||
|
jobId: string;
|
||||||
|
parentJobId: string;
|
||||||
|
state: RiotJobState;
|
||||||
|
priority: RiotJobPriority;
|
||||||
|
queueGroup: RiotQueueGroup;
|
||||||
|
queueIds: number[];
|
||||||
|
puuid: string;
|
||||||
|
userId: string;
|
||||||
|
enqueuedAt: string;
|
||||||
|
startedAt?: string;
|
||||||
|
finishedAt?: string;
|
||||||
|
error?: string;
|
||||||
|
etaSeconds: number;
|
||||||
|
queuePosition: number;
|
||||||
|
alreadyQueued?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RiotParentJobStatus {
|
||||||
|
parentJobId: string;
|
||||||
|
state: RiotJobState;
|
||||||
|
done: number;
|
||||||
|
total: number;
|
||||||
|
etaSeconds: number;
|
||||||
|
subJobs: RiotJobStatus[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RiotRefreshEvent {
|
||||||
|
parentJobId: string;
|
||||||
|
jobId: string;
|
||||||
|
queueGroup: RiotQueueGroup;
|
||||||
|
puuid: string;
|
||||||
|
state: RiotJobState;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EnqueueRefreshResult {
|
||||||
|
parentJobId: string;
|
||||||
|
subJobs: Array<{
|
||||||
|
jobId: string;
|
||||||
|
queueGroup: RiotQueueGroup;
|
||||||
|
alreadyQueued: boolean;
|
||||||
|
etaSeconds: number;
|
||||||
|
queuePosition: number;
|
||||||
|
}>;
|
||||||
|
totalEtaSeconds: number;
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
import {
|
||||||
|
Injectable,
|
||||||
|
Logger,
|
||||||
|
OnModuleDestroy,
|
||||||
|
OnModuleInit,
|
||||||
|
} from '@nestjs/common';
|
||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import { GarnetService, StreamEntry } from '../global/garnet/garnet.service';
|
||||||
|
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||||
|
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
|
||||||
|
import { RiotRecentMatchesService } from '../riot/riot-recent-matches.service';
|
||||||
|
import {
|
||||||
|
RIOT_JOB_GROUP,
|
||||||
|
RIOT_JOB_HIGH_STREAM,
|
||||||
|
RIOT_JOB_LOW_STREAM,
|
||||||
|
RiotJobPayload,
|
||||||
|
RiotRefreshSubJob,
|
||||||
|
} from './riot-job-queue.types';
|
||||||
|
import { RiotJobQueueService } from './riot-job-queue.service';
|
||||||
|
import { RiotRefreshGateway } from './riot-refresh.gateway';
|
||||||
|
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||||
|
import type { PollTournamentResultJob } from '../tournament-result/tournament-result.types';
|
||||||
|
|
||||||
|
const LEADER_KEY = 'riot:worker:leader';
|
||||||
|
const LEADER_TTL_SECONDS = 60;
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||||
|
private readonly logger = new Logger(RiotJobWorkerService.name);
|
||||||
|
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||||
|
private stopped = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly garnetService: GarnetService,
|
||||||
|
private readonly jobQueueService: RiotJobQueueService,
|
||||||
|
private readonly recentMatchesService: RiotRecentMatchesService,
|
||||||
|
private readonly connectionService: ConnectionService,
|
||||||
|
private readonly refreshGateway: RiotRefreshGateway,
|
||||||
|
private readonly tournamentResultService: TournamentResultService,
|
||||||
|
private readonly legacyImportService: LegacyImportService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public async onModuleInit() {
|
||||||
|
await this.ensureConsumerGroups();
|
||||||
|
void this.loop();
|
||||||
|
}
|
||||||
|
|
||||||
|
public onModuleDestroy() {
|
||||||
|
this.stopped = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async loop(): Promise<void> {
|
||||||
|
while (!this.stopped) {
|
||||||
|
try {
|
||||||
|
const isLeader = await this.tryBecomeLeader();
|
||||||
|
if (!isLeader) {
|
||||||
|
await this.sleep(1_000);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.jobQueueService.promoteDueScheduledJobs();
|
||||||
|
const entry =
|
||||||
|
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
|
||||||
|
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
|
||||||
|
if (!entry) continue;
|
||||||
|
|
||||||
|
await this.processEntry(entry.stream, entry.message);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Riot job worker loop failed: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
await this.sleep(1_000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async processEntry(stream: string, entry: StreamEntry): Promise<void> {
|
||||||
|
const job = this.parseJob(entry);
|
||||||
|
if (!job) {
|
||||||
|
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, entry.id);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (this.isTournamentResultJob(job)) {
|
||||||
|
await this.tournamentResultService.processPollJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.isLegacyImportJob(job)) {
|
||||||
|
await this.legacyImportService.processMemberJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.jobQueueService.markProcessing(job);
|
||||||
|
for (const queueId of job.queueIds) {
|
||||||
|
await this.recentMatchesService.ensureFreshForQueue({
|
||||||
|
puuid: job.puuid,
|
||||||
|
region: job.region,
|
||||||
|
queueId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
await this.markUserFetchedAt(job);
|
||||||
|
const event = await this.jobQueueService.markCompleted(job);
|
||||||
|
this.refreshGateway.broadcastRefreshEvent(event);
|
||||||
|
} catch (error) {
|
||||||
|
if (this.isTournamentResultJob(job)) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Tournament result poll job failed ${job.jobId}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.isLegacyImportJob(job)) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Legacy import member job failed ${job.jobId}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const event = await this.jobQueueService.markFailed(job, error);
|
||||||
|
this.refreshGateway.broadcastRefreshEvent(event);
|
||||||
|
this.logger.warn(
|
||||||
|
`Riot refresh job failed ${job.jobId}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await this.garnetService.acknowledgeStream(stream, RIOT_JOB_GROUP, entry.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async markUserFetchedAt(job: RiotRefreshSubJob): Promise<void> {
|
||||||
|
const UserModel = await this.connectionService.getModel('user');
|
||||||
|
const now = new Date();
|
||||||
|
const $set: Record<string, Date> = {
|
||||||
|
'riotAccounts.$.matchIdsFetchedAt': now,
|
||||||
|
};
|
||||||
|
for (const queueId of job.queueIds) {
|
||||||
|
$set[`riotAccounts.$.matchIdsByQueueFetchedAt.${queueId}`] = now;
|
||||||
|
}
|
||||||
|
await UserModel.updateOne(
|
||||||
|
{
|
||||||
|
_id: new mongoose.Types.ObjectId(job.userId),
|
||||||
|
'riotAccounts.puuid': job.puuid,
|
||||||
|
},
|
||||||
|
{ $set },
|
||||||
|
).exec();
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseJob(entry: StreamEntry): RiotJobPayload | null {
|
||||||
|
const raw = entry.message.job;
|
||||||
|
if (!raw) return null;
|
||||||
|
try {
|
||||||
|
return JSON.parse(raw) as RiotJobPayload;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private isTournamentResultJob(
|
||||||
|
job: RiotJobPayload,
|
||||||
|
): job is PollTournamentResultJob {
|
||||||
|
return job.kind === 'poll-tournament-result';
|
||||||
|
}
|
||||||
|
|
||||||
|
private isLegacyImportJob(
|
||||||
|
job: RiotJobPayload,
|
||||||
|
): job is LegacyImportMemberJob {
|
||||||
|
return job.kind === 'legacy-import-member';
|
||||||
|
}
|
||||||
|
|
||||||
|
private async readOne(
|
||||||
|
stream: string,
|
||||||
|
blockMs: number,
|
||||||
|
): Promise<{ stream: string; message: StreamEntry } | null> {
|
||||||
|
const result = await this.garnetService.readGroup(
|
||||||
|
stream,
|
||||||
|
RIOT_JOB_GROUP,
|
||||||
|
this.consumer,
|
||||||
|
'>',
|
||||||
|
{ count: 1, blockMs },
|
||||||
|
);
|
||||||
|
const message = result?.[0]?.messages[0];
|
||||||
|
return message ? { stream, message } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureConsumerGroups(): Promise<void> {
|
||||||
|
await Promise.all(
|
||||||
|
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
|
||||||
|
try {
|
||||||
|
await this.garnetService.createConsumerGroup(stream, RIOT_JOB_GROUP);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (!message.includes('BUSYGROUP')) throw error;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async tryBecomeLeader(): Promise<boolean> {
|
||||||
|
const current = await this.garnetService.getString(LEADER_KEY);
|
||||||
|
if (current === this.consumer) {
|
||||||
|
await this.garnetService.setString(
|
||||||
|
LEADER_KEY,
|
||||||
|
this.consumer,
|
||||||
|
LEADER_TTL_SECONDS,
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (current) return false;
|
||||||
|
return this.garnetService.setStringIfAbsent(
|
||||||
|
LEADER_KEY,
|
||||||
|
this.consumer,
|
||||||
|
LEADER_TTL_SECONDS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
import { Logger } from '@nestjs/common';
|
||||||
|
import {
|
||||||
|
ConnectedSocket,
|
||||||
|
MessageBody,
|
||||||
|
SubscribeMessage,
|
||||||
|
WebSocketGateway,
|
||||||
|
WebSocketServer,
|
||||||
|
} from '@nestjs/websockets';
|
||||||
|
import type { Namespace, Socket } from 'socket.io';
|
||||||
|
import type { RiotRefreshEvent } from './riot-job-queue.types';
|
||||||
|
|
||||||
|
interface SubscribePayload {
|
||||||
|
puuid?: string;
|
||||||
|
parentJobId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const MAX_ROOMS_PER_SOCKET = 5;
|
||||||
|
|
||||||
|
@WebSocketGateway({
|
||||||
|
namespace: '/riot-refresh',
|
||||||
|
cors: {
|
||||||
|
origin: true,
|
||||||
|
credentials: true,
|
||||||
|
allowedHeaders: ['Authorization', 'Content-Type', 'X-Auth-User-Id'],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
export class RiotRefreshGateway {
|
||||||
|
@WebSocketServer()
|
||||||
|
private server!: Namespace;
|
||||||
|
|
||||||
|
private readonly logger = new Logger(RiotRefreshGateway.name);
|
||||||
|
|
||||||
|
@SubscribeMessage('subscribe')
|
||||||
|
public async handleSubscribe(
|
||||||
|
@ConnectedSocket() socket: Socket,
|
||||||
|
@MessageBody() payload: SubscribePayload,
|
||||||
|
) {
|
||||||
|
const rooms = this.resolveRooms(payload);
|
||||||
|
if (rooms.length === 0) {
|
||||||
|
socket.emit('error', { message: 'Invalid riot refresh subscription.' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentRooms = [...socket.rooms].filter((room) => room !== socket.id);
|
||||||
|
if (currentRooms.length + rooms.length > MAX_ROOMS_PER_SOCKET) {
|
||||||
|
socket.emit('error', { message: 'Too many riot refresh subscriptions.' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(rooms.map((room) => socket.join(room)));
|
||||||
|
socket.emit('subscribed', { rooms });
|
||||||
|
}
|
||||||
|
|
||||||
|
public broadcastRefreshEvent(event: RiotRefreshEvent): void {
|
||||||
|
const rooms = [
|
||||||
|
`puuid:${event.puuid}`,
|
||||||
|
`parent:${event.parentJobId}`,
|
||||||
|
`job:${event.jobId}`,
|
||||||
|
];
|
||||||
|
this.logger.debug(
|
||||||
|
`Broadcast riot refresh ${event.state}: ${event.jobId} (${event.queueGroup})`,
|
||||||
|
);
|
||||||
|
for (const room of rooms) {
|
||||||
|
this.server.to(room).emit('riot-refresh', event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveRooms(payload: SubscribePayload): string[] {
|
||||||
|
const rooms: string[] = [];
|
||||||
|
if (typeof payload?.puuid === 'string' && payload.puuid.trim()) {
|
||||||
|
rooms.push(`puuid:${payload.puuid.trim()}`);
|
||||||
|
}
|
||||||
|
if (typeof payload?.parentJobId === 'string' && payload.parentJobId.trim()) {
|
||||||
|
rooms.push(`parent:${payload.parentJobId.trim()}`);
|
||||||
|
}
|
||||||
|
return rooms;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user