Files
persona-lenz/apps/api/src/modules/analysis/analysis-pipeline.service.ts
T
2026-07-03 00:45:04 -07:00

904 lines
25 KiB
TypeScript

import { Injectable, Logger } from '@nestjs/common';
import { FEATURE_SCHEMA_VERSION } from '@persona-lenz/shared';
import { ConnectionService } from '../global/database/connection.service';
import type {
Account,
AnalysisJobStatus,
PlayerMatchFeature,
RiotMatch,
TierInfo,
} from '../global/database/schemas';
import {
RiotApiClient,
RiotNotFoundError,
type RiotLeagueEntryDto,
} from '../riot/riot-api.client';
import {
FEATURE_RECOMPUTE_JOB_NAME,
MAX_ANALYSIS_MATCH_COUNT,
normalizeFeatureRecomputeBatchSize,
normalizeRequestedMatchCount,
USER_ANALYSIS_JOB_NAME,
} from './analysis.constants';
import type {
ExtractedPlayerMatchFeature,
FeatureRecomputeJobData,
UserAnalysisJobData,
} from './analysis.types';
import { FeatureExtractorService } from './feature-extractor.service';
import { ProfileBuilderService } from './profile-builder.service';
import { TimelineDerivedService } from './timeline-derived.service';
import { TimelineMetricService } from './timeline-metric.service';
interface MatchRecord {
matchId: string;
raw: Record<string, unknown>;
timeline?: Record<string, unknown>;
}
interface ParsedMatchSummary {
gameCreation: number;
gameDuration: number;
gameVersion?: string;
queueId?: number;
participantPuuids: string[];
}
const TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_FIELD =
'timelineUnavailableForFeatureSchemaVersion';
const TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_PATH = `derived.${TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_FIELD}`;
function asRecord(value: unknown): Record<string, unknown> {
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
return value as Record<string, unknown>;
}
return {};
}
function isRecordValue(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function isTimelinePayload(value: unknown): value is Record<string, unknown> {
if (!isRecordValue(value)) {
return false;
}
const info = asRecord(value.info);
return Array.isArray(info.frames) && info.frames.length > 0;
}
function asNumber(value: unknown, fallback = 0): number {
return typeof value === 'number' && Number.isFinite(value) ? value : fallback;
}
function asString(value: unknown): string | undefined {
return typeof value === 'string' ? value : undefined;
}
function asStringArray(value: unknown): string[] {
if (!Array.isArray(value)) return [];
return value.filter((item): item is string => typeof item === 'string');
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
@Injectable()
export class AnalysisPipelineService {
private readonly logger = new Logger(AnalysisPipelineService.name);
constructor(
private readonly connectionService: ConnectionService,
private readonly riotApiClient: RiotApiClient,
private readonly featureExtractorService: FeatureExtractorService,
private readonly profileBuilderService: ProfileBuilderService,
private readonly timelineDerivedService: TimelineDerivedService,
private readonly timelineMetricService: TimelineMetricService,
) {}
public async processUserAnalysisJob(
data: UserAnalysisJobData,
jobId: string,
options: { isFinalAttempt?: boolean } = {},
): Promise<void> {
const normalizedData = {
...data,
requestedMatchCount: normalizeRequestedMatchCount(
data.requestedMatchCount,
),
};
try {
await this.updateJob(jobId, {
status: 'processing',
stage: 'resolve-account',
progress: {
fetchedMatches: 0,
totalMatches: 0,
},
});
const account = await this.resolveAndStoreAccount(normalizedData);
await this.updateJob(jobId, {
puuid: account.puuid,
stage: 'fetch-match-ids',
});
const matchIds = await this.riotApiClient.getMatchIds(account.puuid, {
count: normalizedData.requestedMatchCount,
queue: normalizedData.queueId,
regionalRoute: normalizedData.regionalRoute,
});
await this.updateJob(jobId, {
stage: 'fetch-match-details',
progress: {
fetchedMatches: 0,
totalMatches: matchIds.length,
},
});
if (matchIds.length === 0) {
await this.completeJob(jobId);
return;
}
const matchRecords = await this.fetchAndStoreMatches(
matchIds,
normalizedData,
jobId,
);
await this.updateJob(jobId, {
stage: 'extract-features',
});
const extractedFeatures = await this.extractAndStoreFeatures(
matchRecords,
account.puuid,
normalizedData.queueId,
);
await this.updateJob(jobId, {
stage: 'build-profile',
});
if (extractedFeatures.length > 0) {
await this.buildAndStoreProfile(
account,
normalizedData,
extractedFeatures,
);
}
await this.completeJob(jobId);
} catch (error) {
if (options.isFinalAttempt === true) {
const publicError = this.publicErrorMessage(error);
this.logFinalFailure(normalizedData, jobId, publicError, error);
await this.updateJob(jobId, {
status: 'failed',
error: publicError,
});
}
throw error;
}
}
public async processFeatureRecomputeJob(
data: FeatureRecomputeJobData,
jobId: string,
options: { isFinalAttempt?: boolean } = {},
): Promise<void> {
const normalizedData = {
...data,
batchSize: normalizeFeatureRecomputeBatchSize(data.batchSize),
};
try {
await this.updateJob(
jobId,
{
status: 'processing',
stage: 'scan-features',
progress: {
fetchedMatches: 0,
totalMatches: 0,
},
},
FEATURE_RECOMPUTE_JOB_NAME,
);
const targetFeatures =
await this.findFeatureRecomputeTargets(normalizedData);
await this.updateJob(
jobId,
{
stage: 'recompute-features',
progress: {
fetchedMatches: 0,
totalMatches: targetFeatures.length,
},
},
FEATURE_RECOMPUTE_JOB_NAME,
);
const affectedPuuids = new Set<string>();
for (const [index, targetFeature] of targetFeatures.entries()) {
const recomputedFeature =
await this.recomputeStoredFeature(targetFeature);
if (recomputedFeature) {
affectedPuuids.add(recomputedFeature.puuid);
}
await this.updateJob(
jobId,
{
progress: {
fetchedMatches: index + 1,
totalMatches: targetFeatures.length,
},
},
FEATURE_RECOMPUTE_JOB_NAME,
);
}
await this.updateJob(
jobId,
{
stage: 'build-profile',
},
FEATURE_RECOMPUTE_JOB_NAME,
);
for (const puuid of affectedPuuids) {
await this.rebuildStoredProfile(puuid, normalizedData.queueId);
}
await this.completeJob(jobId, FEATURE_RECOMPUTE_JOB_NAME);
} catch (error) {
if (options.isFinalAttempt === true) {
const publicError = this.publicErrorMessage(error);
this.logFeatureRecomputeFinalFailure(
normalizedData,
jobId,
publicError,
error,
);
await this.updateJob(
jobId,
{
status: 'failed',
error: publicError,
},
FEATURE_RECOMPUTE_JOB_NAME,
);
}
throw error;
}
}
private async resolveAndStoreAccount(
data: UserAnalysisJobData,
): Promise<Account> {
const account = await this.riotApiClient.getAccountByRiotId(
data.gameName,
data.tagLine,
data.regionalRoute,
);
const [summoner, tiers] = await Promise.all([
this.riotApiClient.getSummonerByPuuid(account.puuid, data.platformRoute),
this.riotApiClient.getLeagueEntriesByPuuid(
account.puuid,
data.platformRoute,
),
]);
const accountModel = await this.connectionService.getModel('account');
const tierFetchedAt = new Date();
const storedAccount = await accountModel
.findOneAndUpdate(
{ puuid: account.puuid },
{
$set: {
puuid: account.puuid,
gameName: account.gameName,
tagLine: account.tagLine,
platformRoute: data.platformRoute,
regionalRoute: data.regionalRoute,
profileIconId: summoner.profileIconId,
summonerLevel: summoner.summonerLevel,
tiers: this.toTierInfo(tiers, tierFetchedAt),
tiersFetchedAt: tierFetchedAt,
},
$setOnInsert: {
isSeed: false,
},
},
{
new: true,
setDefaultsOnInsert: true,
upsert: true,
},
)
.lean<Account>()
.exec();
if (!storedAccount) {
throw new Error('Failed to store Riot account.');
}
return storedAccount;
}
private async fetchAndStoreMatches(
matchIds: string[],
data: UserAnalysisJobData,
jobId: string,
): Promise<MatchRecord[]> {
const matchRecords: MatchRecord[] = [];
const riotMatchModel = await this.connectionService.getModel('riotMatch');
for (const [index, matchId] of matchIds.entries()) {
try {
const existingMatch = await riotMatchModel
.findOne({ matchId })
.lean<RiotMatch>()
.exec();
if (existingMatch) {
if (this.isEligibleMatch(existingMatch.raw, data.queueId)) {
const timeline = await this.fetchAndStoreMissingTimeline(
matchId,
existingMatch.timeline,
data.regionalRoute,
);
matchRecords.push({
matchId,
raw: existingMatch.raw,
...(timeline ? { timeline } : {}),
});
}
} else {
const rawMatch = await this.riotApiClient.getMatch(
matchId,
data.regionalRoute,
);
await this.storeRiotMatch(matchId, rawMatch);
if (this.isEligibleMatch(rawMatch, data.queueId)) {
const timeline = await this.fetchAndStoreMissingTimeline(
matchId,
undefined,
data.regionalRoute,
);
matchRecords.push({
matchId,
raw: rawMatch,
...(timeline ? { timeline } : {}),
});
}
}
} catch (error) {
this.logger.warn(
`Skipping matchId="${this.safeLogText(matchId)}" after fetch/store failure: ${this.safeLogText(errorMessage(error))}`,
);
} finally {
await this.updateJob(jobId, {
progress: {
fetchedMatches: index + 1,
totalMatches: matchIds.length,
},
});
}
}
return matchRecords;
}
private async storeRiotMatch(
matchId: string,
rawMatch: Record<string, unknown>,
): Promise<void> {
const summary = this.parseMatchSummary(rawMatch);
const riotMatchModel = await this.connectionService.getModel('riotMatch');
await riotMatchModel
.updateOne(
{ matchId },
{
$setOnInsert: {
matchId,
gameCreation: summary.gameCreation,
gameDuration: summary.gameDuration,
gameVersion: summary.gameVersion,
queueId: summary.queueId,
participantPuuids: summary.participantPuuids,
raw: rawMatch,
fetchedAt: new Date(),
},
},
{ upsert: true },
)
.exec();
}
private async fetchAndStoreMissingTimeline(
matchId: string,
existingTimeline: unknown,
regionalRoute: UserAnalysisJobData['regionalRoute'],
): Promise<Record<string, unknown> | undefined> {
if (isTimelinePayload(existingTimeline)) {
return existingTimeline;
}
try {
const timeline = await this.riotApiClient.getMatchTimeline(
matchId,
regionalRoute,
);
if (!isTimelinePayload(timeline)) {
this.logger.warn(
`Skipping malformed timeline for matchId="${this.safeLogText(matchId)}"`,
);
return undefined;
}
await this.storeRiotMatchTimeline(matchId, timeline);
return timeline;
} catch (error) {
this.logger.warn(
`Skipping timeline fetch for matchId="${this.safeLogText(matchId)}": ${this.safeLogText(errorMessage(error))}`,
);
return undefined;
}
}
private async storeRiotMatchTimeline(
matchId: string,
timeline: Record<string, unknown>,
): Promise<void> {
const riotMatchModel = await this.connectionService.getModel('riotMatch');
await riotMatchModel
.updateOne(
{ matchId },
{
$set: {
timeline,
timelineFetchedAt: new Date(),
},
},
)
.exec();
}
private async extractAndStoreFeatures(
matchRecords: MatchRecord[],
puuid: string,
queueId: number,
): Promise<ExtractedPlayerMatchFeature[]> {
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
const extractedFeatures: ExtractedPlayerMatchFeature[] = [];
for (const matchRecord of matchRecords) {
try {
const feature = this.featureExtractorService.extractPlayerMatchFeature(
matchRecord.raw,
puuid,
);
if (feature.queueId !== queueId) continue;
if (matchRecord.timeline) {
const derived = this.timelineDerivedService.buildDerivedData(
matchRecord.timeline,
matchRecord.raw,
);
const timelineMetrics =
this.timelineMetricService.extractTimelineMetrics(
feature,
matchRecord.raw,
matchRecord.timeline,
derived,
);
feature.derived = {
...derived,
};
feature.features = {
...feature.features,
...timelineMetrics.features,
};
feature.normalized = {
...feature.normalized,
...timelineMetrics.normalized,
};
feature.vector = timelineMetrics.vector;
feature.timelineProcessed = true;
}
await playerMatchFeatureModel
.updateOne(
{ matchId: feature.matchId, puuid: feature.puuid },
{ $set: feature },
{ upsert: true },
)
.exec();
extractedFeatures.push(feature);
} catch (error) {
this.logger.warn(
`Skipping feature extraction for matchId="${this.safeLogText(matchRecord.matchId)}": ${this.safeLogText(errorMessage(error))}`,
);
}
}
return extractedFeatures;
}
private async findFeatureRecomputeTargets(
data: FeatureRecomputeJobData,
): Promise<PlayerMatchFeature[]> {
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
return playerMatchFeatureModel
.find(
this.featureRecomputeTargetQuery({
queueId: data.queueId,
puuid: data.puuid,
}),
)
.sort({ gameCreation: -1 })
.limit(data.batchSize)
.lean<PlayerMatchFeature[]>()
.exec();
}
private async recomputeStoredFeature(
targetFeature: PlayerMatchFeature,
): Promise<ExtractedPlayerMatchFeature | undefined> {
const riotMatchModel = await this.connectionService.getModel('riotMatch');
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
const storedMatch = await riotMatchModel
.findOne({ matchId: targetFeature.matchId })
.lean<RiotMatch>()
.exec();
if (!storedMatch) {
this.logger.warn(
`Skipping feature recompute for matchId="${this.safeLogText(targetFeature.matchId)}": stored match was not found`,
);
return undefined;
}
try {
const feature = this.featureExtractorService.extractPlayerMatchFeature(
storedMatch.raw,
targetFeature.puuid,
);
if (feature.queueId !== targetFeature.queueId) {
return undefined;
}
if (isTimelinePayload(storedMatch.timeline)) {
const derived = this.timelineDerivedService.buildDerivedData(
storedMatch.timeline,
storedMatch.raw,
);
const timelineMetrics =
this.timelineMetricService.extractTimelineMetrics(
feature,
storedMatch.raw,
storedMatch.timeline,
derived,
);
feature.derived = {
...derived,
};
feature.features = {
...feature.features,
...timelineMetrics.features,
};
feature.normalized = {
...feature.normalized,
...timelineMetrics.normalized,
};
feature.vector = timelineMetrics.vector;
feature.timelineProcessed = true;
} else {
feature.derived = {
[TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_FIELD]:
FEATURE_SCHEMA_VERSION,
};
}
await playerMatchFeatureModel
.updateOne(
{ matchId: feature.matchId, puuid: feature.puuid },
{ $set: feature },
{ upsert: true },
)
.exec();
return feature;
} catch (error) {
this.logger.warn(
`Skipping feature recompute for matchId="${this.safeLogText(targetFeature.matchId)}": ${this.safeLogText(errorMessage(error))}`,
);
return undefined;
}
}
private async rebuildStoredProfile(
puuid: string,
queueId: number,
): Promise<void> {
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
const hasRemainingTargets =
await this.hasRemainingFeatureRecomputeTargetsForPlayer(puuid, queueId);
if (hasRemainingTargets) {
return;
}
const accountModel = await this.connectionService.getModel('account');
const account = await accountModel
.findOne({ puuid })
.lean<Pick<Account, 'isSeed' | 'puuid'> | null>()
.exec();
const features = await playerMatchFeatureModel
.find({
puuid,
queueId,
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
})
.sort({ gameCreation: -1 })
.limit(MAX_ANALYSIS_MATCH_COUNT)
.lean<PlayerMatchFeature[]>()
.exec();
if (features.length === 0) {
return;
}
await this.storeBuiltProfile({
features,
isSeed: account?.isSeed ?? false,
puuid,
queueId,
});
}
private async buildAndStoreProfile(
account: Account,
data: UserAnalysisJobData,
extractedFeatures: ExtractedPlayerMatchFeature[],
): Promise<void> {
await this.storeBuiltProfile({
features: extractedFeatures,
isSeed: account.isSeed,
puuid: account.puuid,
queueId: data.queueId,
});
const accountModel = await this.connectionService.getModel('account');
const analyzedAt = new Date();
await accountModel
.updateOne(
{ puuid: account.puuid },
{
$set: {
lastAnalyzedAt: analyzedAt,
},
},
)
.exec();
}
private async hasRemainingFeatureRecomputeTargetsForPlayer(
puuid: string,
queueId: number,
): Promise<boolean> {
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
const remainingTargets = await playerMatchFeatureModel
.find(this.featureRecomputeTargetQuery({ puuid, queueId }))
.sort({ gameCreation: -1 })
.limit(1)
.lean<PlayerMatchFeature[]>()
.exec();
return remainingTargets.length > 0;
}
private featureRecomputeTargetQuery(params: {
puuid?: string;
queueId: number;
}): Record<string, unknown> {
return {
...(params.puuid ? { puuid: params.puuid } : {}),
queueId: params.queueId,
$or: [
{ featureSchemaVersion: { $lt: FEATURE_SCHEMA_VERSION } },
{
timelineProcessed: false,
[TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_PATH]: {
$ne: FEATURE_SCHEMA_VERSION,
},
},
],
};
}
private async storeBuiltProfile(params: {
features: PlayerMatchFeature[] | ExtractedPlayerMatchFeature[];
isSeed: boolean;
puuid: string;
queueId: number;
}): Promise<void> {
const profile = this.profileBuilderService.buildProfile({
features: params.features,
isSeed: params.isSeed,
puuid: params.puuid,
queueId: params.queueId,
});
const playerProfileModel =
await this.connectionService.getModel('playerProfile');
await playerProfileModel
.updateOne(
{
puuid: params.puuid,
queueId: params.queueId,
},
{ $set: profile },
{ upsert: true },
)
.exec();
}
private async completeJob(
jobId: string,
jobType = USER_ANALYSIS_JOB_NAME,
): Promise<void> {
await this.updateJob(
jobId,
{
status: 'completed',
stage: 'completed',
completedAt: new Date(),
error: undefined,
},
jobType,
);
}
private async updateJob(
jobId: string,
patch: {
completedAt?: Date;
error?: string;
progress?: { fetchedMatches: number; totalMatches: number };
puuid?: string;
stage?: string;
status?: AnalysisJobStatus;
},
jobType = USER_ANALYSIS_JOB_NAME,
): Promise<void> {
const analysisJobModel =
await this.connectionService.getModel('analysisJob');
const set: Record<string, unknown> = {
jobId,
jobType,
...patch,
};
const unset: Record<string, string> = {};
if (patch.error === undefined) {
delete set.error;
}
if (patch.status === 'completed') {
unset.error = '';
}
await analysisJobModel
.updateOne(
{ jobId },
{
$set: set,
...(Object.keys(unset).length > 0 ? { $unset: unset } : {}),
},
)
.exec();
}
private isEligibleMatch(
rawMatch: Record<string, unknown>,
queueId: number,
): boolean {
const info = asRecord(rawMatch.info);
return (
asNumber(info.queueId) === queueId && asNumber(info.gameDuration) >= 300
);
}
private parseMatchSummary(
rawMatch: Record<string, unknown>,
): ParsedMatchSummary {
const metadata = asRecord(rawMatch.metadata);
const info = asRecord(rawMatch.info);
return {
gameCreation: asNumber(info.gameCreation),
gameDuration: asNumber(info.gameDuration),
gameVersion: asString(info.gameVersion),
queueId:
typeof info.queueId === 'number' && Number.isFinite(info.queueId)
? info.queueId
: undefined,
participantPuuids: asStringArray(metadata.participants),
};
}
private toTierInfo(
entries: RiotLeagueEntryDto[],
fetchedAt: Date,
): TierInfo[] {
return entries.map((entry) => ({
queueType: entry.queueType,
tier: entry.tier,
rank: entry.rank,
leaguePoints: entry.leaguePoints,
wins: entry.wins,
losses: entry.losses,
fetchedAt,
}));
}
private publicErrorMessage(error: unknown): string {
if (
error instanceof Error &&
(error.name === 'RiotApiKeyError' ||
error.message.includes('Riot API key'))
) {
return 'Riot API key is invalid or expired.';
}
if (error instanceof RiotNotFoundError) {
return 'Riot account was not found for the selected region.';
}
return 'Analysis failed during processing.';
}
private logFinalFailure(
data: UserAnalysisJobData,
jobId: string,
publicError: string,
error: unknown,
): void {
this.logger.error(
[
`Analysis job failed jobId="${this.safeLogText(jobId)}"`,
`gameName="${this.safeLogText(data.gameName)}"`,
`tagLine="${this.safeLogText(data.tagLine)}"`,
`platformRoute="${data.platformRoute}"`,
`publicError="${this.safeLogText(publicError)}"`,
`cause="${this.safeLogText(errorMessage(error))}"`,
].join(' '),
error instanceof Error ? error.stack : undefined,
);
}
private logFeatureRecomputeFinalFailure(
data: FeatureRecomputeJobData,
jobId: string,
publicError: string,
error: unknown,
): void {
this.logger.error(
[
`Feature recompute job failed jobId="${this.safeLogText(jobId)}"`,
`puuid="${this.safeLogText(data.puuid ?? '')}"`,
`platformRoute="${data.platformRoute}"`,
`queueId="${data.queueId}"`,
`publicError="${this.safeLogText(publicError)}"`,
`cause="${this.safeLogText(errorMessage(error))}"`,
].join(' '),
error instanceof Error ? error.stack : undefined,
);
}
private safeLogText(value: string): string {
return value.replace(/[\r\n\t]/gu, ' ').slice(0, 300);
}
}