Add feature recompute jobs

This commit is contained in:
dahyeon committed 2026-07-03 00:45:04 -07:00
1 parent 1a7a1cca68
commit 8668234065
8 files changed
+1176 -36

No files matched your search

@@ -1,8 +1,10 @@
import { Injectable, Logger } from '@nestjs/common';
import { FEATURE_SCHEMA_VERSION } from '@persona-lenz/shared';
import { ConnectionService } from '../global/database/connection.service';
import type {
Account,
AnalysisJobStatus,
PlayerMatchFeature,
RiotMatch,
TierInfo,
} from '../global/database/schemas';
@@ -12,11 +14,15 @@ import {
type RiotLeagueEntryDto,
} from '../riot/riot-api.client';
import {
FEATURE_RECOMPUTE_JOB_NAME,
MAX_ANALYSIS_MATCH_COUNT,
normalizeFeatureRecomputeBatchSize,
normalizeRequestedMatchCount,
USER_ANALYSIS_JOB_NAME,
} from './analysis.constants';
import type {
ExtractedPlayerMatchFeature,
FeatureRecomputeJobData,
UserAnalysisJobData,
} from './analysis.types';
import { FeatureExtractorService } from './feature-extractor.service';
@@ -38,6 +44,10 @@ interface ParsedMatchSummary {
participantPuuids: string[];
}
const TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_FIELD =
'timelineUnavailableForFeatureSchemaVersion';
const TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_PATH = `derived.${TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_FIELD}`;
function asRecord(value: unknown): Record<string, unknown> {
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
return value as Record<string, unknown>;
@@ -173,6 +183,97 @@ export class AnalysisPipelineService {
}
}
public async processFeatureRecomputeJob(
data: FeatureRecomputeJobData,
jobId: string,
options: { isFinalAttempt?: boolean } = {},
): Promise<void> {
const normalizedData = {
...data,
batchSize: normalizeFeatureRecomputeBatchSize(data.batchSize),
};
try {
await this.updateJob(
jobId,
{
status: 'processing',
stage: 'scan-features',
progress: {
fetchedMatches: 0,
totalMatches: 0,
},
},
FEATURE_RECOMPUTE_JOB_NAME,
);
const targetFeatures =
await this.findFeatureRecomputeTargets(normalizedData);
await this.updateJob(
jobId,
{
stage: 'recompute-features',
progress: {
fetchedMatches: 0,
totalMatches: targetFeatures.length,
},
},
FEATURE_RECOMPUTE_JOB_NAME,
);
const affectedPuuids = new Set<string>();
for (const [index, targetFeature] of targetFeatures.entries()) {
const recomputedFeature =
await this.recomputeStoredFeature(targetFeature);
if (recomputedFeature) {
affectedPuuids.add(recomputedFeature.puuid);
}
await this.updateJob(
jobId,
{
progress: {
fetchedMatches: index + 1,
totalMatches: targetFeatures.length,
},
},
FEATURE_RECOMPUTE_JOB_NAME,
);
}
await this.updateJob(
jobId,
{
stage: 'build-profile',
},
FEATURE_RECOMPUTE_JOB_NAME,
);
for (const puuid of affectedPuuids) {
await this.rebuildStoredProfile(puuid, normalizedData.queueId);
}
await this.completeJob(jobId, FEATURE_RECOMPUTE_JOB_NAME);
} catch (error) {
if (options.isFinalAttempt === true) {
const publicError = this.publicErrorMessage(error);
this.logFeatureRecomputeFinalFailure(
normalizedData,
jobId,
publicError,
error,
);
await this.updateJob(
jobId,
{
status: 'failed',
error: publicError,
},
FEATURE_RECOMPUTE_JOB_NAME,
);
}
throw error;
}
}
private async resolveAndStoreAccount(
data: UserAnalysisJobData,
): Promise<Account> {
@@ -426,32 +527,154 @@ export class AnalysisPipelineService {
return extractedFeatures;
}
private async findFeatureRecomputeTargets(
data: FeatureRecomputeJobData,
): Promise<PlayerMatchFeature[]> {
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
return playerMatchFeatureModel
.find(
this.featureRecomputeTargetQuery({
queueId: data.queueId,
puuid: data.puuid,
}),
)
.sort({ gameCreation: -1 })
.limit(data.batchSize)
.lean<PlayerMatchFeature[]>()
.exec();
}
private async recomputeStoredFeature(
targetFeature: PlayerMatchFeature,
): Promise<ExtractedPlayerMatchFeature | undefined> {
const riotMatchModel = await this.connectionService.getModel('riotMatch');
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
const storedMatch = await riotMatchModel
.findOne({ matchId: targetFeature.matchId })
.lean<RiotMatch>()
.exec();
if (!storedMatch) {
this.logger.warn(
`Skipping feature recompute for matchId="${this.safeLogText(targetFeature.matchId)}": stored match was not found`,
);
return undefined;
}
try {
const feature = this.featureExtractorService.extractPlayerMatchFeature(
storedMatch.raw,
targetFeature.puuid,
);
if (feature.queueId !== targetFeature.queueId) {
return undefined;
}
if (isTimelinePayload(storedMatch.timeline)) {
const derived = this.timelineDerivedService.buildDerivedData(
storedMatch.timeline,
storedMatch.raw,
);
const timelineMetrics =
this.timelineMetricService.extractTimelineMetrics(
feature,
storedMatch.raw,
storedMatch.timeline,
derived,
);
feature.derived = {
...derived,
};
feature.features = {
...feature.features,
...timelineMetrics.features,
};
feature.normalized = {
...feature.normalized,
...timelineMetrics.normalized,
};
feature.vector = timelineMetrics.vector;
feature.timelineProcessed = true;
} else {
feature.derived = {
[TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_FIELD]:
FEATURE_SCHEMA_VERSION,
};
}
await playerMatchFeatureModel
.updateOne(
{ matchId: feature.matchId, puuid: feature.puuid },
{ $set: feature },
{ upsert: true },
)
.exec();
return feature;
} catch (error) {
this.logger.warn(
`Skipping feature recompute for matchId="${this.safeLogText(targetFeature.matchId)}": ${this.safeLogText(errorMessage(error))}`,
);
return undefined;
}
}
private async rebuildStoredProfile(
puuid: string,
queueId: number,
): Promise<void> {
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
const hasRemainingTargets =
await this.hasRemainingFeatureRecomputeTargetsForPlayer(puuid, queueId);
if (hasRemainingTargets) {
return;
}
const accountModel = await this.connectionService.getModel('account');
const account = await accountModel
.findOne({ puuid })
.lean<Pick<Account, 'isSeed' | 'puuid'> | null>()
.exec();
const features = await playerMatchFeatureModel
.find({
puuid,
queueId,
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
})
.sort({ gameCreation: -1 })
.limit(MAX_ANALYSIS_MATCH_COUNT)
.lean<PlayerMatchFeature[]>()
.exec();
if (features.length === 0) {
return;
}
await this.storeBuiltProfile({
features,
isSeed: account?.isSeed ?? false,
puuid,
queueId,
});
}
private async buildAndStoreProfile(
account: Account,
data: UserAnalysisJobData,
extractedFeatures: ExtractedPlayerMatchFeature[],
): Promise<void> {
const profile = this.profileBuilderService.buildProfile({
await this.storeBuiltProfile({
features: extractedFeatures,
isSeed: account.isSeed,
puuid: account.puuid,
queueId: data.queueId,
});
const playerProfileModel =
await this.connectionService.getModel('playerProfile');
const accountModel = await this.connectionService.getModel('account');
const analyzedAt = new Date();
await playerProfileModel
.updateOne(
{
puuid: account.puuid,
queueId: data.queueId,
},
{ $set: profile },
{ upsert: true },
)
.exec();
await accountModel
.updateOne(
{ puuid: account.puuid },
@@ -464,13 +687,82 @@ export class AnalysisPipelineService {
.exec();
}
private async completeJob(jobId: string): Promise<void> {
await this.updateJob(jobId, {
status: 'completed',
stage: 'completed',
completedAt: new Date(),
error: undefined,
private async hasRemainingFeatureRecomputeTargetsForPlayer(
puuid: string,
queueId: number,
): Promise<boolean> {
const playerMatchFeatureModel =
await this.connectionService.getModel('playerMatchFeature');
const remainingTargets = await playerMatchFeatureModel
.find(this.featureRecomputeTargetQuery({ puuid, queueId }))
.sort({ gameCreation: -1 })
.limit(1)
.lean<PlayerMatchFeature[]>()
.exec();
return remainingTargets.length > 0;
}
private featureRecomputeTargetQuery(params: {
puuid?: string;
queueId: number;
}): Record<string, unknown> {
return {
...(params.puuid ? { puuid: params.puuid } : {}),
queueId: params.queueId,
$or: [
{ featureSchemaVersion: { $lt: FEATURE_SCHEMA_VERSION } },
{
timelineProcessed: false,
[TIMELINE_UNAVAILABLE_FEATURE_SCHEMA_VERSION_PATH]: {
$ne: FEATURE_SCHEMA_VERSION,
},
},
],
};
}
private async storeBuiltProfile(params: {
features: PlayerMatchFeature[] | ExtractedPlayerMatchFeature[];
isSeed: boolean;
puuid: string;
queueId: number;
}): Promise<void> {
const profile = this.profileBuilderService.buildProfile({
features: params.features,
isSeed: params.isSeed,
puuid: params.puuid,
queueId: params.queueId,
});
const playerProfileModel =
await this.connectionService.getModel('playerProfile');
await playerProfileModel
.updateOne(
{
puuid: params.puuid,
queueId: params.queueId,
},
{ $set: profile },
{ upsert: true },
)
.exec();
}
private async completeJob(
jobId: string,
jobType = USER_ANALYSIS_JOB_NAME,
): Promise<void> {
await this.updateJob(
jobId,
{
status: 'completed',
stage: 'completed',
completedAt: new Date(),
error: undefined,
},
jobType,
);
}
private async updateJob(
@@ -483,12 +775,13 @@ export class AnalysisPipelineService {
stage?: string;
status?: AnalysisJobStatus;
},
jobType = USER_ANALYSIS_JOB_NAME,
): Promise<void> {
const analysisJobModel =
await this.connectionService.getModel('analysisJob');
const set: Record<string, unknown> = {
jobId,
jobType: USER_ANALYSIS_JOB_NAME,
jobType,
...patch,
};
const unset: Record<string, string> = {};
@@ -585,6 +878,25 @@ export class AnalysisPipelineService {
);
}
private logFeatureRecomputeFinalFailure(
data: FeatureRecomputeJobData,
jobId: string,
publicError: string,
error: unknown,
): void {
this.logger.error(
[
`Feature recompute job failed jobId="${this.safeLogText(jobId)}"`,
`puuid="${this.safeLogText(data.puuid ?? '')}"`,
`platformRoute="${data.platformRoute}"`,
`queueId="${data.queueId}"`,
`publicError="${this.safeLogText(publicError)}"`,
`cause="${this.safeLogText(errorMessage(error))}"`,
].join(' '),
error instanceof Error ? error.stack : undefined,
);
}
private safeLogText(value: string): string {
return value.replace(/[\r\n\t]/gu, ' ').slice(0, 300);
}