Apply population percentile normalization

This commit is contained in:
dahyeon committed 2026-07-03 01:26:46 -07:00
1 parent 2935a526c7
commit 9a17502d29
8 files changed
+432 -10

No files matched your search

@@ -8,8 +8,10 @@ import type {
} from '../global/database/schemas'; } from '../global/database/schemas';
import { RiotNotFoundError, type RiotApiClient } from '../riot/riot-api.client'; import { RiotNotFoundError, type RiotApiClient } from '../riot/riot-api.client';
import type { MetricBaselineService } from './metric-baseline.service'; import type { MetricBaselineService } from './metric-baseline.service';
import type { MetricNormalizationService } from './metric-normalization.service';
import type { import type {
BaselineComputationJobData, BaselineComputationJobData,
ExtractedPlayerMatchFeature,
FeatureRecomputeJobData, FeatureRecomputeJobData,
UserAnalysisJobData, UserAnalysisJobData,
} from './analysis.types'; } from './analysis.types';
@@ -270,6 +272,7 @@ interface PipelineHarnessOptions {
[string, string] [string, string]
>; >;
matchIds?: string[]; matchIds?: string[];
normalizeFeature?: jest.Mock;
storedAccount?: Account; storedAccount?: Account;
} }
@@ -368,6 +371,8 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) {
patch: 'ALL', patch: 'ALL',
samplesRead: 30, samplesRead: 30,
}); });
const normalizeFeature =
options.normalizeFeature ?? jest.fn((feature) => Promise.resolve(feature));
const riotApiClient = { const riotApiClient = {
getAccountByRiotId, getAccountByRiotId,
getSummonerByPuuid: jest.fn().mockResolvedValue({ getSummonerByPuuid: jest.fn().mockResolvedValue({
@@ -388,6 +393,7 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) {
getMatchIds, getMatchIds,
getMatch, getMatch,
getMatchTimeline, getMatchTimeline,
normalizeFeature,
service: new AnalysisPipelineService( service: new AnalysisPipelineService(
connectionService, connectionService,
riotApiClient, riotApiClient,
@@ -398,6 +404,9 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) {
{ {
computeMetricBaselines, computeMetricBaselines,
} as unknown as MetricBaselineService, } as unknown as MetricBaselineService,
{
normalizeFeature,
} as unknown as MetricNormalizationService,
), ),
updateAnalysisJob, updateAnalysisJob,
updatePlayerMatchFeature, updatePlayerMatchFeature,
@@ -630,6 +639,37 @@ describe('AnalysisPipelineService', () => {
expect(seedStageUpdate).toBeDefined(); expect(seedStageUpdate).toBeDefined();
}); });
it('stores population-percentile normalized features and profiles when baselines are available', async () => {
const normalizeFeature = jest.fn((feature: ExtractedPlayerMatchFeature) =>
Promise.resolve({
...feature,
normalized: {
...feature.normalized,
kda: 0.42,
},
normalizationMethod: 'population-percentile',
}),
);
const { service, updatePlayerMatchFeature, updatePlayerProfile } =
createPipelineHarness({
normalizeFeature,
});
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
expect(normalizeFeature).toHaveBeenCalledTimes(1);
const featureSet = asRecord(
asRecord(updatePlayerMatchFeature.mock.calls[0]?.[1]).$set,
);
expect(asRecord(featureSet.normalized).kda).toBe(0.42);
expect(featureSet.normalizationMethod).toBe('population-percentile');
const profileSet = asRecord(
asRecord(updatePlayerProfile.mock.calls[0]?.[1]).$set,
);
expect(profileSet.normalizationMethod).toBe('population-percentile');
});
it('skips seed pipeline processing when the seed account is missing', async () => { it('skips seed pipeline processing when the seed account is missing', async () => {
const { findOneAccount, getMatchIds, service, updatePlayerProfile } = const { findOneAccount, getMatchIds, service, updatePlayerProfile } =
createPipelineHarness({ createPipelineHarness({
@@ -32,6 +32,7 @@ import type {
} from './analysis.types'; } from './analysis.types';
import { FeatureExtractorService } from './feature-extractor.service'; import { FeatureExtractorService } from './feature-extractor.service';
import { MetricBaselineService } from './metric-baseline.service'; import { MetricBaselineService } from './metric-baseline.service';
import { MetricNormalizationService } from './metric-normalization.service';
import { ProfileBuilderService } from './profile-builder.service'; import { ProfileBuilderService } from './profile-builder.service';
import { TimelineDerivedService } from './timeline-derived.service'; import { TimelineDerivedService } from './timeline-derived.service';
import { TimelineMetricService } from './timeline-metric.service'; import { TimelineMetricService } from './timeline-metric.service';
@@ -103,6 +104,8 @@ export class AnalysisPipelineService {
private readonly timelineMetricService: TimelineMetricService, private readonly timelineMetricService: TimelineMetricService,
@Optional() @Optional()
private readonly metricBaselineService?: MetricBaselineService, private readonly metricBaselineService?: MetricBaselineService,
@Optional()
private readonly metricNormalizationService?: MetricNormalizationService,
) {} ) {}
public async processUserAnalysisJob( public async processUserAnalysisJob(
@@ -684,14 +687,19 @@ export class AnalysisPipelineService {
feature.timelineProcessed = true; feature.timelineProcessed = true;
} }
const normalizedFeature = await this.normalizeExtractedFeature(feature);
await playerMatchFeatureModel await playerMatchFeatureModel
.updateOne( .updateOne(
{ matchId: feature.matchId, puuid: feature.puuid }, {
{ $set: feature }, matchId: normalizedFeature.matchId,
puuid: normalizedFeature.puuid,
},
{ $set: normalizedFeature },
{ upsert: true }, { upsert: true },
) )
.exec(); .exec();
extractedFeatures.push(feature); extractedFeatures.push(normalizedFeature);
} catch (error) { } catch (error) {
this.logger.warn( this.logger.warn(
`Skipping feature extraction for matchId="${this.safeLogText(matchRecord.matchId)}": ${this.safeLogText(errorMessage(error))}`, `Skipping feature extraction for matchId="${this.safeLogText(matchRecord.matchId)}": ${this.safeLogText(errorMessage(error))}`,
@@ -702,6 +710,15 @@ export class AnalysisPipelineService {
return extractedFeatures; return extractedFeatures;
} }
private async normalizeExtractedFeature(
feature: ExtractedPlayerMatchFeature,
): Promise<ExtractedPlayerMatchFeature> {
if (!this.metricNormalizationService) {
return feature;
}
return this.metricNormalizationService.normalizeFeature(feature);
}
private async findFeatureRecomputeTargets( private async findFeatureRecomputeTargets(
data: FeatureRecomputeJobData, data: FeatureRecomputeJobData,
): Promise<PlayerMatchFeature[]> { ): Promise<PlayerMatchFeature[]> {
@@ -780,14 +797,19 @@ export class AnalysisPipelineService {
}; };
} }
const normalizedFeature = await this.normalizeExtractedFeature(feature);
await playerMatchFeatureModel await playerMatchFeatureModel
.updateOne( .updateOne(
{ matchId: feature.matchId, puuid: feature.puuid }, {
{ $set: feature }, matchId: normalizedFeature.matchId,
puuid: normalizedFeature.puuid,
},
{ $set: normalizedFeature },
{ upsert: true }, { upsert: true },
) )
.exec(); .exec();
return feature; return normalizedFeature;
} catch (error) { } catch (error) {
this.logger.warn( this.logger.warn(
`Skipping feature recompute for matchId="${this.safeLogText(targetFeature.matchId)}": ${this.safeLogText(errorMessage(error))}`, `Skipping feature recompute for matchId="${this.safeLogText(targetFeature.matchId)}": ${this.safeLogText(errorMessage(error))}`,
@@ -19,6 +19,7 @@ import { AnalysisProcessor } from './analysis.processor';
import { createBullMqOptions } from './bullmq.config'; import { createBullMqOptions } from './bullmq.config';
import { FeatureExtractorService } from './feature-extractor.service'; import { FeatureExtractorService } from './feature-extractor.service';
import { MetricBaselineService } from './metric-baseline.service'; import { MetricBaselineService } from './metric-baseline.service';
import { MetricNormalizationService } from './metric-normalization.service';
import { ProfileBuilderService } from './profile-builder.service'; import { ProfileBuilderService } from './profile-builder.service';
import { SeedCollectionProcessor } from './seed-collection.processor'; import { SeedCollectionProcessor } from './seed-collection.processor';
import { SeedCollectionService } from './seed-collection.service'; import { SeedCollectionService } from './seed-collection.service';
@@ -58,6 +59,7 @@ import { TimelineMetricService } from './timeline-metric.service';
SeedCollectionProcessor, SeedCollectionProcessor,
FeatureExtractorService, FeatureExtractorService,
MetricBaselineService, MetricBaselineService,
MetricNormalizationService,
ProfileBuilderService, ProfileBuilderService,
SeedCollectionService, SeedCollectionService,
TimelineDerivedService, TimelineDerivedService,
@@ -68,6 +70,7 @@ import { TimelineMetricService } from './timeline-metric.service';
AnalysisQueueService, AnalysisQueueService,
AnalysisPipelineService, AnalysisPipelineService,
MetricBaselineService, MetricBaselineService,
MetricNormalizationService,
SeedCollectionService, SeedCollectionService,
], ],
}) })
@@ -2,6 +2,7 @@ import type { Role } from '@persona-lenz/shared';
import type { import type {
BaselineCohort, BaselineCohort,
LaneOpponentRef, LaneOpponentRef,
NormalizationMethod,
SeedSource, SeedSource,
TeamPositionOrUnknown, TeamPositionOrUnknown,
} from '../global/database/schemas'; } from '../global/database/schemas';
@@ -65,7 +66,7 @@ export interface ExtractedPlayerMatchFeature {
normalized: Record<string, number>; normalized: Record<string, number>;
vector: number[]; vector: number[];
featureSchemaVersion: number; featureSchemaVersion: number;
normalizationMethod: 'registry-baseline'; normalizationMethod: NormalizationMethod;
confidence: number; confidence: number;
timelineProcessed: boolean; timelineProcessed: boolean;
laneOpponent?: LaneOpponentRef; laneOpponent?: LaneOpponentRef;
@@ -86,7 +87,7 @@ export interface BuiltPlayerProfile {
profile: Record<string, number>; profile: Record<string, number>;
vector: number[]; vector: number[];
featureSchemaVersion: number; featureSchemaVersion: number;
normalizationMethod: 'registry-baseline'; normalizationMethod: NormalizationMethod;
confidence: number; confidence: number;
patches: string[]; patches: string[];
lastMatchAt?: Date; lastMatchAt?: Date;
@@ -0,0 +1,148 @@
import { FEATURE_SCHEMA_VERSION, VECTOR_METRICS } from '@persona-lenz/shared';
import type { ConnectionService } from '../global/database/connection.service';
import type { MetricBaseline } from '../global/database/schemas';
import type { ExtractedPlayerMatchFeature } from './analysis.types';
import { MetricNormalizationService } from './metric-normalization.service';
interface ExecQuery<T> {
exec: jest.Mock<Promise<T>, []>;
}
function leanExecResolved<T>(value: T) {
return {
lean: jest.fn((): ExecQuery<T> => ({
exec: jest.fn<Promise<T>, []>().mockResolvedValue(value),
})),
};
}
function createFeature(
overrides: Partial<ExtractedPlayerMatchFeature> = {},
): ExtractedPlayerMatchFeature {
return {
matchId: 'KR_1',
puuid: 'target-puuid',
queueId: 420,
role: 'MIDDLE',
championId: 103,
gameCreation: 1_700_000_000_000,
gameDurationSec: 1_800,
win: true,
patch: '26.13',
features: {
csPerMin: 8,
kda: 30,
},
normalized: {
csPerMin: 0.5,
kda: 0.5,
},
vector: VECTOR_METRICS.map((metric) =>
metric.id === 'kda' || metric.id === 'csPerMin' ? 0.5 : 0,
),
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
normalizationMethod: 'registry-baseline',
confidence: 1,
timelineProcessed: true,
...overrides,
};
}
function createBaseline(
metricId: string,
overrides: Partial<MetricBaseline> = {},
): MetricBaseline {
return {
createdAt: new Date('2026-07-01T00:00:00.000Z'),
updatedAt: new Date('2026-07-01T00:00:00.000Z'),
metricId,
role: 'MIDDLE',
cohort: 'MASTER_PLUS',
patch: 'ALL',
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
sampleCount: 100,
percentileBreakpoints: [
10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160,
170, 180, 190,
],
computedAt: new Date('2026-07-03T12:00:00.000Z'),
...overrides,
};
}
describe('MetricNormalizationService', () => {
it('normalizes observed metrics with population percentile baselines', async () => {
const find = jest.fn().mockReturnValue(
leanExecResolved([
createBaseline('kda'),
createBaseline('csPerMin', {
percentileBreakpoints: [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
],
patch: '26.13',
}),
createBaseline('csPerMin', {
percentileBreakpoints: [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
],
}),
]),
);
const connectionService = {
getModel: jest.fn().mockResolvedValue({ find }),
} as unknown as ConnectionService;
const service = new MetricNormalizationService(connectionService);
const normalizedFeature = await service.normalizeFeature(createFeature());
expect(find).toHaveBeenCalledWith({
cohort: 'MASTER_PLUS',
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
metricId: {
$in: ['csPerMin', 'kda'],
},
patch: {
$in: ['26.13', 'ALL'],
},
role: 'MIDDLE',
});
expect(normalizedFeature.normalizationMethod).toBe('population-percentile');
expect(normalizedFeature.normalized.kda).toBeCloseTo(0.15);
expect(normalizedFeature.normalized.csPerMin).toBeCloseTo(0.4);
expect(normalizedFeature.vector).toEqual(
VECTOR_METRICS.map(
(metric) => normalizedFeature.normalized[metric.id] ?? 0,
),
);
});
it('keeps registry normalization when a baseline is missing', async () => {
const find = jest
.fn()
.mockReturnValue(leanExecResolved([createBaseline('kda')]));
const connectionService = {
getModel: jest.fn().mockResolvedValue({ find }),
} as unknown as ConnectionService;
const service = new MetricNormalizationService(connectionService);
const feature = createFeature();
const normalizedFeature = await service.normalizeFeature(feature);
expect(normalizedFeature).toBe(feature);
expect(normalizedFeature.normalizationMethod).toBe('registry-baseline');
});
it('keeps registry normalization for unknown roles', async () => {
const find = jest.fn();
const connectionService = {
getModel: jest.fn().mockResolvedValue({ find }),
} as unknown as ConnectionService;
const service = new MetricNormalizationService(connectionService);
const feature = createFeature({ role: 'UNKNOWN' });
const normalizedFeature = await service.normalizeFeature(feature);
expect(normalizedFeature).toBe(feature);
expect(find).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,166 @@
import { Injectable } from '@nestjs/common';
import { FEATURE_SCHEMA_VERSION, VECTOR_METRICS } from '@persona-lenz/shared';
import { ConnectionService } from '../global/database/connection.service';
import type {
BaselineCohort,
MetricBaseline,
TeamPosition,
} from '../global/database/schemas';
import type { ExtractedPlayerMatchFeature } from './analysis.types';
interface NormalizeFeatureOptions {
cohort?: BaselineCohort;
}
const BASELINE_PERCENTILE_RANKS = Array.from(
{ length: 19 },
(_, index) => (index + 1) * 0.05,
);
const TEAM_POSITIONS = ['TOP', 'JUNGLE', 'MIDDLE', 'BOTTOM', 'UTILITY'];
function isTeamPosition(value: unknown): value is TeamPosition {
return typeof value === 'string' && TEAM_POSITIONS.includes(value);
}
function finiteNumber(value: unknown): number | undefined {
return typeof value === 'number' && Number.isFinite(value)
? value
: undefined;
}
function percentileRank(
rawValue: number,
percentileBreakpoints: number[],
): number {
const breakpoints = percentileBreakpoints.filter((value) =>
Number.isFinite(value),
);
if (breakpoints.length !== BASELINE_PERCENTILE_RANKS.length) {
return Number.NaN;
}
if (breakpoints[0] === breakpoints[breakpoints.length - 1]) {
return 0.5;
}
if (rawValue < (breakpoints[0] ?? 0)) {
return 0;
}
if (rawValue > (breakpoints[breakpoints.length - 1] ?? 0)) {
return 1;
}
for (let index = 0; index < breakpoints.length; index += 1) {
const currentBreakpoint = breakpoints[index] ?? 0;
const currentRank = BASELINE_PERCENTILE_RANKS[index] ?? 0;
if (rawValue === currentBreakpoint) {
return currentRank;
}
if (rawValue < currentBreakpoint) {
const previousBreakpoint = breakpoints[index - 1] ?? currentBreakpoint;
const previousRank = BASELINE_PERCENTILE_RANKS[index - 1] ?? 0;
if (currentBreakpoint === previousBreakpoint) {
return currentRank;
}
return (
previousRank +
((rawValue - previousBreakpoint) /
(currentBreakpoint - previousBreakpoint)) *
(currentRank - previousRank)
);
}
}
return 1;
}
@Injectable()
export class MetricNormalizationService {
constructor(private readonly connectionService: ConnectionService) {}
public async normalizeFeature(
feature: ExtractedPlayerMatchFeature,
options: NormalizeFeatureOptions = {},
): Promise<ExtractedPlayerMatchFeature> {
if (!isTeamPosition(feature.role)) {
return feature;
}
const observedMetricIds = Object.keys(feature.normalized).filter(
(metricId) => finiteNumber(feature.features[metricId]) !== undefined,
);
if (observedMetricIds.length === 0) {
return feature;
}
const baselinesByMetricId = await this.findBaselinesByMetricId({
cohort: options.cohort ?? 'MASTER_PLUS',
metricIds: observedMetricIds,
patch: feature.patch,
role: feature.role,
});
if (
observedMetricIds.some(
(metricId) => baselinesByMetricId.get(metricId) === undefined,
)
) {
return feature;
}
const normalized = { ...feature.normalized };
for (const metricId of observedMetricIds) {
const rawValue = finiteNumber(feature.features[metricId]);
const baseline = baselinesByMetricId.get(metricId);
if (rawValue === undefined || !baseline) {
return feature;
}
const normalizedValue = percentileRank(
rawValue,
baseline.percentileBreakpoints,
);
if (!Number.isFinite(normalizedValue)) {
return feature;
}
normalized[metricId] = Math.max(0, Math.min(1, normalizedValue));
}
return {
...feature,
normalized,
vector: VECTOR_METRICS.map((metric) => normalized[metric.id] ?? 0),
normalizationMethod: 'population-percentile',
};
}
private async findBaselinesByMetricId(params: {
cohort: BaselineCohort;
metricIds: string[];
patch: string;
role: TeamPosition;
}): Promise<Map<string, MetricBaseline>> {
const metricBaselineModel =
await this.connectionService.getModel('metricBaseline');
const baselines = await metricBaselineModel
.find({
metricId: {
$in: params.metricIds,
},
role: params.role,
cohort: params.cohort,
patch: {
$in: [params.patch, 'ALL'],
},
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
})
.lean<MetricBaseline[]>()
.exec();
const baselinesByMetricId = new Map<string, MetricBaseline>();
for (const baseline of baselines) {
const existingBaseline = baselinesByMetricId.get(baseline.metricId);
if (!existingBaseline || baseline.patch === params.patch) {
baselinesByMetricId.set(baseline.metricId, baseline);
}
}
return baselinesByMetricId;
}
}
@@ -132,6 +132,34 @@ describe('ProfileBuilderService', () => {
expect(profile.confidence).toBeCloseTo(0.34); expect(profile.confidence).toBeCloseTo(0.34);
}); });
it('marks profiles as population percentile only when every feature uses it', () => {
const populationProfile = service.buildProfile({
features: [
createFeature({ normalizationMethod: 'population-percentile' }),
createFeature({
matchId: 'KR_2',
normalizationMethod: 'population-percentile',
}),
],
isSeed: false,
puuid: 'target-puuid',
queueId: 420,
});
const mixedProfile = service.buildProfile({
features: [
createFeature({ normalizationMethod: 'population-percentile' }),
createFeature({ matchId: 'KR_2' }),
],
isSeed: false,
puuid: 'target-puuid',
queueId: 420,
});
expect(populationProfile.normalizationMethod).toBe('population-percentile');
expect(mixedProfile.normalizationMethod).toBe('registry-baseline');
});
it('aggregates ratio metrics from raw numerator and denominator counts', () => { it('aggregates ratio metrics from raw numerator and denominator counts', () => {
const metrics: MetricDefinition[] = [ const metrics: MetricDefinition[] = [
{ {
@@ -6,7 +6,10 @@ import {
type Role, type Role,
} from '@persona-lenz/shared'; } from '@persona-lenz/shared';
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import type { PlayerMatchFeature } from '../global/database/schemas'; import type {
NormalizationMethod,
PlayerMatchFeature,
} from '../global/database/schemas';
import type { BuiltPlayerProfile } from './analysis.types'; import type { BuiltPlayerProfile } from './analysis.types';
type ProfileSourceFeature = Pick< type ProfileSourceFeature = Pick<
@@ -16,6 +19,7 @@ type ProfileSourceFeature = Pick<
| 'features' | 'features'
| 'gameCreation' | 'gameCreation'
| 'normalized' | 'normalized'
| 'normalizationMethod'
| 'patch' | 'patch'
| 'role' | 'role'
| 'timelineProcessed' | 'timelineProcessed'
@@ -105,6 +109,16 @@ function isRole(value: string): value is Role {
return ROLE_ORDER.includes(value as Role); return ROLE_ORDER.includes(value as Role);
} }
function resolveProfileNormalizationMethod(
features: readonly ProfileSourceFeature[],
): NormalizationMethod {
return features.every(
(feature) => feature.normalizationMethod === 'population-percentile',
)
? 'population-percentile'
: 'registry-baseline';
}
@Injectable() @Injectable()
export class ProfileBuilderService { export class ProfileBuilderService {
public buildProfile(params: BuildProfileParams): BuiltPlayerProfile { public buildProfile(params: BuildProfileParams): BuiltPlayerProfile {
@@ -176,7 +190,7 @@ export class ProfileBuilderService {
profile, profile,
vector: VECTOR_METRICS.map((metric) => profile[metric.id] ?? 0), vector: VECTOR_METRICS.map((metric) => profile[metric.id] ?? 0),
featureSchemaVersion: FEATURE_SCHEMA_VERSION, featureSchemaVersion: FEATURE_SCHEMA_VERSION,
normalizationMethod: 'registry-baseline', normalizationMethod: resolveProfileNormalizationMethod(params.features),
confidence: confidence:
averageConfidence * averageConfidence *
sampleSizeConfidenceMultiplier * sampleSizeConfidenceMultiplier *