From 9a17502d292be5baa5479a824d8db3e0be209e30 Mon Sep 17 00:00:00 2001 From: dahyeon Date: Fri, 3 Jul 2026 01:26:46 -0700 Subject: [PATCH] Apply population percentile normalization --- .../analysis-pipeline.service.spec.ts | 40 +++++ .../analysis/analysis-pipeline.service.ts | 34 +++- .../src/modules/analysis/analysis.module.ts | 3 + .../src/modules/analysis/analysis.types.ts | 5 +- .../metric-normalization.service.spec.ts | 148 ++++++++++++++++ .../analysis/metric-normalization.service.ts | 166 ++++++++++++++++++ .../analysis/profile-builder.service.spec.ts | 28 +++ .../analysis/profile-builder.service.ts | 18 +- 8 files changed, 432 insertions(+), 10 deletions(-) create mode 100644 apps/api/src/modules/analysis/metric-normalization.service.spec.ts create mode 100644 apps/api/src/modules/analysis/metric-normalization.service.ts diff --git a/apps/api/src/modules/analysis/analysis-pipeline.service.spec.ts b/apps/api/src/modules/analysis/analysis-pipeline.service.spec.ts index baf1838..0d5dd46 100644 --- a/apps/api/src/modules/analysis/analysis-pipeline.service.spec.ts +++ b/apps/api/src/modules/analysis/analysis-pipeline.service.spec.ts @@ -8,8 +8,10 @@ import type { } from '../global/database/schemas'; import { RiotNotFoundError, type RiotApiClient } from '../riot/riot-api.client'; import type { MetricBaselineService } from './metric-baseline.service'; +import type { MetricNormalizationService } from './metric-normalization.service'; import type { BaselineComputationJobData, + ExtractedPlayerMatchFeature, FeatureRecomputeJobData, UserAnalysisJobData, } from './analysis.types'; @@ -270,6 +272,7 @@ interface PipelineHarnessOptions { [string, string] >; matchIds?: string[]; + normalizeFeature?: jest.Mock; storedAccount?: Account; } @@ -368,6 +371,8 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) { patch: 'ALL', samplesRead: 30, }); + const normalizeFeature = + options.normalizeFeature ?? jest.fn((feature) => Promise.resolve(feature)); const riotApiClient = { getAccountByRiotId, getSummonerByPuuid: jest.fn().mockResolvedValue({ @@ -388,6 +393,7 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) { getMatchIds, getMatch, getMatchTimeline, + normalizeFeature, service: new AnalysisPipelineService( connectionService, riotApiClient, @@ -398,6 +404,9 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) { { computeMetricBaselines, } as unknown as MetricBaselineService, + { + normalizeFeature, + } as unknown as MetricNormalizationService, ), updateAnalysisJob, updatePlayerMatchFeature, @@ -630,6 +639,37 @@ describe('AnalysisPipelineService', () => { expect(seedStageUpdate).toBeDefined(); }); + it('stores population-percentile normalized features and profiles when baselines are available', async () => { + const normalizeFeature = jest.fn((feature: ExtractedPlayerMatchFeature) => + Promise.resolve({ + ...feature, + normalized: { + ...feature.normalized, + kda: 0.42, + }, + normalizationMethod: 'population-percentile', + }), + ); + const { service, updatePlayerMatchFeature, updatePlayerProfile } = + createPipelineHarness({ + normalizeFeature, + }); + + await service.processUserAnalysisJob(createJobData(), 'analysis-job-1'); + + expect(normalizeFeature).toHaveBeenCalledTimes(1); + const featureSet = asRecord( + asRecord(updatePlayerMatchFeature.mock.calls[0]?.[1]).$set, + ); + expect(asRecord(featureSet.normalized).kda).toBe(0.42); + expect(featureSet.normalizationMethod).toBe('population-percentile'); + + const profileSet = asRecord( + asRecord(updatePlayerProfile.mock.calls[0]?.[1]).$set, + ); + expect(profileSet.normalizationMethod).toBe('population-percentile'); + }); + it('skips seed pipeline processing when the seed account is missing', async () => { const { findOneAccount, getMatchIds, service, updatePlayerProfile } = createPipelineHarness({ diff --git a/apps/api/src/modules/analysis/analysis-pipeline.service.ts b/apps/api/src/modules/analysis/analysis-pipeline.service.ts index f3893b4..8d5c151 100644 --- a/apps/api/src/modules/analysis/analysis-pipeline.service.ts +++ b/apps/api/src/modules/analysis/analysis-pipeline.service.ts @@ -32,6 +32,7 @@ import type { } from './analysis.types'; import { FeatureExtractorService } from './feature-extractor.service'; import { MetricBaselineService } from './metric-baseline.service'; +import { MetricNormalizationService } from './metric-normalization.service'; import { ProfileBuilderService } from './profile-builder.service'; import { TimelineDerivedService } from './timeline-derived.service'; import { TimelineMetricService } from './timeline-metric.service'; @@ -103,6 +104,8 @@ export class AnalysisPipelineService { private readonly timelineMetricService: TimelineMetricService, @Optional() private readonly metricBaselineService?: MetricBaselineService, + @Optional() + private readonly metricNormalizationService?: MetricNormalizationService, ) {} public async processUserAnalysisJob( @@ -684,14 +687,19 @@ export class AnalysisPipelineService { feature.timelineProcessed = true; } + const normalizedFeature = await this.normalizeExtractedFeature(feature); + await playerMatchFeatureModel .updateOne( - { matchId: feature.matchId, puuid: feature.puuid }, - { $set: feature }, + { + matchId: normalizedFeature.matchId, + puuid: normalizedFeature.puuid, + }, + { $set: normalizedFeature }, { upsert: true }, ) .exec(); - extractedFeatures.push(feature); + extractedFeatures.push(normalizedFeature); } catch (error) { this.logger.warn( `Skipping feature extraction for matchId="${this.safeLogText(matchRecord.matchId)}": ${this.safeLogText(errorMessage(error))}`, @@ -702,6 +710,15 @@ export class AnalysisPipelineService { return extractedFeatures; } + private async normalizeExtractedFeature( + feature: ExtractedPlayerMatchFeature, + ): Promise { + if (!this.metricNormalizationService) { + return feature; + } + return this.metricNormalizationService.normalizeFeature(feature); + } + private async findFeatureRecomputeTargets( data: FeatureRecomputeJobData, ): Promise { @@ -780,14 +797,19 @@ export class AnalysisPipelineService { }; } + const normalizedFeature = await this.normalizeExtractedFeature(feature); + await playerMatchFeatureModel .updateOne( - { matchId: feature.matchId, puuid: feature.puuid }, - { $set: feature }, + { + matchId: normalizedFeature.matchId, + puuid: normalizedFeature.puuid, + }, + { $set: normalizedFeature }, { upsert: true }, ) .exec(); - return feature; + return normalizedFeature; } catch (error) { this.logger.warn( `Skipping feature recompute for matchId="${this.safeLogText(targetFeature.matchId)}": ${this.safeLogText(errorMessage(error))}`, diff --git a/apps/api/src/modules/analysis/analysis.module.ts b/apps/api/src/modules/analysis/analysis.module.ts index 3f06305..071f5d6 100644 --- a/apps/api/src/modules/analysis/analysis.module.ts +++ b/apps/api/src/modules/analysis/analysis.module.ts @@ -19,6 +19,7 @@ import { AnalysisProcessor } from './analysis.processor'; import { createBullMqOptions } from './bullmq.config'; import { FeatureExtractorService } from './feature-extractor.service'; import { MetricBaselineService } from './metric-baseline.service'; +import { MetricNormalizationService } from './metric-normalization.service'; import { ProfileBuilderService } from './profile-builder.service'; import { SeedCollectionProcessor } from './seed-collection.processor'; import { SeedCollectionService } from './seed-collection.service'; @@ -58,6 +59,7 @@ import { TimelineMetricService } from './timeline-metric.service'; SeedCollectionProcessor, FeatureExtractorService, MetricBaselineService, + MetricNormalizationService, ProfileBuilderService, SeedCollectionService, TimelineDerivedService, @@ -68,6 +70,7 @@ import { TimelineMetricService } from './timeline-metric.service'; AnalysisQueueService, AnalysisPipelineService, MetricBaselineService, + MetricNormalizationService, SeedCollectionService, ], }) diff --git a/apps/api/src/modules/analysis/analysis.types.ts b/apps/api/src/modules/analysis/analysis.types.ts index d255edd..35cbf64 100644 --- a/apps/api/src/modules/analysis/analysis.types.ts +++ b/apps/api/src/modules/analysis/analysis.types.ts @@ -2,6 +2,7 @@ import type { Role } from '@persona-lenz/shared'; import type { BaselineCohort, LaneOpponentRef, + NormalizationMethod, SeedSource, TeamPositionOrUnknown, } from '../global/database/schemas'; @@ -65,7 +66,7 @@ export interface ExtractedPlayerMatchFeature { normalized: Record; vector: number[]; featureSchemaVersion: number; - normalizationMethod: 'registry-baseline'; + normalizationMethod: NormalizationMethod; confidence: number; timelineProcessed: boolean; laneOpponent?: LaneOpponentRef; @@ -86,7 +87,7 @@ export interface BuiltPlayerProfile { profile: Record; vector: number[]; featureSchemaVersion: number; - normalizationMethod: 'registry-baseline'; + normalizationMethod: NormalizationMethod; confidence: number; patches: string[]; lastMatchAt?: Date; diff --git a/apps/api/src/modules/analysis/metric-normalization.service.spec.ts b/apps/api/src/modules/analysis/metric-normalization.service.spec.ts new file mode 100644 index 0000000..663b619 --- /dev/null +++ b/apps/api/src/modules/analysis/metric-normalization.service.spec.ts @@ -0,0 +1,148 @@ +import { FEATURE_SCHEMA_VERSION, VECTOR_METRICS } from '@persona-lenz/shared'; +import type { ConnectionService } from '../global/database/connection.service'; +import type { MetricBaseline } from '../global/database/schemas'; +import type { ExtractedPlayerMatchFeature } from './analysis.types'; +import { MetricNormalizationService } from './metric-normalization.service'; + +interface ExecQuery { + exec: jest.Mock, []>; +} + +function leanExecResolved(value: T) { + return { + lean: jest.fn((): ExecQuery => ({ + exec: jest.fn, []>().mockResolvedValue(value), + })), + }; +} + +function createFeature( + overrides: Partial = {}, +): ExtractedPlayerMatchFeature { + return { + matchId: 'KR_1', + puuid: 'target-puuid', + queueId: 420, + role: 'MIDDLE', + championId: 103, + gameCreation: 1_700_000_000_000, + gameDurationSec: 1_800, + win: true, + patch: '26.13', + features: { + csPerMin: 8, + kda: 30, + }, + normalized: { + csPerMin: 0.5, + kda: 0.5, + }, + vector: VECTOR_METRICS.map((metric) => + metric.id === 'kda' || metric.id === 'csPerMin' ? 0.5 : 0, + ), + featureSchemaVersion: FEATURE_SCHEMA_VERSION, + normalizationMethod: 'registry-baseline', + confidence: 1, + timelineProcessed: true, + ...overrides, + }; +} + +function createBaseline( + metricId: string, + overrides: Partial = {}, +): MetricBaseline { + return { + createdAt: new Date('2026-07-01T00:00:00.000Z'), + updatedAt: new Date('2026-07-01T00:00:00.000Z'), + metricId, + role: 'MIDDLE', + cohort: 'MASTER_PLUS', + patch: 'ALL', + featureSchemaVersion: FEATURE_SCHEMA_VERSION, + sampleCount: 100, + percentileBreakpoints: [ + 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, + 170, 180, 190, + ], + computedAt: new Date('2026-07-03T12:00:00.000Z'), + ...overrides, + }; +} + +describe('MetricNormalizationService', () => { + it('normalizes observed metrics with population percentile baselines', async () => { + const find = jest.fn().mockReturnValue( + leanExecResolved([ + createBaseline('kda'), + createBaseline('csPerMin', { + percentileBreakpoints: [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + ], + patch: '26.13', + }), + createBaseline('csPerMin', { + percentileBreakpoints: [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + ], + }), + ]), + ); + const connectionService = { + getModel: jest.fn().mockResolvedValue({ find }), + } as unknown as ConnectionService; + const service = new MetricNormalizationService(connectionService); + + const normalizedFeature = await service.normalizeFeature(createFeature()); + + expect(find).toHaveBeenCalledWith({ + cohort: 'MASTER_PLUS', + featureSchemaVersion: FEATURE_SCHEMA_VERSION, + metricId: { + $in: ['csPerMin', 'kda'], + }, + patch: { + $in: ['26.13', 'ALL'], + }, + role: 'MIDDLE', + }); + expect(normalizedFeature.normalizationMethod).toBe('population-percentile'); + expect(normalizedFeature.normalized.kda).toBeCloseTo(0.15); + expect(normalizedFeature.normalized.csPerMin).toBeCloseTo(0.4); + expect(normalizedFeature.vector).toEqual( + VECTOR_METRICS.map( + (metric) => normalizedFeature.normalized[metric.id] ?? 0, + ), + ); + }); + + it('keeps registry normalization when a baseline is missing', async () => { + const find = jest + .fn() + .mockReturnValue(leanExecResolved([createBaseline('kda')])); + const connectionService = { + getModel: jest.fn().mockResolvedValue({ find }), + } as unknown as ConnectionService; + const service = new MetricNormalizationService(connectionService); + const feature = createFeature(); + + const normalizedFeature = await service.normalizeFeature(feature); + + expect(normalizedFeature).toBe(feature); + expect(normalizedFeature.normalizationMethod).toBe('registry-baseline'); + }); + + it('keeps registry normalization for unknown roles', async () => { + const find = jest.fn(); + const connectionService = { + getModel: jest.fn().mockResolvedValue({ find }), + } as unknown as ConnectionService; + const service = new MetricNormalizationService(connectionService); + const feature = createFeature({ role: 'UNKNOWN' }); + + const normalizedFeature = await service.normalizeFeature(feature); + + expect(normalizedFeature).toBe(feature); + expect(find).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/modules/analysis/metric-normalization.service.ts b/apps/api/src/modules/analysis/metric-normalization.service.ts new file mode 100644 index 0000000..dead9aa --- /dev/null +++ b/apps/api/src/modules/analysis/metric-normalization.service.ts @@ -0,0 +1,166 @@ +import { Injectable } from '@nestjs/common'; +import { FEATURE_SCHEMA_VERSION, VECTOR_METRICS } from '@persona-lenz/shared'; +import { ConnectionService } from '../global/database/connection.service'; +import type { + BaselineCohort, + MetricBaseline, + TeamPosition, +} from '../global/database/schemas'; +import type { ExtractedPlayerMatchFeature } from './analysis.types'; + +interface NormalizeFeatureOptions { + cohort?: BaselineCohort; +} + +const BASELINE_PERCENTILE_RANKS = Array.from( + { length: 19 }, + (_, index) => (index + 1) * 0.05, +); +const TEAM_POSITIONS = ['TOP', 'JUNGLE', 'MIDDLE', 'BOTTOM', 'UTILITY']; + +function isTeamPosition(value: unknown): value is TeamPosition { + return typeof value === 'string' && TEAM_POSITIONS.includes(value); +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) + ? value + : undefined; +} + +function percentileRank( + rawValue: number, + percentileBreakpoints: number[], +): number { + const breakpoints = percentileBreakpoints.filter((value) => + Number.isFinite(value), + ); + if (breakpoints.length !== BASELINE_PERCENTILE_RANKS.length) { + return Number.NaN; + } + if (breakpoints[0] === breakpoints[breakpoints.length - 1]) { + return 0.5; + } + if (rawValue < (breakpoints[0] ?? 0)) { + return 0; + } + if (rawValue > (breakpoints[breakpoints.length - 1] ?? 0)) { + return 1; + } + + for (let index = 0; index < breakpoints.length; index += 1) { + const currentBreakpoint = breakpoints[index] ?? 0; + const currentRank = BASELINE_PERCENTILE_RANKS[index] ?? 0; + if (rawValue === currentBreakpoint) { + return currentRank; + } + if (rawValue < currentBreakpoint) { + const previousBreakpoint = breakpoints[index - 1] ?? currentBreakpoint; + const previousRank = BASELINE_PERCENTILE_RANKS[index - 1] ?? 0; + if (currentBreakpoint === previousBreakpoint) { + return currentRank; + } + return ( + previousRank + + ((rawValue - previousBreakpoint) / + (currentBreakpoint - previousBreakpoint)) * + (currentRank - previousRank) + ); + } + } + + return 1; +} + +@Injectable() +export class MetricNormalizationService { + constructor(private readonly connectionService: ConnectionService) {} + + public async normalizeFeature( + feature: ExtractedPlayerMatchFeature, + options: NormalizeFeatureOptions = {}, + ): Promise { + if (!isTeamPosition(feature.role)) { + return feature; + } + + const observedMetricIds = Object.keys(feature.normalized).filter( + (metricId) => finiteNumber(feature.features[metricId]) !== undefined, + ); + if (observedMetricIds.length === 0) { + return feature; + } + + const baselinesByMetricId = await this.findBaselinesByMetricId({ + cohort: options.cohort ?? 'MASTER_PLUS', + metricIds: observedMetricIds, + patch: feature.patch, + role: feature.role, + }); + if ( + observedMetricIds.some( + (metricId) => baselinesByMetricId.get(metricId) === undefined, + ) + ) { + return feature; + } + + const normalized = { ...feature.normalized }; + for (const metricId of observedMetricIds) { + const rawValue = finiteNumber(feature.features[metricId]); + const baseline = baselinesByMetricId.get(metricId); + if (rawValue === undefined || !baseline) { + return feature; + } + + const normalizedValue = percentileRank( + rawValue, + baseline.percentileBreakpoints, + ); + if (!Number.isFinite(normalizedValue)) { + return feature; + } + normalized[metricId] = Math.max(0, Math.min(1, normalizedValue)); + } + + return { + ...feature, + normalized, + vector: VECTOR_METRICS.map((metric) => normalized[metric.id] ?? 0), + normalizationMethod: 'population-percentile', + }; + } + + private async findBaselinesByMetricId(params: { + cohort: BaselineCohort; + metricIds: string[]; + patch: string; + role: TeamPosition; + }): Promise> { + const metricBaselineModel = + await this.connectionService.getModel('metricBaseline'); + const baselines = await metricBaselineModel + .find({ + metricId: { + $in: params.metricIds, + }, + role: params.role, + cohort: params.cohort, + patch: { + $in: [params.patch, 'ALL'], + }, + featureSchemaVersion: FEATURE_SCHEMA_VERSION, + }) + .lean() + .exec(); + + const baselinesByMetricId = new Map(); + for (const baseline of baselines) { + const existingBaseline = baselinesByMetricId.get(baseline.metricId); + if (!existingBaseline || baseline.patch === params.patch) { + baselinesByMetricId.set(baseline.metricId, baseline); + } + } + return baselinesByMetricId; + } +} diff --git a/apps/api/src/modules/analysis/profile-builder.service.spec.ts b/apps/api/src/modules/analysis/profile-builder.service.spec.ts index 6046750..f0dca1d 100644 --- a/apps/api/src/modules/analysis/profile-builder.service.spec.ts +++ b/apps/api/src/modules/analysis/profile-builder.service.spec.ts @@ -132,6 +132,34 @@ describe('ProfileBuilderService', () => { expect(profile.confidence).toBeCloseTo(0.34); }); + it('marks profiles as population percentile only when every feature uses it', () => { + const populationProfile = service.buildProfile({ + features: [ + createFeature({ normalizationMethod: 'population-percentile' }), + createFeature({ + matchId: 'KR_2', + normalizationMethod: 'population-percentile', + }), + ], + isSeed: false, + puuid: 'target-puuid', + queueId: 420, + }); + + const mixedProfile = service.buildProfile({ + features: [ + createFeature({ normalizationMethod: 'population-percentile' }), + createFeature({ matchId: 'KR_2' }), + ], + isSeed: false, + puuid: 'target-puuid', + queueId: 420, + }); + + expect(populationProfile.normalizationMethod).toBe('population-percentile'); + expect(mixedProfile.normalizationMethod).toBe('registry-baseline'); + }); + it('aggregates ratio metrics from raw numerator and denominator counts', () => { const metrics: MetricDefinition[] = [ { diff --git a/apps/api/src/modules/analysis/profile-builder.service.ts b/apps/api/src/modules/analysis/profile-builder.service.ts index 73beb2e..dc98958 100644 --- a/apps/api/src/modules/analysis/profile-builder.service.ts +++ b/apps/api/src/modules/analysis/profile-builder.service.ts @@ -6,7 +6,10 @@ import { type Role, } from '@persona-lenz/shared'; import { Injectable } from '@nestjs/common'; -import type { PlayerMatchFeature } from '../global/database/schemas'; +import type { + NormalizationMethod, + PlayerMatchFeature, +} from '../global/database/schemas'; import type { BuiltPlayerProfile } from './analysis.types'; type ProfileSourceFeature = Pick< @@ -16,6 +19,7 @@ type ProfileSourceFeature = Pick< | 'features' | 'gameCreation' | 'normalized' + | 'normalizationMethod' | 'patch' | 'role' | 'timelineProcessed' @@ -105,6 +109,16 @@ function isRole(value: string): value is Role { return ROLE_ORDER.includes(value as Role); } +function resolveProfileNormalizationMethod( + features: readonly ProfileSourceFeature[], +): NormalizationMethod { + return features.every( + (feature) => feature.normalizationMethod === 'population-percentile', + ) + ? 'population-percentile' + : 'registry-baseline'; +} + @Injectable() export class ProfileBuilderService { public buildProfile(params: BuildProfileParams): BuiltPlayerProfile { @@ -176,7 +190,7 @@ export class ProfileBuilderService { profile, vector: VECTOR_METRICS.map((metric) => profile[metric.id] ?? 0), featureSchemaVersion: FEATURE_SCHEMA_VERSION, - normalizationMethod: 'registry-baseline', + normalizationMethod: resolveProfileNormalizationMethod(params.features), confidence: averageConfidence * sampleSizeConfidenceMultiplier *