Avoid storing match timelines
This commit is contained in:
1 parent
08983814b5
commit
e5cbc3e4d4
2 files changed
+365
-97
No files matched your search
@@ -148,6 +148,10 @@ type UpdateOneMock = jest.Mock<
|
|||||||
ExecQuery<AcknowledgedUpdate>,
|
ExecQuery<AcknowledgedUpdate>,
|
||||||
[Record<string, unknown>, Record<string, unknown>, Record<string, unknown>?]
|
[Record<string, unknown>, Record<string, unknown>, Record<string, unknown>?]
|
||||||
>;
|
>;
|
||||||
|
type BulkWriteMock = jest.Mock<
|
||||||
|
Promise<AcknowledgedUpdate>,
|
||||||
|
[Array<Record<string, unknown>>]
|
||||||
|
>;
|
||||||
|
|
||||||
function asRecord(value: unknown): Record<string, unknown> {
|
function asRecord(value: unknown): Record<string, unknown> {
|
||||||
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
||||||
@@ -156,12 +160,35 @@ function asRecord(value: unknown): Record<string, unknown> {
|
|||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
function findTimelineSet(updateRiotMatch: UpdateOneMock) {
|
function findTimelinePersistUpdate(updateRiotMatch: UpdateOneMock) {
|
||||||
const timelineUpdate = updateRiotMatch.mock.calls.find(([, update]) =>
|
return updateRiotMatch.mock.calls.find(([, update]) =>
|
||||||
Boolean(asRecord(asRecord(update).$set).timeline),
|
Boolean(asRecord(asRecord(update).$set).timeline),
|
||||||
);
|
);
|
||||||
expect(timelineUpdate).toBeDefined();
|
}
|
||||||
return asRecord(asRecord(timelineUpdate?.[1]).$set);
|
|
||||||
|
function findTimelineDiscardUpdate(updateRiotMatch: UpdateOneMock) {
|
||||||
|
return updateRiotMatch.mock.calls.find(([, update]) =>
|
||||||
|
Object.prototype.hasOwnProperty.call(
|
||||||
|
asRecord(asRecord(update).$unset),
|
||||||
|
'timeline',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function playerMatchFeatureBulkSet(
|
||||||
|
bulkWritePlayerMatchFeature: BulkWriteMock,
|
||||||
|
puuid = 'target-puuid',
|
||||||
|
) {
|
||||||
|
const operations = bulkWritePlayerMatchFeature.mock.calls.flatMap(
|
||||||
|
([bulkWriteOperations]) => bulkWriteOperations,
|
||||||
|
);
|
||||||
|
const operation = operations.find((candidate) => {
|
||||||
|
const updateOne = asRecord(candidate.updateOne);
|
||||||
|
const filter = asRecord(updateOne.filter);
|
||||||
|
return filter.puuid === puuid;
|
||||||
|
});
|
||||||
|
expect(operation).toBeDefined();
|
||||||
|
return asRecord(asRecord(asRecord(operation?.updateOne).update).$set);
|
||||||
}
|
}
|
||||||
|
|
||||||
function execResolved<T>(value: T): ExecQuery<T> {
|
function execResolved<T>(value: T): ExecQuery<T> {
|
||||||
@@ -264,6 +291,7 @@ interface PipelineHarnessOptions {
|
|||||||
accountError?: Error;
|
accountError?: Error;
|
||||||
computeMetricBaselines?: jest.Mock;
|
computeMetricBaselines?: jest.Mock;
|
||||||
existingMatchRaw?: Record<string, unknown> | null;
|
existingMatchRaw?: Record<string, unknown> | null;
|
||||||
|
existingTargetFeatures?: PlayerMatchFeature[];
|
||||||
existingTimeline?: Record<string, unknown>;
|
existingTimeline?: Record<string, unknown>;
|
||||||
findOneAccountResult?: Account | null;
|
findOneAccountResult?: Account | null;
|
||||||
getMatch?: jest.Mock<Promise<Record<string, unknown>>, [string, string]>;
|
getMatch?: jest.Mock<Promise<Record<string, unknown>>, [string, string]>;
|
||||||
@@ -313,6 +341,12 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) {
|
|||||||
const updatePlayerMatchFeature: UpdateOneMock = jest
|
const updatePlayerMatchFeature: UpdateOneMock = jest
|
||||||
.fn<ExecQuery<AcknowledgedUpdate>, Parameters<UpdateOneMock>>()
|
.fn<ExecQuery<AcknowledgedUpdate>, Parameters<UpdateOneMock>>()
|
||||||
.mockReturnValue(execResolved({ acknowledged: true }));
|
.mockReturnValue(execResolved({ acknowledged: true }));
|
||||||
|
const bulkWritePlayerMatchFeature: BulkWriteMock = jest
|
||||||
|
.fn<Promise<AcknowledgedUpdate>, Parameters<BulkWriteMock>>()
|
||||||
|
.mockResolvedValue({ acknowledged: true });
|
||||||
|
const findPlayerMatchFeatures = jest
|
||||||
|
.fn()
|
||||||
|
.mockReturnValue(leanExecResolved(options.existingTargetFeatures ?? []));
|
||||||
const updatePlayerProfile: UpdateOneMock = jest
|
const updatePlayerProfile: UpdateOneMock = jest
|
||||||
.fn<ExecQuery<AcknowledgedUpdate>, Parameters<UpdateOneMock>>()
|
.fn<ExecQuery<AcknowledgedUpdate>, Parameters<UpdateOneMock>>()
|
||||||
.mockReturnValue(execResolved({ acknowledged: true }));
|
.mockReturnValue(execResolved({ acknowledged: true }));
|
||||||
@@ -327,6 +361,8 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) {
|
|||||||
updateOne: updateAnalysisJob,
|
updateOne: updateAnalysisJob,
|
||||||
},
|
},
|
||||||
playerMatchFeature: {
|
playerMatchFeature: {
|
||||||
|
bulkWrite: bulkWritePlayerMatchFeature,
|
||||||
|
find: findPlayerMatchFeatures,
|
||||||
updateOne: updatePlayerMatchFeature,
|
updateOne: updatePlayerMatchFeature,
|
||||||
},
|
},
|
||||||
playerProfile: {
|
playerProfile: {
|
||||||
@@ -388,7 +424,9 @@ function createPipelineHarness(options: PipelineHarnessOptions = {}) {
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
findOneAccount,
|
findOneAccount,
|
||||||
|
findPlayerMatchFeatures,
|
||||||
findOneRiotMatch,
|
findOneRiotMatch,
|
||||||
|
bulkWritePlayerMatchFeature,
|
||||||
computeMetricBaselines,
|
computeMetricBaselines,
|
||||||
getMatchIds,
|
getMatchIds,
|
||||||
getMatch,
|
getMatch,
|
||||||
@@ -526,11 +564,11 @@ describe('AnalysisPipelineService', () => {
|
|||||||
it('fetches missing match details, stores features, builds a profile, and completes the job', async () => {
|
it('fetches missing match details, stores features, builds a profile, and completes the job', async () => {
|
||||||
const {
|
const {
|
||||||
findOneRiotMatch,
|
findOneRiotMatch,
|
||||||
|
bulkWritePlayerMatchFeature,
|
||||||
getMatch,
|
getMatch,
|
||||||
getMatchTimeline,
|
getMatchTimeline,
|
||||||
service,
|
service,
|
||||||
updateAnalysisJob,
|
updateAnalysisJob,
|
||||||
updatePlayerMatchFeature,
|
|
||||||
updatePlayerProfile,
|
updatePlayerProfile,
|
||||||
updateRiotMatch,
|
updateRiotMatch,
|
||||||
} = createPipelineHarness();
|
} = createPipelineHarness();
|
||||||
@@ -548,16 +586,17 @@ describe('AnalysisPipelineService', () => {
|
|||||||
queueId: 420,
|
queueId: 420,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const timelineSet = findTimelineSet(updateRiotMatch);
|
expect(findTimelinePersistUpdate(updateRiotMatch)).toBeUndefined();
|
||||||
expect(timelineSet.timeline).toEqual(createTimelinePayload());
|
|
||||||
expect(timelineSet.timelineFetchedAt).toBeInstanceOf(Date);
|
|
||||||
|
|
||||||
const playerMatchFeatureUpdate = updatePlayerMatchFeature.mock.calls[0][1];
|
expect(bulkWritePlayerMatchFeature).toHaveBeenCalledTimes(1);
|
||||||
expect(playerMatchFeatureUpdate).toMatchObject({
|
expect(bulkWritePlayerMatchFeature.mock.calls[0]?.[0]).toHaveLength(3);
|
||||||
$set: {
|
const playerMatchFeatureSet = playerMatchFeatureBulkSet(
|
||||||
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
|
bulkWritePlayerMatchFeature,
|
||||||
matchId: 'KR_1',
|
);
|
||||||
},
|
expect(playerMatchFeatureSet).toMatchObject({
|
||||||
|
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
|
||||||
|
matchId: 'KR_1',
|
||||||
|
puuid: 'target-puuid',
|
||||||
});
|
});
|
||||||
|
|
||||||
const playerProfileUpdate = updatePlayerProfile.mock.calls[0][1];
|
const playerProfileUpdate = updatePlayerProfile.mock.calls[0][1];
|
||||||
@@ -582,10 +621,10 @@ describe('AnalysisPipelineService', () => {
|
|||||||
it('processes a stored seed account through the shared match feature pipeline', async () => {
|
it('processes a stored seed account through the shared match feature pipeline', async () => {
|
||||||
const {
|
const {
|
||||||
findOneAccount,
|
findOneAccount,
|
||||||
|
bulkWritePlayerMatchFeature,
|
||||||
getMatchIds,
|
getMatchIds,
|
||||||
service,
|
service,
|
||||||
updateAnalysisJob,
|
updateAnalysisJob,
|
||||||
updatePlayerMatchFeature,
|
|
||||||
updatePlayerProfile,
|
updatePlayerProfile,
|
||||||
} = createPipelineHarness({
|
} = createPipelineHarness({
|
||||||
findOneAccountResult: createStoredAccount({
|
findOneAccountResult: createStoredAccount({
|
||||||
@@ -620,7 +659,7 @@ describe('AnalysisPipelineService', () => {
|
|||||||
queue: 420,
|
queue: 420,
|
||||||
regionalRoute: 'asia',
|
regionalRoute: 'asia',
|
||||||
});
|
});
|
||||||
expect(updatePlayerMatchFeature).toHaveBeenCalledTimes(1);
|
expect(bulkWritePlayerMatchFeature).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
const playerProfileUpdate = updatePlayerProfile.mock.calls[0][1];
|
const playerProfileUpdate = updatePlayerProfile.mock.calls[0][1];
|
||||||
expect(playerProfileUpdate).toMatchObject({
|
expect(playerProfileUpdate).toMatchObject({
|
||||||
@@ -650,17 +689,15 @@ describe('AnalysisPipelineService', () => {
|
|||||||
normalizationMethod: 'population-percentile',
|
normalizationMethod: 'population-percentile',
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
const { service, updatePlayerMatchFeature, updatePlayerProfile } =
|
const { bulkWritePlayerMatchFeature, service, updatePlayerProfile } =
|
||||||
createPipelineHarness({
|
createPipelineHarness({
|
||||||
normalizeFeature,
|
normalizeFeature,
|
||||||
});
|
});
|
||||||
|
|
||||||
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
||||||
|
|
||||||
expect(normalizeFeature).toHaveBeenCalledTimes(1);
|
expect(normalizeFeature).toHaveBeenCalledTimes(3);
|
||||||
const featureSet = asRecord(
|
const featureSet = playerMatchFeatureBulkSet(bulkWritePlayerMatchFeature);
|
||||||
asRecord(updatePlayerMatchFeature.mock.calls[0]?.[1]).$set,
|
|
||||||
);
|
|
||||||
expect(asRecord(featureSet.normalized).kda).toBe(0.42);
|
expect(asRecord(featureSet.normalized).kda).toBe(0.42);
|
||||||
expect(featureSet.normalizationMethod).toBe('population-percentile');
|
expect(featureSet.normalizationMethod).toBe('population-percentile');
|
||||||
|
|
||||||
@@ -787,7 +824,7 @@ describe('AnalysisPipelineService', () => {
|
|||||||
getMatch,
|
getMatch,
|
||||||
getMatchTimeline,
|
getMatchTimeline,
|
||||||
service,
|
service,
|
||||||
updatePlayerMatchFeature,
|
bulkWritePlayerMatchFeature,
|
||||||
updateRiotMatch,
|
updateRiotMatch,
|
||||||
} = createPipelineHarness({
|
} = createPipelineHarness({
|
||||||
existingMatchRaw: createMatchPayload(),
|
existingMatchRaw: createMatchPayload(),
|
||||||
@@ -798,41 +835,88 @@ describe('AnalysisPipelineService', () => {
|
|||||||
|
|
||||||
expect(getMatch).not.toHaveBeenCalled();
|
expect(getMatch).not.toHaveBeenCalled();
|
||||||
expect(getMatchTimeline).not.toHaveBeenCalled();
|
expect(getMatchTimeline).not.toHaveBeenCalled();
|
||||||
expect(
|
expect(findTimelinePersistUpdate(updateRiotMatch)).toBeUndefined();
|
||||||
updateRiotMatch.mock.calls.find(([, update]) =>
|
expect(findTimelineDiscardUpdate(updateRiotMatch)).toBeDefined();
|
||||||
Boolean(asRecord(asRecord(update).$set).timeline),
|
expect(bulkWritePlayerMatchFeature).toHaveBeenCalledTimes(1);
|
||||||
),
|
|
||||||
).toBeUndefined();
|
|
||||||
expect(updatePlayerMatchFeature).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('fetches a missing timeline for cached match details', async () => {
|
it('uses a current precomputed target feature without fetching match details or timeline', async () => {
|
||||||
const { getMatch, getMatchTimeline, service, updateRiotMatch } =
|
const existingFeature = createStoredFeature({
|
||||||
createPipelineHarness({
|
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
|
||||||
existingMatchRaw: createMatchPayload(),
|
normalized: { kda: 1 },
|
||||||
});
|
vector: VECTOR_METRICS.map((metric) => (metric.id === 'kda' ? 1 : 0)),
|
||||||
|
timelineProcessed: true,
|
||||||
|
});
|
||||||
|
const {
|
||||||
|
bulkWritePlayerMatchFeature,
|
||||||
|
findOneRiotMatch,
|
||||||
|
findPlayerMatchFeatures,
|
||||||
|
getMatch,
|
||||||
|
getMatchTimeline,
|
||||||
|
service,
|
||||||
|
updatePlayerProfile,
|
||||||
|
} = createPipelineHarness({
|
||||||
|
existingTargetFeatures: [existingFeature],
|
||||||
|
});
|
||||||
|
|
||||||
|
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
||||||
|
|
||||||
|
expect(findPlayerMatchFeatures).toHaveBeenCalledWith({
|
||||||
|
matchId: { $in: ['KR_1'] },
|
||||||
|
puuid: 'target-puuid',
|
||||||
|
queueId: 420,
|
||||||
|
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
|
||||||
|
timelineProcessed: true,
|
||||||
|
});
|
||||||
|
expect(findOneRiotMatch).not.toHaveBeenCalled();
|
||||||
|
expect(getMatch).not.toHaveBeenCalled();
|
||||||
|
expect(getMatchTimeline).not.toHaveBeenCalled();
|
||||||
|
expect(bulkWritePlayerMatchFeature).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
const playerProfileUpdate = updatePlayerProfile.mock.calls[0][1];
|
||||||
|
expect(playerProfileUpdate).toMatchObject({
|
||||||
|
$set: {
|
||||||
|
puuid: 'target-puuid',
|
||||||
|
sampleGameCount: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fetches but does not persist a missing timeline for cached match details', async () => {
|
||||||
|
const {
|
||||||
|
bulkWritePlayerMatchFeature,
|
||||||
|
getMatch,
|
||||||
|
getMatchTimeline,
|
||||||
|
service,
|
||||||
|
updateRiotMatch,
|
||||||
|
} = createPipelineHarness({
|
||||||
|
existingMatchRaw: createMatchPayload(),
|
||||||
|
});
|
||||||
|
|
||||||
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
||||||
|
|
||||||
expect(getMatch).not.toHaveBeenCalled();
|
expect(getMatch).not.toHaveBeenCalled();
|
||||||
expect(getMatchTimeline).toHaveBeenCalledWith('KR_1', 'asia');
|
expect(getMatchTimeline).toHaveBeenCalledWith('KR_1', 'asia');
|
||||||
const timelineSet = findTimelineSet(updateRiotMatch);
|
expect(findTimelinePersistUpdate(updateRiotMatch)).toBeUndefined();
|
||||||
expect(timelineSet.timeline).toEqual(createTimelinePayload());
|
expect(bulkWritePlayerMatchFeature).toHaveBeenCalledTimes(1);
|
||||||
expect(timelineSet.timelineFetchedAt).toBeInstanceOf(Date);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('refetches cached timeline data when it is malformed', async () => {
|
it('refetches cached timeline data when it is malformed', async () => {
|
||||||
const { getMatchTimeline, service, updateRiotMatch } =
|
const {
|
||||||
createPipelineHarness({
|
bulkWritePlayerMatchFeature,
|
||||||
existingMatchRaw: createMatchPayload(),
|
getMatchTimeline,
|
||||||
existingTimeline: {},
|
service,
|
||||||
});
|
updateRiotMatch,
|
||||||
|
} = createPipelineHarness({
|
||||||
|
existingMatchRaw: createMatchPayload(),
|
||||||
|
existingTimeline: {},
|
||||||
|
});
|
||||||
|
|
||||||
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
||||||
|
|
||||||
expect(getMatchTimeline).toHaveBeenCalledWith('KR_1', 'asia');
|
expect(getMatchTimeline).toHaveBeenCalledWith('KR_1', 'asia');
|
||||||
const timelineSet = findTimelineSet(updateRiotMatch);
|
expect(findTimelinePersistUpdate(updateRiotMatch)).toBeUndefined();
|
||||||
expect(timelineSet.timeline).toEqual(createTimelinePayload());
|
expect(bulkWritePlayerMatchFeature).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('stores derived timeline data when a timeline is available', async () => {
|
it('stores derived timeline data when a timeline is available', async () => {
|
||||||
@@ -894,15 +978,15 @@ describe('AnalysisPipelineService', () => {
|
|||||||
const getMatchTimeline = jest
|
const getMatchTimeline = jest
|
||||||
.fn<Promise<Record<string, unknown>>, [string, string]>()
|
.fn<Promise<Record<string, unknown>>, [string, string]>()
|
||||||
.mockResolvedValue(timeline);
|
.mockResolvedValue(timeline);
|
||||||
const { service, updatePlayerMatchFeature, updatePlayerProfile } =
|
const { bulkWritePlayerMatchFeature, service, updatePlayerProfile } =
|
||||||
createPipelineHarness({
|
createPipelineHarness({
|
||||||
getMatchTimeline,
|
getMatchTimeline,
|
||||||
});
|
});
|
||||||
|
|
||||||
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
await service.processUserAnalysisJob(createJobData(), 'analysis-job-1');
|
||||||
|
|
||||||
const playerMatchFeatureSet = asRecord(
|
const playerMatchFeatureSet = playerMatchFeatureBulkSet(
|
||||||
asRecord(updatePlayerMatchFeature.mock.calls[0][1]).$set,
|
bulkWritePlayerMatchFeature,
|
||||||
);
|
);
|
||||||
const playerMatchFeatureSetFeatures = asRecord(
|
const playerMatchFeatureSetFeatures = asRecord(
|
||||||
playerMatchFeatureSet.features,
|
playerMatchFeatureSet.features,
|
||||||
|
|||||||
@@ -41,9 +41,15 @@ import { TimelineMetricService } from './timeline-metric.service';
|
|||||||
interface MatchRecord {
|
interface MatchRecord {
|
||||||
matchId: string;
|
matchId: string;
|
||||||
raw: Record<string, unknown>;
|
raw: Record<string, unknown>;
|
||||||
|
shouldDiscardStoredTimeline?: boolean;
|
||||||
timeline?: Record<string, unknown>;
|
timeline?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface MatchFetchProgressOptions {
|
||||||
|
initialFetchedMatches?: number;
|
||||||
|
totalMatches?: number;
|
||||||
|
}
|
||||||
|
|
||||||
interface ParsedMatchSummary {
|
interface ParsedMatchSummary {
|
||||||
gameCreation: number;
|
gameCreation: number;
|
||||||
gameDuration: number;
|
gameDuration: number;
|
||||||
@@ -155,10 +161,32 @@ export class AnalysisPipelineService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const matchRecords = await this.fetchAndStoreMatches(
|
const reusableFeaturesByMatchId = await this.findReusableMatchFeatures({
|
||||||
matchIds,
|
matchIds,
|
||||||
|
puuid: account.puuid,
|
||||||
|
queueId: normalizedData.queueId,
|
||||||
|
});
|
||||||
|
const matchIdsRequiringFeatureExtraction = matchIds.filter(
|
||||||
|
(matchId) => !reusableFeaturesByMatchId.has(matchId),
|
||||||
|
);
|
||||||
|
if (matchIdsRequiringFeatureExtraction.length === 0) {
|
||||||
|
await this.updateJob(jobId, {
|
||||||
|
progress: {
|
||||||
|
fetchedMatches: matchIds.length,
|
||||||
|
totalMatches: matchIds.length,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const matchRecords = await this.fetchAndStoreMatches(
|
||||||
|
matchIdsRequiringFeatureExtraction,
|
||||||
normalizedData,
|
normalizedData,
|
||||||
jobId,
|
jobId,
|
||||||
|
USER_ANALYSIS_JOB_NAME,
|
||||||
|
{
|
||||||
|
initialFetchedMatches: reusableFeaturesByMatchId.size,
|
||||||
|
totalMatches: matchIds.length,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
await this.updateJob(jobId, {
|
await this.updateJob(jobId, {
|
||||||
stage: 'extract-features',
|
stage: 'extract-features',
|
||||||
@@ -169,15 +197,20 @@ export class AnalysisPipelineService {
|
|||||||
account.puuid,
|
account.puuid,
|
||||||
normalizedData.queueId,
|
normalizedData.queueId,
|
||||||
);
|
);
|
||||||
|
const profileFeatures = this.mergeProfileFeatures(
|
||||||
|
matchIds,
|
||||||
|
reusableFeaturesByMatchId,
|
||||||
|
extractedFeatures,
|
||||||
|
);
|
||||||
await this.updateJob(jobId, {
|
await this.updateJob(jobId, {
|
||||||
stage: 'build-profile',
|
stage: 'build-profile',
|
||||||
});
|
});
|
||||||
|
|
||||||
if (extractedFeatures.length > 0) {
|
if (profileFeatures.length > 0) {
|
||||||
await this.buildAndStoreProfile(
|
await this.buildAndStoreProfile(
|
||||||
account,
|
account,
|
||||||
normalizedData,
|
normalizedData,
|
||||||
extractedFeatures,
|
profileFeatures,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -405,11 +438,36 @@ export class AnalysisPipelineService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const matchRecords = await this.fetchAndStoreMatches(
|
const reusableFeaturesByMatchId = await this.findReusableMatchFeatures({
|
||||||
matchIds,
|
matchIds,
|
||||||
|
puuid: account.puuid,
|
||||||
|
queueId: normalizedData.queueId,
|
||||||
|
});
|
||||||
|
const matchIdsRequiringFeatureExtraction = matchIds.filter(
|
||||||
|
(matchId) => !reusableFeaturesByMatchId.has(matchId),
|
||||||
|
);
|
||||||
|
if (matchIdsRequiringFeatureExtraction.length === 0) {
|
||||||
|
await this.updateJob(
|
||||||
|
jobId,
|
||||||
|
{
|
||||||
|
progress: {
|
||||||
|
fetchedMatches: matchIds.length,
|
||||||
|
totalMatches: matchIds.length,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
SEED_COLLECTION_JOB_NAME,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const matchRecords = await this.fetchAndStoreMatches(
|
||||||
|
matchIdsRequiringFeatureExtraction,
|
||||||
normalizedData,
|
normalizedData,
|
||||||
jobId,
|
jobId,
|
||||||
SEED_COLLECTION_JOB_NAME,
|
SEED_COLLECTION_JOB_NAME,
|
||||||
|
{
|
||||||
|
initialFetchedMatches: reusableFeaturesByMatchId.size,
|
||||||
|
totalMatches: matchIds.length,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
await this.updateJob(
|
await this.updateJob(
|
||||||
jobId,
|
jobId,
|
||||||
@@ -424,6 +482,11 @@ export class AnalysisPipelineService {
|
|||||||
account.puuid,
|
account.puuid,
|
||||||
normalizedData.queueId,
|
normalizedData.queueId,
|
||||||
);
|
);
|
||||||
|
const profileFeatures = this.mergeProfileFeatures(
|
||||||
|
matchIds,
|
||||||
|
reusableFeaturesByMatchId,
|
||||||
|
extractedFeatures,
|
||||||
|
);
|
||||||
await this.updateJob(
|
await this.updateJob(
|
||||||
jobId,
|
jobId,
|
||||||
{
|
{
|
||||||
@@ -432,19 +495,15 @@ export class AnalysisPipelineService {
|
|||||||
SEED_COLLECTION_JOB_NAME,
|
SEED_COLLECTION_JOB_NAME,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (extractedFeatures.length > 0) {
|
if (profileFeatures.length > 0) {
|
||||||
await this.buildAndStoreProfile(
|
await this.buildAndStoreProfile(account, normalizedData, profileFeatures);
|
||||||
account,
|
|
||||||
normalizedData,
|
|
||||||
extractedFeatures,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
puuid: account.puuid,
|
puuid: account.puuid,
|
||||||
fetchedMatches: matchRecords.length,
|
fetchedMatches: matchRecords.length,
|
||||||
extractedFeatures: extractedFeatures.length,
|
extractedFeatures: extractedFeatures.length,
|
||||||
profileBuilt: extractedFeatures.length > 0,
|
profileBuilt: profileFeatures.length > 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -500,14 +559,72 @@ export class AnalysisPipelineService {
|
|||||||
return storedAccount;
|
return storedAccount;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async findReusableMatchFeatures(params: {
|
||||||
|
matchIds: string[];
|
||||||
|
puuid: string;
|
||||||
|
queueId: number;
|
||||||
|
}): Promise<Map<string, PlayerMatchFeature>> {
|
||||||
|
if (params.matchIds.length === 0) {
|
||||||
|
return new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
const playerMatchFeatureModel =
|
||||||
|
await this.connectionService.getModel('playerMatchFeature');
|
||||||
|
const features = await playerMatchFeatureModel
|
||||||
|
.find({
|
||||||
|
matchId: { $in: params.matchIds },
|
||||||
|
puuid: params.puuid,
|
||||||
|
queueId: params.queueId,
|
||||||
|
featureSchemaVersion: FEATURE_SCHEMA_VERSION,
|
||||||
|
timelineProcessed: true,
|
||||||
|
})
|
||||||
|
.lean<PlayerMatchFeature[]>()
|
||||||
|
.exec();
|
||||||
|
|
||||||
|
return new Map(
|
||||||
|
features.map((feature) => [feature.matchId, feature] as const),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private mergeProfileFeatures(
|
||||||
|
matchIds: string[],
|
||||||
|
reusableFeaturesByMatchId: Map<string, PlayerMatchFeature>,
|
||||||
|
extractedFeatures: ExtractedPlayerMatchFeature[],
|
||||||
|
): Array<PlayerMatchFeature | ExtractedPlayerMatchFeature> {
|
||||||
|
const extractedFeaturesByMatchId = new Map(
|
||||||
|
extractedFeatures.map((feature) => [feature.matchId, feature] as const),
|
||||||
|
);
|
||||||
|
const profileFeatures: Array<
|
||||||
|
PlayerMatchFeature | ExtractedPlayerMatchFeature
|
||||||
|
> = [];
|
||||||
|
|
||||||
|
for (const matchId of matchIds) {
|
||||||
|
const reusableFeature = reusableFeaturesByMatchId.get(matchId);
|
||||||
|
if (reusableFeature) {
|
||||||
|
profileFeatures.push(reusableFeature);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const extractedFeature = extractedFeaturesByMatchId.get(matchId);
|
||||||
|
if (extractedFeature) {
|
||||||
|
profileFeatures.push(extractedFeature);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return profileFeatures;
|
||||||
|
}
|
||||||
|
|
||||||
private async fetchAndStoreMatches(
|
private async fetchAndStoreMatches(
|
||||||
matchIds: string[],
|
matchIds: string[],
|
||||||
data: Pick<UserAnalysisJobData, 'queueId' | 'regionalRoute'>,
|
data: Pick<UserAnalysisJobData, 'queueId' | 'regionalRoute'>,
|
||||||
jobId: string,
|
jobId: string,
|
||||||
jobType = USER_ANALYSIS_JOB_NAME,
|
jobType = USER_ANALYSIS_JOB_NAME,
|
||||||
|
progressOptions: MatchFetchProgressOptions = {},
|
||||||
): Promise<MatchRecord[]> {
|
): Promise<MatchRecord[]> {
|
||||||
const matchRecords: MatchRecord[] = [];
|
const matchRecords: MatchRecord[] = [];
|
||||||
const riotMatchModel = await this.connectionService.getModel('riotMatch');
|
const riotMatchModel = await this.connectionService.getModel('riotMatch');
|
||||||
|
const totalMatches = progressOptions.totalMatches ?? matchIds.length;
|
||||||
|
const initialFetchedMatches = progressOptions.initialFetchedMatches ?? 0;
|
||||||
|
|
||||||
for (const [index, matchId] of matchIds.entries()) {
|
for (const [index, matchId] of matchIds.entries()) {
|
||||||
try {
|
try {
|
||||||
@@ -518,7 +635,7 @@ export class AnalysisPipelineService {
|
|||||||
|
|
||||||
if (existingMatch) {
|
if (existingMatch) {
|
||||||
if (this.isEligibleMatch(existingMatch.raw, data.queueId)) {
|
if (this.isEligibleMatch(existingMatch.raw, data.queueId)) {
|
||||||
const timeline = await this.fetchAndStoreMissingTimeline(
|
const timeline = await this.fetchMissingTimeline(
|
||||||
matchId,
|
matchId,
|
||||||
existingMatch.timeline,
|
existingMatch.timeline,
|
||||||
data.regionalRoute,
|
data.regionalRoute,
|
||||||
@@ -526,6 +643,7 @@ export class AnalysisPipelineService {
|
|||||||
matchRecords.push({
|
matchRecords.push({
|
||||||
matchId,
|
matchId,
|
||||||
raw: existingMatch.raw,
|
raw: existingMatch.raw,
|
||||||
|
shouldDiscardStoredTimeline: existingMatch.timeline !== undefined,
|
||||||
...(timeline ? { timeline } : {}),
|
...(timeline ? { timeline } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -536,7 +654,7 @@ export class AnalysisPipelineService {
|
|||||||
);
|
);
|
||||||
await this.storeRiotMatch(matchId, rawMatch);
|
await this.storeRiotMatch(matchId, rawMatch);
|
||||||
if (this.isEligibleMatch(rawMatch, data.queueId)) {
|
if (this.isEligibleMatch(rawMatch, data.queueId)) {
|
||||||
const timeline = await this.fetchAndStoreMissingTimeline(
|
const timeline = await this.fetchMissingTimeline(
|
||||||
matchId,
|
matchId,
|
||||||
undefined,
|
undefined,
|
||||||
data.regionalRoute,
|
data.regionalRoute,
|
||||||
@@ -557,8 +675,8 @@ export class AnalysisPipelineService {
|
|||||||
jobId,
|
jobId,
|
||||||
{
|
{
|
||||||
progress: {
|
progress: {
|
||||||
fetchedMatches: index + 1,
|
fetchedMatches: initialFetchedMatches + index + 1,
|
||||||
totalMatches: matchIds.length,
|
totalMatches,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
jobType,
|
jobType,
|
||||||
@@ -596,7 +714,7 @@ export class AnalysisPipelineService {
|
|||||||
.exec();
|
.exec();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async fetchAndStoreMissingTimeline(
|
private async fetchMissingTimeline(
|
||||||
matchId: string,
|
matchId: string,
|
||||||
existingTimeline: unknown,
|
existingTimeline: unknown,
|
||||||
regionalRoute: UserAnalysisJobData['regionalRoute'],
|
regionalRoute: UserAnalysisJobData['regionalRoute'],
|
||||||
@@ -616,7 +734,6 @@ export class AnalysisPipelineService {
|
|||||||
);
|
);
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
await this.storeRiotMatchTimeline(matchId, timeline);
|
|
||||||
return timeline;
|
return timeline;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
@@ -626,19 +743,16 @@ export class AnalysisPipelineService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async storeRiotMatchTimeline(
|
private async discardStoredMatchTimeline(matchId: string): Promise<void> {
|
||||||
matchId: string,
|
|
||||||
timeline: Record<string, unknown>,
|
|
||||||
): Promise<void> {
|
|
||||||
const riotMatchModel = await this.connectionService.getModel('riotMatch');
|
const riotMatchModel = await this.connectionService.getModel('riotMatch');
|
||||||
|
|
||||||
await riotMatchModel
|
await riotMatchModel
|
||||||
.updateOne(
|
.updateOne(
|
||||||
{ matchId },
|
{ matchId, timeline: { $exists: true } },
|
||||||
{
|
{
|
||||||
$set: {
|
$unset: {
|
||||||
timeline,
|
timeline: '',
|
||||||
timelineFetchedAt: new Date(),
|
timelineFetchedAt: '',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -650,22 +764,62 @@ export class AnalysisPipelineService {
|
|||||||
puuid: string,
|
puuid: string,
|
||||||
queueId: number,
|
queueId: number,
|
||||||
): Promise<ExtractedPlayerMatchFeature[]> {
|
): Promise<ExtractedPlayerMatchFeature[]> {
|
||||||
const playerMatchFeatureModel =
|
|
||||||
await this.connectionService.getModel('playerMatchFeature');
|
|
||||||
const extractedFeatures: ExtractedPlayerMatchFeature[] = [];
|
const extractedFeatures: ExtractedPlayerMatchFeature[] = [];
|
||||||
|
|
||||||
for (const matchRecord of matchRecords) {
|
for (const matchRecord of matchRecords) {
|
||||||
|
try {
|
||||||
|
const participantPuuids = matchRecord.timeline
|
||||||
|
? this.resolveMatchParticipantPuuids(matchRecord.raw)
|
||||||
|
: [puuid];
|
||||||
|
const matchFeatures = await this.extractMatchParticipantFeatures(
|
||||||
|
matchRecord,
|
||||||
|
participantPuuids,
|
||||||
|
queueId,
|
||||||
|
);
|
||||||
|
await this.storePlayerMatchFeatures(matchFeatures);
|
||||||
|
if (matchRecord.shouldDiscardStoredTimeline === true) {
|
||||||
|
await this.discardStoredMatchTimeline(matchRecord.matchId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const targetFeature = matchFeatures.find(
|
||||||
|
(feature) => feature.puuid === puuid,
|
||||||
|
);
|
||||||
|
if (targetFeature) {
|
||||||
|
extractedFeatures.push(targetFeature);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Skipping feature extraction for matchId="${this.safeLogText(matchRecord.matchId)}": ${this.safeLogText(errorMessage(error))}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return extractedFeatures;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async extractMatchParticipantFeatures(
|
||||||
|
matchRecord: MatchRecord,
|
||||||
|
participantPuuids: string[],
|
||||||
|
queueId: number,
|
||||||
|
): Promise<ExtractedPlayerMatchFeature[]> {
|
||||||
|
const features: ExtractedPlayerMatchFeature[] = [];
|
||||||
|
const uniqueParticipantPuuids = [...new Set(participantPuuids)];
|
||||||
|
const derived = matchRecord.timeline
|
||||||
|
? this.timelineDerivedService.buildDerivedData(
|
||||||
|
matchRecord.timeline,
|
||||||
|
matchRecord.raw,
|
||||||
|
)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
for (const participantPuuid of uniqueParticipantPuuids) {
|
||||||
try {
|
try {
|
||||||
const feature = this.featureExtractorService.extractPlayerMatchFeature(
|
const feature = this.featureExtractorService.extractPlayerMatchFeature(
|
||||||
matchRecord.raw,
|
matchRecord.raw,
|
||||||
puuid,
|
participantPuuid,
|
||||||
);
|
);
|
||||||
if (feature.queueId !== queueId) continue;
|
if (feature.queueId !== queueId) continue;
|
||||||
if (matchRecord.timeline) {
|
|
||||||
const derived = this.timelineDerivedService.buildDerivedData(
|
if (matchRecord.timeline && derived) {
|
||||||
matchRecord.timeline,
|
|
||||||
matchRecord.raw,
|
|
||||||
);
|
|
||||||
const timelineMetrics =
|
const timelineMetrics =
|
||||||
this.timelineMetricService.extractTimelineMetrics(
|
this.timelineMetricService.extractTimelineMetrics(
|
||||||
feature,
|
feature,
|
||||||
@@ -688,27 +842,57 @@ export class AnalysisPipelineService {
|
|||||||
feature.timelineProcessed = true;
|
feature.timelineProcessed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
const normalizedFeature = await this.normalizeExtractedFeature(feature);
|
features.push(await this.normalizeExtractedFeature(feature));
|
||||||
|
|
||||||
await playerMatchFeatureModel
|
|
||||||
.updateOne(
|
|
||||||
{
|
|
||||||
matchId: normalizedFeature.matchId,
|
|
||||||
puuid: normalizedFeature.puuid,
|
|
||||||
},
|
|
||||||
{ $set: normalizedFeature },
|
|
||||||
{ upsert: true },
|
|
||||||
)
|
|
||||||
.exec();
|
|
||||||
extractedFeatures.push(normalizedFeature);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`Skipping feature extraction for matchId="${this.safeLogText(matchRecord.matchId)}": ${this.safeLogText(errorMessage(error))}`,
|
`Skipping feature extraction for matchId="${this.safeLogText(matchRecord.matchId)}" puuid="${this.safeLogText(participantPuuid)}": ${this.safeLogText(errorMessage(error))}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return extractedFeatures;
|
return features;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async storePlayerMatchFeatures(
|
||||||
|
features: ExtractedPlayerMatchFeature[],
|
||||||
|
): Promise<void> {
|
||||||
|
if (features.length === 0) return;
|
||||||
|
|
||||||
|
const playerMatchFeatureModel =
|
||||||
|
await this.connectionService.getModel('playerMatchFeature');
|
||||||
|
await playerMatchFeatureModel.bulkWrite(
|
||||||
|
features.map((feature) => ({
|
||||||
|
updateOne: {
|
||||||
|
filter: {
|
||||||
|
matchId: feature.matchId,
|
||||||
|
puuid: feature.puuid,
|
||||||
|
},
|
||||||
|
update: {
|
||||||
|
$set: feature,
|
||||||
|
},
|
||||||
|
upsert: true,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveMatchParticipantPuuids(
|
||||||
|
rawMatch: Record<string, unknown>,
|
||||||
|
): string[] {
|
||||||
|
const info = asRecord(rawMatch.info);
|
||||||
|
const participants = Array.isArray(info.participants)
|
||||||
|
? info.participants
|
||||||
|
: [];
|
||||||
|
const participantPuuids = participants
|
||||||
|
.map((participant) => asString(asRecord(participant).puuid))
|
||||||
|
.filter((participantPuuid): participantPuuid is string =>
|
||||||
|
Boolean(participantPuuid),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (participantPuuids.length > 0) {
|
||||||
|
return participantPuuids;
|
||||||
|
}
|
||||||
|
return this.parseMatchSummary(rawMatch).participantPuuids;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async normalizeExtractedFeature(
|
private async normalizeExtractedFeature(
|
||||||
@@ -863,10 +1047,10 @@ export class AnalysisPipelineService {
|
|||||||
private async buildAndStoreProfile(
|
private async buildAndStoreProfile(
|
||||||
account: Account,
|
account: Account,
|
||||||
data: Pick<UserAnalysisJobData, 'queueId'>,
|
data: Pick<UserAnalysisJobData, 'queueId'>,
|
||||||
extractedFeatures: ExtractedPlayerMatchFeature[],
|
profileFeatures: Array<PlayerMatchFeature | ExtractedPlayerMatchFeature>,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await this.storeBuiltProfile({
|
await this.storeBuiltProfile({
|
||||||
features: extractedFeatures,
|
features: profileFeatures,
|
||||||
isSeed: account.isSeed,
|
isSeed: account.isSeed,
|
||||||
tierInfo: this.resolveRankedSoloTier(account),
|
tierInfo: this.resolveRankedSoloTier(account),
|
||||||
puuid: account.puuid,
|
puuid: account.puuid,
|
||||||
@@ -929,7 +1113,7 @@ export class AnalysisPipelineService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async storeBuiltProfile(params: {
|
private async storeBuiltProfile(params: {
|
||||||
features: PlayerMatchFeature[] | ExtractedPlayerMatchFeature[];
|
features: Array<PlayerMatchFeature | ExtractedPlayerMatchFeature>;
|
||||||
isSeed: boolean;
|
isSeed: boolean;
|
||||||
tierInfo?: TierInfo;
|
tierInfo?: TierInfo;
|
||||||
puuid: string;
|
puuid: string;
|
||||||
|
|||||||
Reference in new issue
Block a user