Wire seed collection queue
This commit is contained in:
1 parent
a6c1acbc31
commit
63a4aeace7
8 files changed
+608
-3
No files matched your search
@@ -2,7 +2,10 @@ import { getQueueToken } from '@nestjs/bullmq';
|
|||||||
import { Test } from '@nestjs/testing';
|
import { Test } from '@nestjs/testing';
|
||||||
import { AppModule } from './app.module';
|
import { AppModule } from './app.module';
|
||||||
import { AnalysisQueueService } from './modules/analysis/analysis-queue.service';
|
import { AnalysisQueueService } from './modules/analysis/analysis-queue.service';
|
||||||
import { ANALYSIS_QUEUE_NAME } from './modules/analysis/analysis.constants';
|
import {
|
||||||
|
ANALYSIS_QUEUE_NAME,
|
||||||
|
SEED_COLLECTION_QUEUE_NAME,
|
||||||
|
} from './modules/analysis/analysis.constants';
|
||||||
import { ConnectionService } from './modules/global/database/connection.service';
|
import { ConnectionService } from './modules/global/database/connection.service';
|
||||||
import { REDIS_CLIENT } from './modules/global/redis/redis.constants';
|
import { REDIS_CLIENT } from './modules/global/redis/redis.constants';
|
||||||
import { SecretService } from './modules/global/secret/secret.service';
|
import { SecretService } from './modules/global/secret/secret.service';
|
||||||
@@ -36,6 +39,10 @@ describe('AppModule', () => {
|
|||||||
.useValue({
|
.useValue({
|
||||||
add: jest.fn(),
|
add: jest.fn(),
|
||||||
})
|
})
|
||||||
|
.overrideProvider(getQueueToken(SEED_COLLECTION_QUEUE_NAME))
|
||||||
|
.useValue({
|
||||||
|
add: jest.fn(),
|
||||||
|
})
|
||||||
.compile();
|
.compile();
|
||||||
|
|
||||||
expect(moduleRef.get(AnalysisQueueService)).toBeInstanceOf(
|
expect(moduleRef.get(AnalysisQueueService)).toBeInstanceOf(
|
||||||
|
|||||||
@@ -1,8 +1,13 @@
|
|||||||
import type { AnalysisJob } from '../global/database/schemas';
|
import type { AnalysisJob } from '../global/database/schemas';
|
||||||
import type { ConnectionService } from '../global/database/connection.service';
|
import type { ConnectionService } from '../global/database/connection.service';
|
||||||
import type { UserAnalysisJobData } from './analysis.types';
|
import type {
|
||||||
|
SeedCollectionJobData,
|
||||||
|
UserAnalysisJobData,
|
||||||
|
} from './analysis.types';
|
||||||
import {
|
import {
|
||||||
|
DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||||
FEATURE_RECOMPUTE_JOB_NAME,
|
FEATURE_RECOMPUTE_JOB_NAME,
|
||||||
|
SEED_COLLECTION_JOB_NAME,
|
||||||
USER_ANALYSIS_JOB_NAME,
|
USER_ANALYSIS_JOB_NAME,
|
||||||
} from './analysis.constants';
|
} from './analysis.constants';
|
||||||
import { AnalysisQueueService } from './analysis-queue.service';
|
import { AnalysisQueueService } from './analysis-queue.service';
|
||||||
@@ -37,9 +42,25 @@ function createFindOneChain(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createSeedCollectionJobData(
|
||||||
|
overrides: Partial<SeedCollectionJobData> = {},
|
||||||
|
): SeedCollectionJobData {
|
||||||
|
return {
|
||||||
|
platformRoute: 'kr',
|
||||||
|
queueId: 420,
|
||||||
|
sourceLimits: {
|
||||||
|
challenger: 50,
|
||||||
|
grandmaster: 50,
|
||||||
|
master: 50,
|
||||||
|
},
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe('AnalysisQueueService', () => {
|
describe('AnalysisQueueService', () => {
|
||||||
it('creates an analysis job document and enqueues a BullMQ job', async () => {
|
it('creates an analysis job document and enqueues a BullMQ job', async () => {
|
||||||
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
||||||
|
const seedAdd = jest.fn();
|
||||||
const findOneChain = createFindOneChain(null);
|
const findOneChain = createFindOneChain(null);
|
||||||
const create = jest.fn().mockResolvedValue({});
|
const create = jest.fn().mockResolvedValue({});
|
||||||
const updateOne = jest.fn(() => ({
|
const updateOne = jest.fn(() => ({
|
||||||
@@ -55,6 +76,7 @@ describe('AnalysisQueueService', () => {
|
|||||||
} as unknown as ConnectionService;
|
} as unknown as ConnectionService;
|
||||||
const service = new AnalysisQueueService(
|
const service = new AnalysisQueueService(
|
||||||
{ add } as never,
|
{ add } as never,
|
||||||
|
{ add: seedAdd } as never,
|
||||||
connectionService,
|
connectionService,
|
||||||
() => 'analysis-job-1',
|
() => 'analysis-job-1',
|
||||||
);
|
);
|
||||||
@@ -79,10 +101,12 @@ describe('AnalysisQueueService', () => {
|
|||||||
jobId: 'analysis-job-1',
|
jobId: 'analysis-job-1',
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
expect(seedAdd).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('creates a feature recompute job document and enqueues a BullMQ job', async () => {
|
it('creates a feature recompute job document and enqueues a BullMQ job', async () => {
|
||||||
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
||||||
|
const seedAdd = jest.fn();
|
||||||
const create = jest.fn().mockResolvedValue({});
|
const create = jest.fn().mockResolvedValue({});
|
||||||
const updateOne = jest.fn(() => ({
|
const updateOne = jest.fn(() => ({
|
||||||
exec: jest.fn().mockResolvedValue({}),
|
exec: jest.fn().mockResolvedValue({}),
|
||||||
@@ -96,6 +120,7 @@ describe('AnalysisQueueService', () => {
|
|||||||
} as unknown as ConnectionService;
|
} as unknown as ConnectionService;
|
||||||
const service = new AnalysisQueueService(
|
const service = new AnalysisQueueService(
|
||||||
{ add } as never,
|
{ add } as never,
|
||||||
|
{ add: seedAdd } as never,
|
||||||
connectionService,
|
connectionService,
|
||||||
() => 'recompute-job-1',
|
() => 'recompute-job-1',
|
||||||
);
|
);
|
||||||
@@ -132,6 +157,54 @@ describe('AnalysisQueueService', () => {
|
|||||||
jobId: 'recompute-job-1',
|
jobId: 'recompute-job-1',
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
expect(seedAdd).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('creates a seed collection job document and enqueues the seed queue', async () => {
|
||||||
|
const analysisAdd = jest.fn();
|
||||||
|
const seedAdd = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
||||||
|
const create = jest.fn().mockResolvedValue({});
|
||||||
|
const updateOne = jest.fn(() => ({
|
||||||
|
exec: jest.fn().mockResolvedValue({}),
|
||||||
|
}));
|
||||||
|
const analysisJobModel = {
|
||||||
|
create,
|
||||||
|
updateOne,
|
||||||
|
};
|
||||||
|
const connectionService = {
|
||||||
|
getModel: jest.fn().mockResolvedValue(analysisJobModel),
|
||||||
|
} as unknown as ConnectionService;
|
||||||
|
const service = new AnalysisQueueService(
|
||||||
|
{ add: analysisAdd } as never,
|
||||||
|
{ add: seedAdd } as never,
|
||||||
|
connectionService,
|
||||||
|
() => 'seed-job-1',
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await service.enqueueSeedCollection(
|
||||||
|
createSeedCollectionJobData(),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toEqual({ jobId: 'seed-job-1', status: 'queued' });
|
||||||
|
expect(create).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
jobId: 'seed-job-1',
|
||||||
|
jobType: SEED_COLLECTION_JOB_NAME,
|
||||||
|
platformRoute: 'kr',
|
||||||
|
queueId: 420,
|
||||||
|
requestedMatchCount: DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||||
|
status: 'queued',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(seedAdd).toHaveBeenCalledWith(
|
||||||
|
SEED_COLLECTION_JOB_NAME,
|
||||||
|
createSeedCollectionJobData(),
|
||||||
|
expect.objectContaining({
|
||||||
|
attempts: 1,
|
||||||
|
jobId: 'seed-job-1',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(analysisAdd).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns an existing active job instead of enqueueing a duplicate', async () => {
|
it('returns an existing active job instead of enqueueing a duplicate', async () => {
|
||||||
@@ -149,6 +222,7 @@ describe('AnalysisQueueService', () => {
|
|||||||
} as unknown as ConnectionService;
|
} as unknown as ConnectionService;
|
||||||
const service = new AnalysisQueueService(
|
const service = new AnalysisQueueService(
|
||||||
{ add } as never,
|
{ add } as never,
|
||||||
|
{ add: jest.fn() } as never,
|
||||||
connectionService,
|
connectionService,
|
||||||
() => 'new-job',
|
() => 'new-job',
|
||||||
);
|
);
|
||||||
@@ -180,6 +254,7 @@ describe('AnalysisQueueService', () => {
|
|||||||
} as unknown as ConnectionService;
|
} as unknown as ConnectionService;
|
||||||
const service = new AnalysisQueueService(
|
const service = new AnalysisQueueService(
|
||||||
{ add } as never,
|
{ add } as never,
|
||||||
|
{ add: jest.fn() } as never,
|
||||||
connectionService,
|
connectionService,
|
||||||
() => 'analysis-job-1',
|
() => 'analysis-job-1',
|
||||||
);
|
);
|
||||||
@@ -214,6 +289,7 @@ describe('AnalysisQueueService', () => {
|
|||||||
} as unknown as ConnectionService;
|
} as unknown as ConnectionService;
|
||||||
const service = new AnalysisQueueService(
|
const service = new AnalysisQueueService(
|
||||||
{ add } as never,
|
{ add } as never,
|
||||||
|
{ add: jest.fn() } as never,
|
||||||
connectionService,
|
connectionService,
|
||||||
() => 'recompute-job-1',
|
() => 'recompute-job-1',
|
||||||
);
|
);
|
||||||
@@ -237,6 +313,43 @@ describe('AnalysisQueueService', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('marks a pre-created seed collection job failed when BullMQ enqueue fails', async () => {
|
||||||
|
const analysisAdd = jest.fn();
|
||||||
|
const seedAdd = jest.fn().mockRejectedValue(new Error('redis unavailable'));
|
||||||
|
const create = jest.fn().mockResolvedValue({});
|
||||||
|
const updateOne = jest.fn(() => ({
|
||||||
|
exec: jest.fn().mockResolvedValue({}),
|
||||||
|
}));
|
||||||
|
const analysisJobModel = {
|
||||||
|
create,
|
||||||
|
updateOne,
|
||||||
|
};
|
||||||
|
const connectionService = {
|
||||||
|
getModel: jest.fn().mockResolvedValue(analysisJobModel),
|
||||||
|
} as unknown as ConnectionService;
|
||||||
|
const service = new AnalysisQueueService(
|
||||||
|
{ add: analysisAdd } as never,
|
||||||
|
{ add: seedAdd } as never,
|
||||||
|
connectionService,
|
||||||
|
() => 'seed-job-1',
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.enqueueSeedCollection(createSeedCollectionJobData()),
|
||||||
|
).rejects.toThrow('redis unavailable');
|
||||||
|
expect(updateOne).toHaveBeenCalledWith(
|
||||||
|
{ jobId: 'seed-job-1' },
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
status: 'failed',
|
||||||
|
stage: 'enqueue',
|
||||||
|
error: 'Analysis could not be queued.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
expect(analysisAdd).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it('serializes identical in-process requests before creating a duplicate job', async () => {
|
it('serializes identical in-process requests before creating a duplicate job', async () => {
|
||||||
const add = jest.fn().mockResolvedValue({ id: 'analysis-job-1' });
|
const add = jest.fn().mockResolvedValue({ id: 'analysis-job-1' });
|
||||||
let activeJob: Pick<AnalysisJob, 'jobId' | 'status'> | null = null;
|
let activeJob: Pick<AnalysisJob, 'jobId' | 'status'> | null = null;
|
||||||
@@ -263,6 +376,7 @@ describe('AnalysisQueueService', () => {
|
|||||||
} as unknown as ConnectionService;
|
} as unknown as ConnectionService;
|
||||||
const service = new AnalysisQueueService(
|
const service = new AnalysisQueueService(
|
||||||
{ add } as never,
|
{ add } as never,
|
||||||
|
{ add: jest.fn() } as never,
|
||||||
connectionService,
|
connectionService,
|
||||||
() => (create.mock.calls.length === 0 ? 'analysis-job-1' : 'unexpected'),
|
() => (create.mock.calls.length === 0 ? 'analysis-job-1' : 'unexpected'),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -4,18 +4,22 @@ import type { AnalysisRequestAcceptedDto } from '@persona-lenz/shared';
|
|||||||
import type { Queue } from 'bullmq';
|
import type { Queue } from 'bullmq';
|
||||||
import { randomUUID } from 'node:crypto';
|
import { randomUUID } from 'node:crypto';
|
||||||
import { ConnectionService } from '../global/database/connection.service';
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
import type { AnalysisJob } from '../global/database/schemas';
|
import type { AnalysisJob, SeedSource } from '../global/database/schemas';
|
||||||
import {
|
import {
|
||||||
ANALYSIS_JOB_ID_FACTORY,
|
ANALYSIS_JOB_ID_FACTORY,
|
||||||
ACTIVE_ANALYSIS_JOB_STATUSES,
|
ACTIVE_ANALYSIS_JOB_STATUSES,
|
||||||
ANALYSIS_QUEUE_NAME,
|
ANALYSIS_QUEUE_NAME,
|
||||||
|
DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||||
FEATURE_RECOMPUTE_JOB_NAME,
|
FEATURE_RECOMPUTE_JOB_NAME,
|
||||||
normalizeFeatureRecomputeBatchSize,
|
normalizeFeatureRecomputeBatchSize,
|
||||||
normalizeRequestedMatchCount,
|
normalizeRequestedMatchCount,
|
||||||
|
SEED_COLLECTION_JOB_NAME,
|
||||||
|
SEED_COLLECTION_QUEUE_NAME,
|
||||||
USER_ANALYSIS_JOB_NAME,
|
USER_ANALYSIS_JOB_NAME,
|
||||||
} from './analysis.constants';
|
} from './analysis.constants';
|
||||||
import type {
|
import type {
|
||||||
FeatureRecomputeJobData,
|
FeatureRecomputeJobData,
|
||||||
|
SeedCollectionJobData,
|
||||||
UserAnalysisJobData,
|
UserAnalysisJobData,
|
||||||
} from './analysis.types';
|
} from './analysis.types';
|
||||||
|
|
||||||
@@ -23,6 +27,7 @@ type JobIdFactory = () => string;
|
|||||||
type AnalysisQueueJobData = UserAnalysisJobData | FeatureRecomputeJobData;
|
type AnalysisQueueJobData = UserAnalysisJobData | FeatureRecomputeJobData;
|
||||||
type AnalysisQueueJobName =
|
type AnalysisQueueJobName =
|
||||||
typeof USER_ANALYSIS_JOB_NAME | typeof FEATURE_RECOMPUTE_JOB_NAME;
|
typeof USER_ANALYSIS_JOB_NAME | typeof FEATURE_RECOMPUTE_JOB_NAME;
|
||||||
|
type SeedCollectionQueueJobName = typeof SEED_COLLECTION_JOB_NAME;
|
||||||
type ActiveAnalysisJob = Pick<AnalysisJob, 'jobId'> & {
|
type ActiveAnalysisJob = Pick<AnalysisJob, 'jobId'> & {
|
||||||
status: Extract<AnalysisJob['status'], 'queued' | 'processing'>;
|
status: Extract<AnalysisJob['status'], 'queued' | 'processing'>;
|
||||||
};
|
};
|
||||||
@@ -36,6 +41,12 @@ export class AnalysisQueueService {
|
|||||||
void,
|
void,
|
||||||
AnalysisQueueJobName
|
AnalysisQueueJobName
|
||||||
>,
|
>,
|
||||||
|
@InjectQueue(SEED_COLLECTION_QUEUE_NAME)
|
||||||
|
private readonly seedCollectionQueue: Queue<
|
||||||
|
SeedCollectionJobData,
|
||||||
|
void,
|
||||||
|
SeedCollectionQueueJobName
|
||||||
|
>,
|
||||||
private readonly connectionService: ConnectionService,
|
private readonly connectionService: ConnectionService,
|
||||||
@Inject(ANALYSIS_JOB_ID_FACTORY)
|
@Inject(ANALYSIS_JOB_ID_FACTORY)
|
||||||
private readonly jobIdFactory: JobIdFactory = randomUUID,
|
private readonly jobIdFactory: JobIdFactory = randomUUID,
|
||||||
@@ -115,6 +126,59 @@ export class AnalysisQueueService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async enqueueSeedCollection(
|
||||||
|
data: SeedCollectionJobData,
|
||||||
|
): Promise<AnalysisRequestAcceptedDto> {
|
||||||
|
const normalizedData = this.normalizeSeedCollectionData(data);
|
||||||
|
const analysisJobModel =
|
||||||
|
await this.connectionService.getModel('analysisJob');
|
||||||
|
const jobId = this.jobIdFactory();
|
||||||
|
|
||||||
|
await analysisJobModel.create({
|
||||||
|
jobId,
|
||||||
|
jobType: SEED_COLLECTION_JOB_NAME,
|
||||||
|
platformRoute: normalizedData.platformRoute,
|
||||||
|
queueId: normalizedData.queueId,
|
||||||
|
requestedMatchCount: DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||||
|
status: 'queued',
|
||||||
|
stage: 'queued',
|
||||||
|
progress: {
|
||||||
|
fetchedMatches: 0,
|
||||||
|
totalMatches: 0,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.seedCollectionQueue.add(
|
||||||
|
SEED_COLLECTION_JOB_NAME,
|
||||||
|
normalizedData,
|
||||||
|
{
|
||||||
|
attempts: 1,
|
||||||
|
jobId,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
await analysisJobModel
|
||||||
|
.updateOne(
|
||||||
|
{ jobId },
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
status: 'failed',
|
||||||
|
stage: 'enqueue',
|
||||||
|
error: this.publicErrorMessage(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.exec();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
jobId,
|
||||||
|
status: 'queued',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private readonly pendingDuplicateRequestKeys = new Set<string>();
|
private readonly pendingDuplicateRequestKeys = new Set<string>();
|
||||||
|
|
||||||
private async enqueueUserAnalysisWithoutDuplicateRequestRace(
|
private async enqueueUserAnalysisWithoutDuplicateRequestRace(
|
||||||
@@ -218,6 +282,30 @@ export class AnalysisQueueService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private normalizeSeedCollectionData(
|
||||||
|
data: SeedCollectionJobData,
|
||||||
|
): SeedCollectionJobData {
|
||||||
|
return {
|
||||||
|
platformRoute: data.platformRoute,
|
||||||
|
queueId: data.queueId,
|
||||||
|
...(data.sourceLimits
|
||||||
|
? { sourceLimits: this.normalizeSeedSourceLimits(data.sourceLimits) }
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeSeedSourceLimits(
|
||||||
|
sourceLimits: NonNullable<SeedCollectionJobData['sourceLimits']>,
|
||||||
|
): NonNullable<SeedCollectionJobData['sourceLimits']> {
|
||||||
|
const normalizedSourceLimits: Partial<Record<SeedSource, number>> = {};
|
||||||
|
for (const [source, limit] of Object.entries(sourceLimits)) {
|
||||||
|
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
|
||||||
|
normalizedSourceLimits[source as SeedSource] = Math.trunc(limit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return normalizedSourceLimits;
|
||||||
|
}
|
||||||
|
|
||||||
private duplicateRequestKey(data: UserAnalysisJobData): string {
|
private duplicateRequestKey(data: UserAnalysisJobData): string {
|
||||||
return [
|
return [
|
||||||
data.platformRoute,
|
data.platformRoute,
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
export const ANALYSIS_QUEUE_NAME = 'analysis';
|
export const ANALYSIS_QUEUE_NAME = 'analysis';
|
||||||
|
export const SEED_COLLECTION_QUEUE_NAME = 'seed-collection';
|
||||||
export const USER_ANALYSIS_JOB_NAME = 'user-analysis';
|
export const USER_ANALYSIS_JOB_NAME = 'user-analysis';
|
||||||
export const FEATURE_RECOMPUTE_JOB_NAME = 'feature-recompute';
|
export const FEATURE_RECOMPUTE_JOB_NAME = 'feature-recompute';
|
||||||
|
export const SEED_COLLECTION_JOB_NAME = 'seed-collection';
|
||||||
export const ANALYSIS_JOB_ID_FACTORY = Symbol('ANALYSIS_JOB_ID_FACTORY');
|
export const ANALYSIS_JOB_ID_FACTORY = Symbol('ANALYSIS_JOB_ID_FACTORY');
|
||||||
|
|
||||||
export const DEFAULT_ANALYSIS_MATCH_COUNT = 20;
|
export const DEFAULT_ANALYSIS_MATCH_COUNT = 20;
|
||||||
export const MAX_ANALYSIS_MATCH_COUNT = 50;
|
export const MAX_ANALYSIS_MATCH_COUNT = 50;
|
||||||
|
export const DEFAULT_SEED_COLLECTION_MATCH_COUNT = 30;
|
||||||
export const DEFAULT_SEED_COLLECTION_QUEUE_TYPE = 'RANKED_SOLO_5x5';
|
export const DEFAULT_SEED_COLLECTION_QUEUE_TYPE = 'RANKED_SOLO_5x5';
|
||||||
export const DEFAULT_SEED_COLLECTION_SOURCE_LIMIT = 50;
|
export const DEFAULT_SEED_COLLECTION_SOURCE_LIMIT = 50;
|
||||||
export const DEFAULT_FEATURE_RECOMPUTE_BATCH_SIZE = 100;
|
export const DEFAULT_FEATURE_RECOMPUTE_BATCH_SIZE = 100;
|
||||||
|
|||||||
@@ -12,12 +12,14 @@ import { AnalysisQueueService } from './analysis-queue.service';
|
|||||||
import {
|
import {
|
||||||
ANALYSIS_JOB_ID_FACTORY,
|
ANALYSIS_JOB_ID_FACTORY,
|
||||||
ANALYSIS_QUEUE_NAME,
|
ANALYSIS_QUEUE_NAME,
|
||||||
|
SEED_COLLECTION_QUEUE_NAME,
|
||||||
} from './analysis.constants';
|
} from './analysis.constants';
|
||||||
import { AnalysisPipelineService } from './analysis-pipeline.service';
|
import { AnalysisPipelineService } from './analysis-pipeline.service';
|
||||||
import { AnalysisProcessor } from './analysis.processor';
|
import { AnalysisProcessor } from './analysis.processor';
|
||||||
import { createBullMqOptions } from './bullmq.config';
|
import { createBullMqOptions } from './bullmq.config';
|
||||||
import { FeatureExtractorService } from './feature-extractor.service';
|
import { FeatureExtractorService } from './feature-extractor.service';
|
||||||
import { ProfileBuilderService } from './profile-builder.service';
|
import { ProfileBuilderService } from './profile-builder.service';
|
||||||
|
import { SeedCollectionProcessor } from './seed-collection.processor';
|
||||||
import { SeedCollectionService } from './seed-collection.service';
|
import { SeedCollectionService } from './seed-collection.service';
|
||||||
import { TimelineDerivedService } from './timeline-derived.service';
|
import { TimelineDerivedService } from './timeline-derived.service';
|
||||||
import { TimelineMetricService } from './timeline-metric.service';
|
import { TimelineMetricService } from './timeline-metric.service';
|
||||||
@@ -38,6 +40,9 @@ import { TimelineMetricService } from './timeline-metric.service';
|
|||||||
BullModule.registerQueue({
|
BullModule.registerQueue({
|
||||||
name: ANALYSIS_QUEUE_NAME,
|
name: ANALYSIS_QUEUE_NAME,
|
||||||
}),
|
}),
|
||||||
|
BullModule.registerQueue({
|
||||||
|
name: SEED_COLLECTION_QUEUE_NAME,
|
||||||
|
}),
|
||||||
],
|
],
|
||||||
providers: [
|
providers: [
|
||||||
{
|
{
|
||||||
@@ -49,6 +54,7 @@ import { TimelineMetricService } from './timeline-metric.service';
|
|||||||
AnalysisRequestThrottleService,
|
AnalysisRequestThrottleService,
|
||||||
AnalysisPipelineService,
|
AnalysisPipelineService,
|
||||||
AnalysisProcessor,
|
AnalysisProcessor,
|
||||||
|
SeedCollectionProcessor,
|
||||||
FeatureExtractorService,
|
FeatureExtractorService,
|
||||||
ProfileBuilderService,
|
ProfileBuilderService,
|
||||||
SeedCollectionService,
|
SeedCollectionService,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import type { Role } from '@persona-lenz/shared';
|
import type { Role } from '@persona-lenz/shared';
|
||||||
import type {
|
import type {
|
||||||
LaneOpponentRef,
|
LaneOpponentRef,
|
||||||
|
SeedSource,
|
||||||
TeamPositionOrUnknown,
|
TeamPositionOrUnknown,
|
||||||
} from '../global/database/schemas';
|
} from '../global/database/schemas';
|
||||||
import type { RiotPlatformRoute, RiotRegionalRoute } from '../riot/riot.types';
|
import type { RiotPlatformRoute, RiotRegionalRoute } from '../riot/riot.types';
|
||||||
@@ -21,6 +22,12 @@ export interface FeatureRecomputeJobData {
|
|||||||
puuid?: string;
|
puuid?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SeedCollectionJobData {
|
||||||
|
platformRoute: RiotPlatformRoute;
|
||||||
|
queueId: number;
|
||||||
|
sourceLimits?: Partial<Record<SeedSource, number>>;
|
||||||
|
}
|
||||||
|
|
||||||
export interface ExtractedPlayerMatchFeature {
|
export interface ExtractedPlayerMatchFeature {
|
||||||
matchId: string;
|
matchId: string;
|
||||||
puuid: string;
|
puuid: string;
|
||||||
|
|||||||
@@ -0,0 +1,221 @@
|
|||||||
|
import type { Job } from 'bullmq';
|
||||||
|
import type { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import type { SeedCollectionService } from './seed-collection.service';
|
||||||
|
import { SEED_COLLECTION_JOB_NAME } from './analysis.constants';
|
||||||
|
import type { SeedCollectionJobData } from './analysis.types';
|
||||||
|
import { SeedCollectionProcessor } from './seed-collection.processor';
|
||||||
|
|
||||||
|
function createSeedCollectionJobData(
|
||||||
|
overrides: Partial<SeedCollectionJobData> = {},
|
||||||
|
): SeedCollectionJobData {
|
||||||
|
return {
|
||||||
|
platformRoute: 'kr',
|
||||||
|
queueId: 420,
|
||||||
|
sourceLimits: {
|
||||||
|
challenger: 50,
|
||||||
|
grandmaster: 50,
|
||||||
|
master: 50,
|
||||||
|
},
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createUpdateOneMock() {
|
||||||
|
return jest.fn(() => ({
|
||||||
|
exec: jest.fn().mockResolvedValue({ acknowledged: true }),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function asRecord(value: unknown): Record<string, unknown> {
|
||||||
|
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
||||||
|
return value as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('SeedCollectionProcessor', () => {
|
||||||
|
it('delegates seed collection jobs and marks them completed', async () => {
|
||||||
|
const collectSeedAccounts = jest.fn().mockResolvedValue({
|
||||||
|
selectedEntries: 150,
|
||||||
|
storedAccounts: 149,
|
||||||
|
skippedAccounts: 1,
|
||||||
|
});
|
||||||
|
const updateOne = createUpdateOneMock();
|
||||||
|
const connectionService = {
|
||||||
|
getModel: jest.fn().mockResolvedValue({ updateOne }),
|
||||||
|
} as unknown as ConnectionService;
|
||||||
|
const processor = new SeedCollectionProcessor(
|
||||||
|
{
|
||||||
|
collectSeedAccounts,
|
||||||
|
} as unknown as SeedCollectionService,
|
||||||
|
connectionService,
|
||||||
|
);
|
||||||
|
|
||||||
|
await processor.process({
|
||||||
|
attemptsMade: 0,
|
||||||
|
id: 'seed-job-1',
|
||||||
|
name: SEED_COLLECTION_JOB_NAME,
|
||||||
|
opts: { attempts: 1 },
|
||||||
|
data: createSeedCollectionJobData(),
|
||||||
|
} as Job<unknown>);
|
||||||
|
|
||||||
|
expect(collectSeedAccounts).toHaveBeenCalledWith({
|
||||||
|
platformRoute: 'kr',
|
||||||
|
sourceLimits: {
|
||||||
|
challenger: 50,
|
||||||
|
grandmaster: 50,
|
||||||
|
master: 50,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(updateOne).toHaveBeenCalledWith(
|
||||||
|
{ jobId: 'seed-job-1' },
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
status: 'processing',
|
||||||
|
stage: 'collect-seed-accounts',
|
||||||
|
progress: {
|
||||||
|
fetchedMatches: 0,
|
||||||
|
totalMatches: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const completedUpdate = updateOne.mock.calls.find(
|
||||||
|
([, update]) => asRecord(asRecord(update).$set).status === 'completed',
|
||||||
|
);
|
||||||
|
expect(completedUpdate).toBeDefined();
|
||||||
|
expect(asRecord(asRecord(completedUpdate?.[1]).$set)).toMatchObject({
|
||||||
|
status: 'completed',
|
||||||
|
stage: 'completed',
|
||||||
|
progress: {
|
||||||
|
fetchedMatches: 149,
|
||||||
|
totalMatches: 150,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
asRecord(asRecord(completedUpdate?.[1]).$set).completedAt,
|
||||||
|
).toBeInstanceOf(Date);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
{
|
||||||
|
data: null,
|
||||||
|
expectedError: 'Seed collection job data must be an object.',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
data: {
|
||||||
|
...createSeedCollectionJobData(),
|
||||||
|
platformRoute: 'invalid',
|
||||||
|
},
|
||||||
|
expectedError: 'Seed collection job platformRoute is unsupported.',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
data: {
|
||||||
|
...createSeedCollectionJobData(),
|
||||||
|
queueId: 430,
|
||||||
|
},
|
||||||
|
expectedError: 'Seed collection job queueId is unsupported.',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
data: {
|
||||||
|
...createSeedCollectionJobData(),
|
||||||
|
sourceLimits: {
|
||||||
|
challenger: Number.NaN,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectedError:
|
||||||
|
'Seed collection job sourceLimits.challenger must be a non-negative finite number.',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
data: {
|
||||||
|
...createSeedCollectionJobData(),
|
||||||
|
sourceLimits: {
|
||||||
|
diamond: 50,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectedError: 'Seed collection job sourceLimits.diamond is unsupported.',
|
||||||
|
},
|
||||||
|
])(
|
||||||
|
'rejects malformed seed collection payloads',
|
||||||
|
async ({ data, expectedError }) => {
|
||||||
|
const collectSeedAccounts = jest.fn();
|
||||||
|
const updateOne = createUpdateOneMock();
|
||||||
|
const connectionService = {
|
||||||
|
getModel: jest.fn().mockResolvedValue({ updateOne }),
|
||||||
|
} as unknown as ConnectionService;
|
||||||
|
const processor = new SeedCollectionProcessor(
|
||||||
|
{
|
||||||
|
collectSeedAccounts,
|
||||||
|
} as unknown as SeedCollectionService,
|
||||||
|
connectionService,
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
processor.process({
|
||||||
|
attemptsMade: 0,
|
||||||
|
id: 'seed-job-1',
|
||||||
|
name: SEED_COLLECTION_JOB_NAME,
|
||||||
|
opts: { attempts: 1 },
|
||||||
|
data,
|
||||||
|
} as Job<unknown>),
|
||||||
|
).rejects.toThrow(expectedError);
|
||||||
|
expect(collectSeedAccounts).not.toHaveBeenCalled();
|
||||||
|
expect(updateOne).not.toHaveBeenCalled();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it('marks final-attempt failures as failed', async () => {
|
||||||
|
const collectSeedAccounts = jest
|
||||||
|
.fn()
|
||||||
|
.mockRejectedValue(new Error('Riot unavailable'));
|
||||||
|
const updateOne = createUpdateOneMock();
|
||||||
|
const connectionService = {
|
||||||
|
getModel: jest.fn().mockResolvedValue({ updateOne }),
|
||||||
|
} as unknown as ConnectionService;
|
||||||
|
const processor = new SeedCollectionProcessor(
|
||||||
|
{
|
||||||
|
collectSeedAccounts,
|
||||||
|
} as unknown as SeedCollectionService,
|
||||||
|
connectionService,
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
processor.process({
|
||||||
|
attemptsMade: 0,
|
||||||
|
id: 'seed-job-1',
|
||||||
|
name: SEED_COLLECTION_JOB_NAME,
|
||||||
|
opts: { attempts: 1 },
|
||||||
|
data: createSeedCollectionJobData(),
|
||||||
|
} as Job<unknown>),
|
||||||
|
).rejects.toThrow('Riot unavailable');
|
||||||
|
|
||||||
|
expect(updateOne).toHaveBeenCalledWith(
|
||||||
|
{ jobId: 'seed-job-1' },
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
status: 'failed',
|
||||||
|
error: 'Seed collection failed during processing.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects unsupported job names', async () => {
|
||||||
|
const processor = new SeedCollectionProcessor(
|
||||||
|
{
|
||||||
|
collectSeedAccounts: jest.fn(),
|
||||||
|
} as unknown as SeedCollectionService,
|
||||||
|
{
|
||||||
|
getModel: jest.fn(),
|
||||||
|
} as unknown as ConnectionService,
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
processor.process({
|
||||||
|
id: 'seed-job-1',
|
||||||
|
name: 'unexpected',
|
||||||
|
data: createSeedCollectionJobData(),
|
||||||
|
} as Job<unknown>),
|
||||||
|
).rejects.toThrow('Unsupported seed collection job name "unexpected".');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
import { Processor, WorkerHost } from '@nestjs/bullmq';
|
||||||
|
import type { Job } from 'bullmq';
|
||||||
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import { SEED_SOURCES, type SeedSource } from '../global/database/schemas';
|
||||||
|
import {
|
||||||
|
SEED_COLLECTION_JOB_NAME,
|
||||||
|
SEED_COLLECTION_QUEUE_NAME,
|
||||||
|
} from './analysis.constants';
|
||||||
|
import type { SeedCollectionJobData } from './analysis.types';
|
||||||
|
import {
|
||||||
|
RIOT_PLATFORM_ROUTES,
|
||||||
|
type RiotPlatformRoute,
|
||||||
|
} from '../riot/riot.types';
|
||||||
|
import { SeedCollectionService } from './seed-collection.service';
|
||||||
|
|
||||||
|
@Processor(SEED_COLLECTION_QUEUE_NAME, { concurrency: 1 })
|
||||||
|
export class SeedCollectionProcessor extends WorkerHost {
|
||||||
|
constructor(
|
||||||
|
private readonly seedCollectionService: SeedCollectionService,
|
||||||
|
private readonly connectionService: ConnectionService,
|
||||||
|
) {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async process(job: Job<unknown>): Promise<void> {
|
||||||
|
if (!job.id) {
|
||||||
|
throw new Error('Seed collection job is missing a BullMQ job id.');
|
||||||
|
}
|
||||||
|
if (job.name !== SEED_COLLECTION_JOB_NAME) {
|
||||||
|
throw new Error(`Unsupported seed collection job name "${job.name}".`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = this.parseSeedCollectionJobData(job.data);
|
||||||
|
try {
|
||||||
|
await this.updateJob(job.id, {
|
||||||
|
status: 'processing',
|
||||||
|
stage: 'collect-seed-accounts',
|
||||||
|
progress: {
|
||||||
|
fetchedMatches: 0,
|
||||||
|
totalMatches: 0,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await this.seedCollectionService.collectSeedAccounts({
|
||||||
|
platformRoute: data.platformRoute,
|
||||||
|
...(data.sourceLimits ? { sourceLimits: data.sourceLimits } : {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.updateJob(job.id, {
|
||||||
|
status: 'completed',
|
||||||
|
stage: 'completed',
|
||||||
|
progress: {
|
||||||
|
fetchedMatches: result.storedAccounts,
|
||||||
|
totalMatches: result.selectedEntries,
|
||||||
|
},
|
||||||
|
completedAt: new Date(),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (this.isFinalAttempt(job)) {
|
||||||
|
await this.updateJob(job.id, {
|
||||||
|
status: 'failed',
|
||||||
|
error: this.publicErrorMessage(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseSeedCollectionJobData(value: unknown): SeedCollectionJobData {
|
||||||
|
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||||
|
throw new Error('Seed collection job data must be an object.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const record = value as Record<string, unknown>;
|
||||||
|
const platformRoute = this.parsePlatformRoute(record.platformRoute);
|
||||||
|
const queueId = this.parseQueueId(record.queueId);
|
||||||
|
const sourceLimits = this.parseSourceLimits(record.sourceLimits);
|
||||||
|
|
||||||
|
return {
|
||||||
|
platformRoute,
|
||||||
|
queueId,
|
||||||
|
...(sourceLimits ? { sourceLimits } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private parsePlatformRoute(value: unknown): RiotPlatformRoute {
|
||||||
|
if (
|
||||||
|
typeof value !== 'string' ||
|
||||||
|
!RIOT_PLATFORM_ROUTES.includes(value as RiotPlatformRoute)
|
||||||
|
) {
|
||||||
|
throw new Error('Seed collection job platformRoute is unsupported.');
|
||||||
|
}
|
||||||
|
return value as RiotPlatformRoute;
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseQueueId(value: unknown): number {
|
||||||
|
if (value !== 420) {
|
||||||
|
throw new Error('Seed collection job queueId is unsupported.');
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseSourceLimits(
|
||||||
|
value: unknown,
|
||||||
|
): Partial<Record<SeedSource, number>> | undefined {
|
||||||
|
if (value === undefined) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||||
|
throw new Error('Seed collection job sourceLimits must be an object.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const sourceLimits = value as Record<string, unknown>;
|
||||||
|
const parsedSourceLimits: Partial<Record<SeedSource, number>> = {};
|
||||||
|
for (const [source, limit] of Object.entries(sourceLimits)) {
|
||||||
|
if (!SEED_SOURCES.includes(source as SeedSource)) {
|
||||||
|
throw new Error(
|
||||||
|
`Seed collection job sourceLimits.${source} is unsupported.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (typeof limit !== 'number' || !Number.isFinite(limit) || limit < 0) {
|
||||||
|
throw new Error(
|
||||||
|
`Seed collection job sourceLimits.${source} must be a non-negative finite number.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
parsedSourceLimits[source as SeedSource] = Math.trunc(limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsedSourceLimits;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async updateJob(
|
||||||
|
jobId: string,
|
||||||
|
updates: Record<string, unknown>,
|
||||||
|
): Promise<void> {
|
||||||
|
const analysisJobModel =
|
||||||
|
await this.connectionService.getModel('analysisJob');
|
||||||
|
await analysisJobModel
|
||||||
|
.updateOne(
|
||||||
|
{ jobId },
|
||||||
|
{
|
||||||
|
$set: updates,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.exec();
|
||||||
|
}
|
||||||
|
|
||||||
|
private isFinalAttempt(job: Job<unknown>): boolean {
|
||||||
|
const attempts =
|
||||||
|
typeof job.opts?.attempts === 'number' && job.opts.attempts > 0
|
||||||
|
? job.opts.attempts
|
||||||
|
: 1;
|
||||||
|
return job.attemptsMade + 1 >= attempts;
|
||||||
|
}
|
||||||
|
|
||||||
|
private publicErrorMessage(): string {
|
||||||
|
return 'Seed collection failed during processing.';
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user