Wire seed collection queue

This commit is contained in:
dahyeon committed 2026-07-03 01:04:45 -07:00
1 parent a6c1acbc31
commit 63a4aeace7
8 files changed
+608 -3

No files matched your search

+8 -1
View File
@@ -2,7 +2,10 @@ import { getQueueToken } from '@nestjs/bullmq';
import { Test } from '@nestjs/testing'; import { Test } from '@nestjs/testing';
import { AppModule } from './app.module'; import { AppModule } from './app.module';
import { AnalysisQueueService } from './modules/analysis/analysis-queue.service'; import { AnalysisQueueService } from './modules/analysis/analysis-queue.service';
import { ANALYSIS_QUEUE_NAME } from './modules/analysis/analysis.constants'; import {
ANALYSIS_QUEUE_NAME,
SEED_COLLECTION_QUEUE_NAME,
} from './modules/analysis/analysis.constants';
import { ConnectionService } from './modules/global/database/connection.service'; import { ConnectionService } from './modules/global/database/connection.service';
import { REDIS_CLIENT } from './modules/global/redis/redis.constants'; import { REDIS_CLIENT } from './modules/global/redis/redis.constants';
import { SecretService } from './modules/global/secret/secret.service'; import { SecretService } from './modules/global/secret/secret.service';
@@ -36,6 +39,10 @@ describe('AppModule', () => {
.useValue({ .useValue({
add: jest.fn(), add: jest.fn(),
}) })
.overrideProvider(getQueueToken(SEED_COLLECTION_QUEUE_NAME))
.useValue({
add: jest.fn(),
})
.compile(); .compile();
expect(moduleRef.get(AnalysisQueueService)).toBeInstanceOf( expect(moduleRef.get(AnalysisQueueService)).toBeInstanceOf(
@@ -1,8 +1,13 @@
import type { AnalysisJob } from '../global/database/schemas'; import type { AnalysisJob } from '../global/database/schemas';
import type { ConnectionService } from '../global/database/connection.service'; import type { ConnectionService } from '../global/database/connection.service';
import type { UserAnalysisJobData } from './analysis.types'; import type {
SeedCollectionJobData,
UserAnalysisJobData,
} from './analysis.types';
import { import {
DEFAULT_SEED_COLLECTION_MATCH_COUNT,
FEATURE_RECOMPUTE_JOB_NAME, FEATURE_RECOMPUTE_JOB_NAME,
SEED_COLLECTION_JOB_NAME,
USER_ANALYSIS_JOB_NAME, USER_ANALYSIS_JOB_NAME,
} from './analysis.constants'; } from './analysis.constants';
import { AnalysisQueueService } from './analysis-queue.service'; import { AnalysisQueueService } from './analysis-queue.service';
@@ -37,9 +42,25 @@ function createFindOneChain(
}; };
} }
function createSeedCollectionJobData(
overrides: Partial<SeedCollectionJobData> = {},
): SeedCollectionJobData {
return {
platformRoute: 'kr',
queueId: 420,
sourceLimits: {
challenger: 50,
grandmaster: 50,
master: 50,
},
...overrides,
};
}
describe('AnalysisQueueService', () => { describe('AnalysisQueueService', () => {
it('creates an analysis job document and enqueues a BullMQ job', async () => { it('creates an analysis job document and enqueues a BullMQ job', async () => {
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' }); const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
const seedAdd = jest.fn();
const findOneChain = createFindOneChain(null); const findOneChain = createFindOneChain(null);
const create = jest.fn().mockResolvedValue({}); const create = jest.fn().mockResolvedValue({});
const updateOne = jest.fn(() => ({ const updateOne = jest.fn(() => ({
@@ -55,6 +76,7 @@ describe('AnalysisQueueService', () => {
} as unknown as ConnectionService; } as unknown as ConnectionService;
const service = new AnalysisQueueService( const service = new AnalysisQueueService(
{ add } as never, { add } as never,
{ add: seedAdd } as never,
connectionService, connectionService,
() => 'analysis-job-1', () => 'analysis-job-1',
); );
@@ -79,10 +101,12 @@ describe('AnalysisQueueService', () => {
jobId: 'analysis-job-1', jobId: 'analysis-job-1',
}), }),
); );
expect(seedAdd).not.toHaveBeenCalled();
}); });
it('creates a feature recompute job document and enqueues a BullMQ job', async () => { it('creates a feature recompute job document and enqueues a BullMQ job', async () => {
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' }); const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
const seedAdd = jest.fn();
const create = jest.fn().mockResolvedValue({}); const create = jest.fn().mockResolvedValue({});
const updateOne = jest.fn(() => ({ const updateOne = jest.fn(() => ({
exec: jest.fn().mockResolvedValue({}), exec: jest.fn().mockResolvedValue({}),
@@ -96,6 +120,7 @@ describe('AnalysisQueueService', () => {
} as unknown as ConnectionService; } as unknown as ConnectionService;
const service = new AnalysisQueueService( const service = new AnalysisQueueService(
{ add } as never, { add } as never,
{ add: seedAdd } as never,
connectionService, connectionService,
() => 'recompute-job-1', () => 'recompute-job-1',
); );
@@ -132,6 +157,54 @@ describe('AnalysisQueueService', () => {
jobId: 'recompute-job-1', jobId: 'recompute-job-1',
}), }),
); );
expect(seedAdd).not.toHaveBeenCalled();
});
it('creates a seed collection job document and enqueues the seed queue', async () => {
const analysisAdd = jest.fn();
const seedAdd = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
const create = jest.fn().mockResolvedValue({});
const updateOne = jest.fn(() => ({
exec: jest.fn().mockResolvedValue({}),
}));
const analysisJobModel = {
create,
updateOne,
};
const connectionService = {
getModel: jest.fn().mockResolvedValue(analysisJobModel),
} as unknown as ConnectionService;
const service = new AnalysisQueueService(
{ add: analysisAdd } as never,
{ add: seedAdd } as never,
connectionService,
() => 'seed-job-1',
);
const result = await service.enqueueSeedCollection(
createSeedCollectionJobData(),
);
expect(result).toEqual({ jobId: 'seed-job-1', status: 'queued' });
expect(create).toHaveBeenCalledWith(
expect.objectContaining({
jobId: 'seed-job-1',
jobType: SEED_COLLECTION_JOB_NAME,
platformRoute: 'kr',
queueId: 420,
requestedMatchCount: DEFAULT_SEED_COLLECTION_MATCH_COUNT,
status: 'queued',
}),
);
expect(seedAdd).toHaveBeenCalledWith(
SEED_COLLECTION_JOB_NAME,
createSeedCollectionJobData(),
expect.objectContaining({
attempts: 1,
jobId: 'seed-job-1',
}),
);
expect(analysisAdd).not.toHaveBeenCalled();
}); });
it('returns an existing active job instead of enqueueing a duplicate', async () => { it('returns an existing active job instead of enqueueing a duplicate', async () => {
@@ -149,6 +222,7 @@ describe('AnalysisQueueService', () => {
} as unknown as ConnectionService; } as unknown as ConnectionService;
const service = new AnalysisQueueService( const service = new AnalysisQueueService(
{ add } as never, { add } as never,
{ add: jest.fn() } as never,
connectionService, connectionService,
() => 'new-job', () => 'new-job',
); );
@@ -180,6 +254,7 @@ describe('AnalysisQueueService', () => {
} as unknown as ConnectionService; } as unknown as ConnectionService;
const service = new AnalysisQueueService( const service = new AnalysisQueueService(
{ add } as never, { add } as never,
{ add: jest.fn() } as never,
connectionService, connectionService,
() => 'analysis-job-1', () => 'analysis-job-1',
); );
@@ -214,6 +289,7 @@ describe('AnalysisQueueService', () => {
} as unknown as ConnectionService; } as unknown as ConnectionService;
const service = new AnalysisQueueService( const service = new AnalysisQueueService(
{ add } as never, { add } as never,
{ add: jest.fn() } as never,
connectionService, connectionService,
() => 'recompute-job-1', () => 'recompute-job-1',
); );
@@ -237,6 +313,43 @@ describe('AnalysisQueueService', () => {
); );
}); });
it('marks a pre-created seed collection job failed when BullMQ enqueue fails', async () => {
const analysisAdd = jest.fn();
const seedAdd = jest.fn().mockRejectedValue(new Error('redis unavailable'));
const create = jest.fn().mockResolvedValue({});
const updateOne = jest.fn(() => ({
exec: jest.fn().mockResolvedValue({}),
}));
const analysisJobModel = {
create,
updateOne,
};
const connectionService = {
getModel: jest.fn().mockResolvedValue(analysisJobModel),
} as unknown as ConnectionService;
const service = new AnalysisQueueService(
{ add: analysisAdd } as never,
{ add: seedAdd } as never,
connectionService,
() => 'seed-job-1',
);
await expect(
service.enqueueSeedCollection(createSeedCollectionJobData()),
).rejects.toThrow('redis unavailable');
expect(updateOne).toHaveBeenCalledWith(
{ jobId: 'seed-job-1' },
{
$set: {
status: 'failed',
stage: 'enqueue',
error: 'Analysis could not be queued.',
},
},
);
expect(analysisAdd).not.toHaveBeenCalled();
});
it('serializes identical in-process requests before creating a duplicate job', async () => { it('serializes identical in-process requests before creating a duplicate job', async () => {
const add = jest.fn().mockResolvedValue({ id: 'analysis-job-1' }); const add = jest.fn().mockResolvedValue({ id: 'analysis-job-1' });
let activeJob: Pick<AnalysisJob, 'jobId' | 'status'> | null = null; let activeJob: Pick<AnalysisJob, 'jobId' | 'status'> | null = null;
@@ -263,6 +376,7 @@ describe('AnalysisQueueService', () => {
} as unknown as ConnectionService; } as unknown as ConnectionService;
const service = new AnalysisQueueService( const service = new AnalysisQueueService(
{ add } as never, { add } as never,
{ add: jest.fn() } as never,
connectionService, connectionService,
() => (create.mock.calls.length === 0 ? 'analysis-job-1' : 'unexpected'), () => (create.mock.calls.length === 0 ? 'analysis-job-1' : 'unexpected'),
); );
@@ -4,18 +4,22 @@ import type { AnalysisRequestAcceptedDto } from '@persona-lenz/shared';
import type { Queue } from 'bullmq'; import type { Queue } from 'bullmq';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { ConnectionService } from '../global/database/connection.service'; import { ConnectionService } from '../global/database/connection.service';
import type { AnalysisJob } from '../global/database/schemas'; import type { AnalysisJob, SeedSource } from '../global/database/schemas';
import { import {
ANALYSIS_JOB_ID_FACTORY, ANALYSIS_JOB_ID_FACTORY,
ACTIVE_ANALYSIS_JOB_STATUSES, ACTIVE_ANALYSIS_JOB_STATUSES,
ANALYSIS_QUEUE_NAME, ANALYSIS_QUEUE_NAME,
DEFAULT_SEED_COLLECTION_MATCH_COUNT,
FEATURE_RECOMPUTE_JOB_NAME, FEATURE_RECOMPUTE_JOB_NAME,
normalizeFeatureRecomputeBatchSize, normalizeFeatureRecomputeBatchSize,
normalizeRequestedMatchCount, normalizeRequestedMatchCount,
SEED_COLLECTION_JOB_NAME,
SEED_COLLECTION_QUEUE_NAME,
USER_ANALYSIS_JOB_NAME, USER_ANALYSIS_JOB_NAME,
} from './analysis.constants'; } from './analysis.constants';
import type { import type {
FeatureRecomputeJobData, FeatureRecomputeJobData,
SeedCollectionJobData,
UserAnalysisJobData, UserAnalysisJobData,
} from './analysis.types'; } from './analysis.types';
@@ -23,6 +27,7 @@ type JobIdFactory = () => string;
type AnalysisQueueJobData = UserAnalysisJobData | FeatureRecomputeJobData; type AnalysisQueueJobData = UserAnalysisJobData | FeatureRecomputeJobData;
type AnalysisQueueJobName = type AnalysisQueueJobName =
typeof USER_ANALYSIS_JOB_NAME | typeof FEATURE_RECOMPUTE_JOB_NAME; typeof USER_ANALYSIS_JOB_NAME | typeof FEATURE_RECOMPUTE_JOB_NAME;
type SeedCollectionQueueJobName = typeof SEED_COLLECTION_JOB_NAME;
type ActiveAnalysisJob = Pick<AnalysisJob, 'jobId'> & { type ActiveAnalysisJob = Pick<AnalysisJob, 'jobId'> & {
status: Extract<AnalysisJob['status'], 'queued' | 'processing'>; status: Extract<AnalysisJob['status'], 'queued' | 'processing'>;
}; };
@@ -36,6 +41,12 @@ export class AnalysisQueueService {
void, void,
AnalysisQueueJobName AnalysisQueueJobName
>, >,
@InjectQueue(SEED_COLLECTION_QUEUE_NAME)
private readonly seedCollectionQueue: Queue<
SeedCollectionJobData,
void,
SeedCollectionQueueJobName
>,
private readonly connectionService: ConnectionService, private readonly connectionService: ConnectionService,
@Inject(ANALYSIS_JOB_ID_FACTORY) @Inject(ANALYSIS_JOB_ID_FACTORY)
private readonly jobIdFactory: JobIdFactory = randomUUID, private readonly jobIdFactory: JobIdFactory = randomUUID,
@@ -115,6 +126,59 @@ export class AnalysisQueueService {
}; };
} }
public async enqueueSeedCollection(
data: SeedCollectionJobData,
): Promise<AnalysisRequestAcceptedDto> {
const normalizedData = this.normalizeSeedCollectionData(data);
const analysisJobModel =
await this.connectionService.getModel('analysisJob');
const jobId = this.jobIdFactory();
await analysisJobModel.create({
jobId,
jobType: SEED_COLLECTION_JOB_NAME,
platformRoute: normalizedData.platformRoute,
queueId: normalizedData.queueId,
requestedMatchCount: DEFAULT_SEED_COLLECTION_MATCH_COUNT,
status: 'queued',
stage: 'queued',
progress: {
fetchedMatches: 0,
totalMatches: 0,
},
});
try {
await this.seedCollectionQueue.add(
SEED_COLLECTION_JOB_NAME,
normalizedData,
{
attempts: 1,
jobId,
},
);
} catch (error) {
await analysisJobModel
.updateOne(
{ jobId },
{
$set: {
status: 'failed',
stage: 'enqueue',
error: this.publicErrorMessage(),
},
},
)
.exec();
throw error;
}
return {
jobId,
status: 'queued',
};
}
private readonly pendingDuplicateRequestKeys = new Set<string>(); private readonly pendingDuplicateRequestKeys = new Set<string>();
private async enqueueUserAnalysisWithoutDuplicateRequestRace( private async enqueueUserAnalysisWithoutDuplicateRequestRace(
@@ -218,6 +282,30 @@ export class AnalysisQueueService {
}; };
} }
private normalizeSeedCollectionData(
data: SeedCollectionJobData,
): SeedCollectionJobData {
return {
platformRoute: data.platformRoute,
queueId: data.queueId,
...(data.sourceLimits
? { sourceLimits: this.normalizeSeedSourceLimits(data.sourceLimits) }
: {}),
};
}
private normalizeSeedSourceLimits(
sourceLimits: NonNullable<SeedCollectionJobData['sourceLimits']>,
): NonNullable<SeedCollectionJobData['sourceLimits']> {
const normalizedSourceLimits: Partial<Record<SeedSource, number>> = {};
for (const [source, limit] of Object.entries(sourceLimits)) {
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
normalizedSourceLimits[source as SeedSource] = Math.trunc(limit);
}
}
return normalizedSourceLimits;
}
private duplicateRequestKey(data: UserAnalysisJobData): string { private duplicateRequestKey(data: UserAnalysisJobData): string {
return [ return [
data.platformRoute, data.platformRoute,
@@ -1,10 +1,13 @@
export const ANALYSIS_QUEUE_NAME = 'analysis'; export const ANALYSIS_QUEUE_NAME = 'analysis';
export const SEED_COLLECTION_QUEUE_NAME = 'seed-collection';
export const USER_ANALYSIS_JOB_NAME = 'user-analysis'; export const USER_ANALYSIS_JOB_NAME = 'user-analysis';
export const FEATURE_RECOMPUTE_JOB_NAME = 'feature-recompute'; export const FEATURE_RECOMPUTE_JOB_NAME = 'feature-recompute';
export const SEED_COLLECTION_JOB_NAME = 'seed-collection';
export const ANALYSIS_JOB_ID_FACTORY = Symbol('ANALYSIS_JOB_ID_FACTORY'); export const ANALYSIS_JOB_ID_FACTORY = Symbol('ANALYSIS_JOB_ID_FACTORY');
export const DEFAULT_ANALYSIS_MATCH_COUNT = 20; export const DEFAULT_ANALYSIS_MATCH_COUNT = 20;
export const MAX_ANALYSIS_MATCH_COUNT = 50; export const MAX_ANALYSIS_MATCH_COUNT = 50;
export const DEFAULT_SEED_COLLECTION_MATCH_COUNT = 30;
export const DEFAULT_SEED_COLLECTION_QUEUE_TYPE = 'RANKED_SOLO_5x5'; export const DEFAULT_SEED_COLLECTION_QUEUE_TYPE = 'RANKED_SOLO_5x5';
export const DEFAULT_SEED_COLLECTION_SOURCE_LIMIT = 50; export const DEFAULT_SEED_COLLECTION_SOURCE_LIMIT = 50;
export const DEFAULT_FEATURE_RECOMPUTE_BATCH_SIZE = 100; export const DEFAULT_FEATURE_RECOMPUTE_BATCH_SIZE = 100;
@@ -12,12 +12,14 @@ import { AnalysisQueueService } from './analysis-queue.service';
import { import {
ANALYSIS_JOB_ID_FACTORY, ANALYSIS_JOB_ID_FACTORY,
ANALYSIS_QUEUE_NAME, ANALYSIS_QUEUE_NAME,
SEED_COLLECTION_QUEUE_NAME,
} from './analysis.constants'; } from './analysis.constants';
import { AnalysisPipelineService } from './analysis-pipeline.service'; import { AnalysisPipelineService } from './analysis-pipeline.service';
import { AnalysisProcessor } from './analysis.processor'; import { AnalysisProcessor } from './analysis.processor';
import { createBullMqOptions } from './bullmq.config'; import { createBullMqOptions } from './bullmq.config';
import { FeatureExtractorService } from './feature-extractor.service'; import { FeatureExtractorService } from './feature-extractor.service';
import { ProfileBuilderService } from './profile-builder.service'; import { ProfileBuilderService } from './profile-builder.service';
import { SeedCollectionProcessor } from './seed-collection.processor';
import { SeedCollectionService } from './seed-collection.service'; import { SeedCollectionService } from './seed-collection.service';
import { TimelineDerivedService } from './timeline-derived.service'; import { TimelineDerivedService } from './timeline-derived.service';
import { TimelineMetricService } from './timeline-metric.service'; import { TimelineMetricService } from './timeline-metric.service';
@@ -38,6 +40,9 @@ import { TimelineMetricService } from './timeline-metric.service';
BullModule.registerQueue({ BullModule.registerQueue({
name: ANALYSIS_QUEUE_NAME, name: ANALYSIS_QUEUE_NAME,
}), }),
BullModule.registerQueue({
name: SEED_COLLECTION_QUEUE_NAME,
}),
], ],
providers: [ providers: [
{ {
@@ -49,6 +54,7 @@ import { TimelineMetricService } from './timeline-metric.service';
AnalysisRequestThrottleService, AnalysisRequestThrottleService,
AnalysisPipelineService, AnalysisPipelineService,
AnalysisProcessor, AnalysisProcessor,
SeedCollectionProcessor,
FeatureExtractorService, FeatureExtractorService,
ProfileBuilderService, ProfileBuilderService,
SeedCollectionService, SeedCollectionService,
@@ -1,6 +1,7 @@
import type { Role } from '@persona-lenz/shared'; import type { Role } from '@persona-lenz/shared';
import type { import type {
LaneOpponentRef, LaneOpponentRef,
SeedSource,
TeamPositionOrUnknown, TeamPositionOrUnknown,
} from '../global/database/schemas'; } from '../global/database/schemas';
import type { RiotPlatformRoute, RiotRegionalRoute } from '../riot/riot.types'; import type { RiotPlatformRoute, RiotRegionalRoute } from '../riot/riot.types';
@@ -21,6 +22,12 @@ export interface FeatureRecomputeJobData {
puuid?: string; puuid?: string;
} }
export interface SeedCollectionJobData {
platformRoute: RiotPlatformRoute;
queueId: number;
sourceLimits?: Partial<Record<SeedSource, number>>;
}
export interface ExtractedPlayerMatchFeature { export interface ExtractedPlayerMatchFeature {
matchId: string; matchId: string;
puuid: string; puuid: string;
@@ -0,0 +1,221 @@
import type { Job } from 'bullmq';
import type { ConnectionService } from '../global/database/connection.service';
import type { SeedCollectionService } from './seed-collection.service';
import { SEED_COLLECTION_JOB_NAME } from './analysis.constants';
import type { SeedCollectionJobData } from './analysis.types';
import { SeedCollectionProcessor } from './seed-collection.processor';
function createSeedCollectionJobData(
overrides: Partial<SeedCollectionJobData> = {},
): SeedCollectionJobData {
return {
platformRoute: 'kr',
queueId: 420,
sourceLimits: {
challenger: 50,
grandmaster: 50,
master: 50,
},
...overrides,
};
}
function createUpdateOneMock() {
return jest.fn(() => ({
exec: jest.fn().mockResolvedValue({ acknowledged: true }),
}));
}
function asRecord(value: unknown): Record<string, unknown> {
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
return value as Record<string, unknown>;
}
return {};
}
describe('SeedCollectionProcessor', () => {
it('delegates seed collection jobs and marks them completed', async () => {
const collectSeedAccounts = jest.fn().mockResolvedValue({
selectedEntries: 150,
storedAccounts: 149,
skippedAccounts: 1,
});
const updateOne = createUpdateOneMock();
const connectionService = {
getModel: jest.fn().mockResolvedValue({ updateOne }),
} as unknown as ConnectionService;
const processor = new SeedCollectionProcessor(
{
collectSeedAccounts,
} as unknown as SeedCollectionService,
connectionService,
);
await processor.process({
attemptsMade: 0,
id: 'seed-job-1',
name: SEED_COLLECTION_JOB_NAME,
opts: { attempts: 1 },
data: createSeedCollectionJobData(),
} as Job<unknown>);
expect(collectSeedAccounts).toHaveBeenCalledWith({
platformRoute: 'kr',
sourceLimits: {
challenger: 50,
grandmaster: 50,
master: 50,
},
});
expect(updateOne).toHaveBeenCalledWith(
{ jobId: 'seed-job-1' },
{
$set: {
status: 'processing',
stage: 'collect-seed-accounts',
progress: {
fetchedMatches: 0,
totalMatches: 0,
},
},
},
);
const completedUpdate = updateOne.mock.calls.find(
([, update]) => asRecord(asRecord(update).$set).status === 'completed',
);
expect(completedUpdate).toBeDefined();
expect(asRecord(asRecord(completedUpdate?.[1]).$set)).toMatchObject({
status: 'completed',
stage: 'completed',
progress: {
fetchedMatches: 149,
totalMatches: 150,
},
});
expect(
asRecord(asRecord(completedUpdate?.[1]).$set).completedAt,
).toBeInstanceOf(Date);
});
it.each([
{
data: null,
expectedError: 'Seed collection job data must be an object.',
},
{
data: {
...createSeedCollectionJobData(),
platformRoute: 'invalid',
},
expectedError: 'Seed collection job platformRoute is unsupported.',
},
{
data: {
...createSeedCollectionJobData(),
queueId: 430,
},
expectedError: 'Seed collection job queueId is unsupported.',
},
{
data: {
...createSeedCollectionJobData(),
sourceLimits: {
challenger: Number.NaN,
},
},
expectedError:
'Seed collection job sourceLimits.challenger must be a non-negative finite number.',
},
{
data: {
...createSeedCollectionJobData(),
sourceLimits: {
diamond: 50,
},
},
expectedError: 'Seed collection job sourceLimits.diamond is unsupported.',
},
])(
'rejects malformed seed collection payloads',
async ({ data, expectedError }) => {
const collectSeedAccounts = jest.fn();
const updateOne = createUpdateOneMock();
const connectionService = {
getModel: jest.fn().mockResolvedValue({ updateOne }),
} as unknown as ConnectionService;
const processor = new SeedCollectionProcessor(
{
collectSeedAccounts,
} as unknown as SeedCollectionService,
connectionService,
);
await expect(
processor.process({
attemptsMade: 0,
id: 'seed-job-1',
name: SEED_COLLECTION_JOB_NAME,
opts: { attempts: 1 },
data,
} as Job<unknown>),
).rejects.toThrow(expectedError);
expect(collectSeedAccounts).not.toHaveBeenCalled();
expect(updateOne).not.toHaveBeenCalled();
},
);
it('marks final-attempt failures as failed', async () => {
const collectSeedAccounts = jest
.fn()
.mockRejectedValue(new Error('Riot unavailable'));
const updateOne = createUpdateOneMock();
const connectionService = {
getModel: jest.fn().mockResolvedValue({ updateOne }),
} as unknown as ConnectionService;
const processor = new SeedCollectionProcessor(
{
collectSeedAccounts,
} as unknown as SeedCollectionService,
connectionService,
);
await expect(
processor.process({
attemptsMade: 0,
id: 'seed-job-1',
name: SEED_COLLECTION_JOB_NAME,
opts: { attempts: 1 },
data: createSeedCollectionJobData(),
} as Job<unknown>),
).rejects.toThrow('Riot unavailable');
expect(updateOne).toHaveBeenCalledWith(
{ jobId: 'seed-job-1' },
{
$set: {
status: 'failed',
error: 'Seed collection failed during processing.',
},
},
);
});
it('rejects unsupported job names', async () => {
const processor = new SeedCollectionProcessor(
{
collectSeedAccounts: jest.fn(),
} as unknown as SeedCollectionService,
{
getModel: jest.fn(),
} as unknown as ConnectionService,
);
await expect(
processor.process({
id: 'seed-job-1',
name: 'unexpected',
data: createSeedCollectionJobData(),
} as Job<unknown>),
).rejects.toThrow('Unsupported seed collection job name "unexpected".');
});
});
@@ -0,0 +1,159 @@
import { Processor, WorkerHost } from '@nestjs/bullmq';
import type { Job } from 'bullmq';
import { ConnectionService } from '../global/database/connection.service';
import { SEED_SOURCES, type SeedSource } from '../global/database/schemas';
import {
SEED_COLLECTION_JOB_NAME,
SEED_COLLECTION_QUEUE_NAME,
} from './analysis.constants';
import type { SeedCollectionJobData } from './analysis.types';
import {
RIOT_PLATFORM_ROUTES,
type RiotPlatformRoute,
} from '../riot/riot.types';
import { SeedCollectionService } from './seed-collection.service';
@Processor(SEED_COLLECTION_QUEUE_NAME, { concurrency: 1 })
export class SeedCollectionProcessor extends WorkerHost {
constructor(
private readonly seedCollectionService: SeedCollectionService,
private readonly connectionService: ConnectionService,
) {
super();
}
public async process(job: Job<unknown>): Promise<void> {
if (!job.id) {
throw new Error('Seed collection job is missing a BullMQ job id.');
}
if (job.name !== SEED_COLLECTION_JOB_NAME) {
throw new Error(`Unsupported seed collection job name "${job.name}".`);
}
const data = this.parseSeedCollectionJobData(job.data);
try {
await this.updateJob(job.id, {
status: 'processing',
stage: 'collect-seed-accounts',
progress: {
fetchedMatches: 0,
totalMatches: 0,
},
});
const result = await this.seedCollectionService.collectSeedAccounts({
platformRoute: data.platformRoute,
...(data.sourceLimits ? { sourceLimits: data.sourceLimits } : {}),
});
await this.updateJob(job.id, {
status: 'completed',
stage: 'completed',
progress: {
fetchedMatches: result.storedAccounts,
totalMatches: result.selectedEntries,
},
completedAt: new Date(),
});
} catch (error) {
if (this.isFinalAttempt(job)) {
await this.updateJob(job.id, {
status: 'failed',
error: this.publicErrorMessage(),
});
}
throw error;
}
}
private parseSeedCollectionJobData(value: unknown): SeedCollectionJobData {
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
throw new Error('Seed collection job data must be an object.');
}
const record = value as Record<string, unknown>;
const platformRoute = this.parsePlatformRoute(record.platformRoute);
const queueId = this.parseQueueId(record.queueId);
const sourceLimits = this.parseSourceLimits(record.sourceLimits);
return {
platformRoute,
queueId,
...(sourceLimits ? { sourceLimits } : {}),
};
}
private parsePlatformRoute(value: unknown): RiotPlatformRoute {
if (
typeof value !== 'string' ||
!RIOT_PLATFORM_ROUTES.includes(value as RiotPlatformRoute)
) {
throw new Error('Seed collection job platformRoute is unsupported.');
}
return value as RiotPlatformRoute;
}
private parseQueueId(value: unknown): number {
if (value !== 420) {
throw new Error('Seed collection job queueId is unsupported.');
}
return value;
}
private parseSourceLimits(
value: unknown,
): Partial<Record<SeedSource, number>> | undefined {
if (value === undefined) {
return undefined;
}
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
throw new Error('Seed collection job sourceLimits must be an object.');
}
const sourceLimits = value as Record<string, unknown>;
const parsedSourceLimits: Partial<Record<SeedSource, number>> = {};
for (const [source, limit] of Object.entries(sourceLimits)) {
if (!SEED_SOURCES.includes(source as SeedSource)) {
throw new Error(
`Seed collection job sourceLimits.${source} is unsupported.`,
);
}
if (typeof limit !== 'number' || !Number.isFinite(limit) || limit < 0) {
throw new Error(
`Seed collection job sourceLimits.${source} must be a non-negative finite number.`,
);
}
parsedSourceLimits[source as SeedSource] = Math.trunc(limit);
}
return parsedSourceLimits;
}
private async updateJob(
jobId: string,
updates: Record<string, unknown>,
): Promise<void> {
const analysisJobModel =
await this.connectionService.getModel('analysisJob');
await analysisJobModel
.updateOne(
{ jobId },
{
$set: updates,
},
)
.exec();
}
private isFinalAttempt(job: Job<unknown>): boolean {
const attempts =
typeof job.opts?.attempts === 'number' && job.opts.attempts > 0
? job.opts.attempts
: 1;
return job.attemptsMade + 1 >= attempts;
}
private publicErrorMessage(): string {
return 'Seed collection failed during processing.';
}
}