Wire seed collection queue
This commit is contained in:
1 parent
a6c1acbc31
commit
63a4aeace7
8 files changed
+608
-3
No files matched your search
@@ -2,7 +2,10 @@ import { getQueueToken } from '@nestjs/bullmq';
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { AppModule } from './app.module';
|
||||
import { AnalysisQueueService } from './modules/analysis/analysis-queue.service';
|
||||
import { ANALYSIS_QUEUE_NAME } from './modules/analysis/analysis.constants';
|
||||
import {
|
||||
ANALYSIS_QUEUE_NAME,
|
||||
SEED_COLLECTION_QUEUE_NAME,
|
||||
} from './modules/analysis/analysis.constants';
|
||||
import { ConnectionService } from './modules/global/database/connection.service';
|
||||
import { REDIS_CLIENT } from './modules/global/redis/redis.constants';
|
||||
import { SecretService } from './modules/global/secret/secret.service';
|
||||
@@ -36,6 +39,10 @@ describe('AppModule', () => {
|
||||
.useValue({
|
||||
add: jest.fn(),
|
||||
})
|
||||
.overrideProvider(getQueueToken(SEED_COLLECTION_QUEUE_NAME))
|
||||
.useValue({
|
||||
add: jest.fn(),
|
||||
})
|
||||
.compile();
|
||||
|
||||
expect(moduleRef.get(AnalysisQueueService)).toBeInstanceOf(
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import type { AnalysisJob } from '../global/database/schemas';
|
||||
import type { ConnectionService } from '../global/database/connection.service';
|
||||
import type { UserAnalysisJobData } from './analysis.types';
|
||||
import type {
|
||||
SeedCollectionJobData,
|
||||
UserAnalysisJobData,
|
||||
} from './analysis.types';
|
||||
import {
|
||||
DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||
FEATURE_RECOMPUTE_JOB_NAME,
|
||||
SEED_COLLECTION_JOB_NAME,
|
||||
USER_ANALYSIS_JOB_NAME,
|
||||
} from './analysis.constants';
|
||||
import { AnalysisQueueService } from './analysis-queue.service';
|
||||
@@ -37,9 +42,25 @@ function createFindOneChain(
|
||||
};
|
||||
}
|
||||
|
||||
function createSeedCollectionJobData(
|
||||
overrides: Partial<SeedCollectionJobData> = {},
|
||||
): SeedCollectionJobData {
|
||||
return {
|
||||
platformRoute: 'kr',
|
||||
queueId: 420,
|
||||
sourceLimits: {
|
||||
challenger: 50,
|
||||
grandmaster: 50,
|
||||
master: 50,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('AnalysisQueueService', () => {
|
||||
it('creates an analysis job document and enqueues a BullMQ job', async () => {
|
||||
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
||||
const seedAdd = jest.fn();
|
||||
const findOneChain = createFindOneChain(null);
|
||||
const create = jest.fn().mockResolvedValue({});
|
||||
const updateOne = jest.fn(() => ({
|
||||
@@ -55,6 +76,7 @@ describe('AnalysisQueueService', () => {
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add } as never,
|
||||
{ add: seedAdd } as never,
|
||||
connectionService,
|
||||
() => 'analysis-job-1',
|
||||
);
|
||||
@@ -79,10 +101,12 @@ describe('AnalysisQueueService', () => {
|
||||
jobId: 'analysis-job-1',
|
||||
}),
|
||||
);
|
||||
expect(seedAdd).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('creates a feature recompute job document and enqueues a BullMQ job', async () => {
|
||||
const add = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
||||
const seedAdd = jest.fn();
|
||||
const create = jest.fn().mockResolvedValue({});
|
||||
const updateOne = jest.fn(() => ({
|
||||
exec: jest.fn().mockResolvedValue({}),
|
||||
@@ -96,6 +120,7 @@ describe('AnalysisQueueService', () => {
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add } as never,
|
||||
{ add: seedAdd } as never,
|
||||
connectionService,
|
||||
() => 'recompute-job-1',
|
||||
);
|
||||
@@ -132,6 +157,54 @@ describe('AnalysisQueueService', () => {
|
||||
jobId: 'recompute-job-1',
|
||||
}),
|
||||
);
|
||||
expect(seedAdd).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('creates a seed collection job document and enqueues the seed queue', async () => {
|
||||
const analysisAdd = jest.fn();
|
||||
const seedAdd = jest.fn().mockResolvedValue({ id: 'job-from-bull' });
|
||||
const create = jest.fn().mockResolvedValue({});
|
||||
const updateOne = jest.fn(() => ({
|
||||
exec: jest.fn().mockResolvedValue({}),
|
||||
}));
|
||||
const analysisJobModel = {
|
||||
create,
|
||||
updateOne,
|
||||
};
|
||||
const connectionService = {
|
||||
getModel: jest.fn().mockResolvedValue(analysisJobModel),
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add: analysisAdd } as never,
|
||||
{ add: seedAdd } as never,
|
||||
connectionService,
|
||||
() => 'seed-job-1',
|
||||
);
|
||||
|
||||
const result = await service.enqueueSeedCollection(
|
||||
createSeedCollectionJobData(),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ jobId: 'seed-job-1', status: 'queued' });
|
||||
expect(create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
jobId: 'seed-job-1',
|
||||
jobType: SEED_COLLECTION_JOB_NAME,
|
||||
platformRoute: 'kr',
|
||||
queueId: 420,
|
||||
requestedMatchCount: DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||
status: 'queued',
|
||||
}),
|
||||
);
|
||||
expect(seedAdd).toHaveBeenCalledWith(
|
||||
SEED_COLLECTION_JOB_NAME,
|
||||
createSeedCollectionJobData(),
|
||||
expect.objectContaining({
|
||||
attempts: 1,
|
||||
jobId: 'seed-job-1',
|
||||
}),
|
||||
);
|
||||
expect(analysisAdd).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns an existing active job instead of enqueueing a duplicate', async () => {
|
||||
@@ -149,6 +222,7 @@ describe('AnalysisQueueService', () => {
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add } as never,
|
||||
{ add: jest.fn() } as never,
|
||||
connectionService,
|
||||
() => 'new-job',
|
||||
);
|
||||
@@ -180,6 +254,7 @@ describe('AnalysisQueueService', () => {
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add } as never,
|
||||
{ add: jest.fn() } as never,
|
||||
connectionService,
|
||||
() => 'analysis-job-1',
|
||||
);
|
||||
@@ -214,6 +289,7 @@ describe('AnalysisQueueService', () => {
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add } as never,
|
||||
{ add: jest.fn() } as never,
|
||||
connectionService,
|
||||
() => 'recompute-job-1',
|
||||
);
|
||||
@@ -237,6 +313,43 @@ describe('AnalysisQueueService', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('marks a pre-created seed collection job failed when BullMQ enqueue fails', async () => {
|
||||
const analysisAdd = jest.fn();
|
||||
const seedAdd = jest.fn().mockRejectedValue(new Error('redis unavailable'));
|
||||
const create = jest.fn().mockResolvedValue({});
|
||||
const updateOne = jest.fn(() => ({
|
||||
exec: jest.fn().mockResolvedValue({}),
|
||||
}));
|
||||
const analysisJobModel = {
|
||||
create,
|
||||
updateOne,
|
||||
};
|
||||
const connectionService = {
|
||||
getModel: jest.fn().mockResolvedValue(analysisJobModel),
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add: analysisAdd } as never,
|
||||
{ add: seedAdd } as never,
|
||||
connectionService,
|
||||
() => 'seed-job-1',
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.enqueueSeedCollection(createSeedCollectionJobData()),
|
||||
).rejects.toThrow('redis unavailable');
|
||||
expect(updateOne).toHaveBeenCalledWith(
|
||||
{ jobId: 'seed-job-1' },
|
||||
{
|
||||
$set: {
|
||||
status: 'failed',
|
||||
stage: 'enqueue',
|
||||
error: 'Analysis could not be queued.',
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(analysisAdd).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('serializes identical in-process requests before creating a duplicate job', async () => {
|
||||
const add = jest.fn().mockResolvedValue({ id: 'analysis-job-1' });
|
||||
let activeJob: Pick<AnalysisJob, 'jobId' | 'status'> | null = null;
|
||||
@@ -263,6 +376,7 @@ describe('AnalysisQueueService', () => {
|
||||
} as unknown as ConnectionService;
|
||||
const service = new AnalysisQueueService(
|
||||
{ add } as never,
|
||||
{ add: jest.fn() } as never,
|
||||
connectionService,
|
||||
() => (create.mock.calls.length === 0 ? 'analysis-job-1' : 'unexpected'),
|
||||
);
|
||||
|
||||
@@ -4,18 +4,22 @@ import type { AnalysisRequestAcceptedDto } from '@persona-lenz/shared';
|
||||
import type { Queue } from 'bullmq';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import type { AnalysisJob } from '../global/database/schemas';
|
||||
import type { AnalysisJob, SeedSource } from '../global/database/schemas';
|
||||
import {
|
||||
ANALYSIS_JOB_ID_FACTORY,
|
||||
ACTIVE_ANALYSIS_JOB_STATUSES,
|
||||
ANALYSIS_QUEUE_NAME,
|
||||
DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||
FEATURE_RECOMPUTE_JOB_NAME,
|
||||
normalizeFeatureRecomputeBatchSize,
|
||||
normalizeRequestedMatchCount,
|
||||
SEED_COLLECTION_JOB_NAME,
|
||||
SEED_COLLECTION_QUEUE_NAME,
|
||||
USER_ANALYSIS_JOB_NAME,
|
||||
} from './analysis.constants';
|
||||
import type {
|
||||
FeatureRecomputeJobData,
|
||||
SeedCollectionJobData,
|
||||
UserAnalysisJobData,
|
||||
} from './analysis.types';
|
||||
|
||||
@@ -23,6 +27,7 @@ type JobIdFactory = () => string;
|
||||
type AnalysisQueueJobData = UserAnalysisJobData | FeatureRecomputeJobData;
|
||||
type AnalysisQueueJobName =
|
||||
typeof USER_ANALYSIS_JOB_NAME | typeof FEATURE_RECOMPUTE_JOB_NAME;
|
||||
type SeedCollectionQueueJobName = typeof SEED_COLLECTION_JOB_NAME;
|
||||
type ActiveAnalysisJob = Pick<AnalysisJob, 'jobId'> & {
|
||||
status: Extract<AnalysisJob['status'], 'queued' | 'processing'>;
|
||||
};
|
||||
@@ -36,6 +41,12 @@ export class AnalysisQueueService {
|
||||
void,
|
||||
AnalysisQueueJobName
|
||||
>,
|
||||
@InjectQueue(SEED_COLLECTION_QUEUE_NAME)
|
||||
private readonly seedCollectionQueue: Queue<
|
||||
SeedCollectionJobData,
|
||||
void,
|
||||
SeedCollectionQueueJobName
|
||||
>,
|
||||
private readonly connectionService: ConnectionService,
|
||||
@Inject(ANALYSIS_JOB_ID_FACTORY)
|
||||
private readonly jobIdFactory: JobIdFactory = randomUUID,
|
||||
@@ -115,6 +126,59 @@ export class AnalysisQueueService {
|
||||
};
|
||||
}
|
||||
|
||||
public async enqueueSeedCollection(
|
||||
data: SeedCollectionJobData,
|
||||
): Promise<AnalysisRequestAcceptedDto> {
|
||||
const normalizedData = this.normalizeSeedCollectionData(data);
|
||||
const analysisJobModel =
|
||||
await this.connectionService.getModel('analysisJob');
|
||||
const jobId = this.jobIdFactory();
|
||||
|
||||
await analysisJobModel.create({
|
||||
jobId,
|
||||
jobType: SEED_COLLECTION_JOB_NAME,
|
||||
platformRoute: normalizedData.platformRoute,
|
||||
queueId: normalizedData.queueId,
|
||||
requestedMatchCount: DEFAULT_SEED_COLLECTION_MATCH_COUNT,
|
||||
status: 'queued',
|
||||
stage: 'queued',
|
||||
progress: {
|
||||
fetchedMatches: 0,
|
||||
totalMatches: 0,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await this.seedCollectionQueue.add(
|
||||
SEED_COLLECTION_JOB_NAME,
|
||||
normalizedData,
|
||||
{
|
||||
attempts: 1,
|
||||
jobId,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
await analysisJobModel
|
||||
.updateOne(
|
||||
{ jobId },
|
||||
{
|
||||
$set: {
|
||||
status: 'failed',
|
||||
stage: 'enqueue',
|
||||
error: this.publicErrorMessage(),
|
||||
},
|
||||
},
|
||||
)
|
||||
.exec();
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
jobId,
|
||||
status: 'queued',
|
||||
};
|
||||
}
|
||||
|
||||
private readonly pendingDuplicateRequestKeys = new Set<string>();
|
||||
|
||||
private async enqueueUserAnalysisWithoutDuplicateRequestRace(
|
||||
@@ -218,6 +282,30 @@ export class AnalysisQueueService {
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeSeedCollectionData(
|
||||
data: SeedCollectionJobData,
|
||||
): SeedCollectionJobData {
|
||||
return {
|
||||
platformRoute: data.platformRoute,
|
||||
queueId: data.queueId,
|
||||
...(data.sourceLimits
|
||||
? { sourceLimits: this.normalizeSeedSourceLimits(data.sourceLimits) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeSeedSourceLimits(
|
||||
sourceLimits: NonNullable<SeedCollectionJobData['sourceLimits']>,
|
||||
): NonNullable<SeedCollectionJobData['sourceLimits']> {
|
||||
const normalizedSourceLimits: Partial<Record<SeedSource, number>> = {};
|
||||
for (const [source, limit] of Object.entries(sourceLimits)) {
|
||||
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
|
||||
normalizedSourceLimits[source as SeedSource] = Math.trunc(limit);
|
||||
}
|
||||
}
|
||||
return normalizedSourceLimits;
|
||||
}
|
||||
|
||||
private duplicateRequestKey(data: UserAnalysisJobData): string {
|
||||
return [
|
||||
data.platformRoute,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
export const ANALYSIS_QUEUE_NAME = 'analysis';
|
||||
export const SEED_COLLECTION_QUEUE_NAME = 'seed-collection';
|
||||
export const USER_ANALYSIS_JOB_NAME = 'user-analysis';
|
||||
export const FEATURE_RECOMPUTE_JOB_NAME = 'feature-recompute';
|
||||
export const SEED_COLLECTION_JOB_NAME = 'seed-collection';
|
||||
export const ANALYSIS_JOB_ID_FACTORY = Symbol('ANALYSIS_JOB_ID_FACTORY');
|
||||
|
||||
export const DEFAULT_ANALYSIS_MATCH_COUNT = 20;
|
||||
export const MAX_ANALYSIS_MATCH_COUNT = 50;
|
||||
export const DEFAULT_SEED_COLLECTION_MATCH_COUNT = 30;
|
||||
export const DEFAULT_SEED_COLLECTION_QUEUE_TYPE = 'RANKED_SOLO_5x5';
|
||||
export const DEFAULT_SEED_COLLECTION_SOURCE_LIMIT = 50;
|
||||
export const DEFAULT_FEATURE_RECOMPUTE_BATCH_SIZE = 100;
|
||||
|
||||
@@ -12,12 +12,14 @@ import { AnalysisQueueService } from './analysis-queue.service';
|
||||
import {
|
||||
ANALYSIS_JOB_ID_FACTORY,
|
||||
ANALYSIS_QUEUE_NAME,
|
||||
SEED_COLLECTION_QUEUE_NAME,
|
||||
} from './analysis.constants';
|
||||
import { AnalysisPipelineService } from './analysis-pipeline.service';
|
||||
import { AnalysisProcessor } from './analysis.processor';
|
||||
import { createBullMqOptions } from './bullmq.config';
|
||||
import { FeatureExtractorService } from './feature-extractor.service';
|
||||
import { ProfileBuilderService } from './profile-builder.service';
|
||||
import { SeedCollectionProcessor } from './seed-collection.processor';
|
||||
import { SeedCollectionService } from './seed-collection.service';
|
||||
import { TimelineDerivedService } from './timeline-derived.service';
|
||||
import { TimelineMetricService } from './timeline-metric.service';
|
||||
@@ -38,6 +40,9 @@ import { TimelineMetricService } from './timeline-metric.service';
|
||||
BullModule.registerQueue({
|
||||
name: ANALYSIS_QUEUE_NAME,
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: SEED_COLLECTION_QUEUE_NAME,
|
||||
}),
|
||||
],
|
||||
providers: [
|
||||
{
|
||||
@@ -49,6 +54,7 @@ import { TimelineMetricService } from './timeline-metric.service';
|
||||
AnalysisRequestThrottleService,
|
||||
AnalysisPipelineService,
|
||||
AnalysisProcessor,
|
||||
SeedCollectionProcessor,
|
||||
FeatureExtractorService,
|
||||
ProfileBuilderService,
|
||||
SeedCollectionService,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Role } from '@persona-lenz/shared';
|
||||
import type {
|
||||
LaneOpponentRef,
|
||||
SeedSource,
|
||||
TeamPositionOrUnknown,
|
||||
} from '../global/database/schemas';
|
||||
import type { RiotPlatformRoute, RiotRegionalRoute } from '../riot/riot.types';
|
||||
@@ -21,6 +22,12 @@ export interface FeatureRecomputeJobData {
|
||||
puuid?: string;
|
||||
}
|
||||
|
||||
export interface SeedCollectionJobData {
|
||||
platformRoute: RiotPlatformRoute;
|
||||
queueId: number;
|
||||
sourceLimits?: Partial<Record<SeedSource, number>>;
|
||||
}
|
||||
|
||||
export interface ExtractedPlayerMatchFeature {
|
||||
matchId: string;
|
||||
puuid: string;
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import type { Job } from 'bullmq';
|
||||
import type { ConnectionService } from '../global/database/connection.service';
|
||||
import type { SeedCollectionService } from './seed-collection.service';
|
||||
import { SEED_COLLECTION_JOB_NAME } from './analysis.constants';
|
||||
import type { SeedCollectionJobData } from './analysis.types';
|
||||
import { SeedCollectionProcessor } from './seed-collection.processor';
|
||||
|
||||
function createSeedCollectionJobData(
|
||||
overrides: Partial<SeedCollectionJobData> = {},
|
||||
): SeedCollectionJobData {
|
||||
return {
|
||||
platformRoute: 'kr',
|
||||
queueId: 420,
|
||||
sourceLimits: {
|
||||
challenger: 50,
|
||||
grandmaster: 50,
|
||||
master: 50,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createUpdateOneMock() {
|
||||
return jest.fn(() => ({
|
||||
exec: jest.fn().mockResolvedValue({ acknowledged: true }),
|
||||
}));
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
describe('SeedCollectionProcessor', () => {
|
||||
it('delegates seed collection jobs and marks them completed', async () => {
|
||||
const collectSeedAccounts = jest.fn().mockResolvedValue({
|
||||
selectedEntries: 150,
|
||||
storedAccounts: 149,
|
||||
skippedAccounts: 1,
|
||||
});
|
||||
const updateOne = createUpdateOneMock();
|
||||
const connectionService = {
|
||||
getModel: jest.fn().mockResolvedValue({ updateOne }),
|
||||
} as unknown as ConnectionService;
|
||||
const processor = new SeedCollectionProcessor(
|
||||
{
|
||||
collectSeedAccounts,
|
||||
} as unknown as SeedCollectionService,
|
||||
connectionService,
|
||||
);
|
||||
|
||||
await processor.process({
|
||||
attemptsMade: 0,
|
||||
id: 'seed-job-1',
|
||||
name: SEED_COLLECTION_JOB_NAME,
|
||||
opts: { attempts: 1 },
|
||||
data: createSeedCollectionJobData(),
|
||||
} as Job<unknown>);
|
||||
|
||||
expect(collectSeedAccounts).toHaveBeenCalledWith({
|
||||
platformRoute: 'kr',
|
||||
sourceLimits: {
|
||||
challenger: 50,
|
||||
grandmaster: 50,
|
||||
master: 50,
|
||||
},
|
||||
});
|
||||
expect(updateOne).toHaveBeenCalledWith(
|
||||
{ jobId: 'seed-job-1' },
|
||||
{
|
||||
$set: {
|
||||
status: 'processing',
|
||||
stage: 'collect-seed-accounts',
|
||||
progress: {
|
||||
fetchedMatches: 0,
|
||||
totalMatches: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
const completedUpdate = updateOne.mock.calls.find(
|
||||
([, update]) => asRecord(asRecord(update).$set).status === 'completed',
|
||||
);
|
||||
expect(completedUpdate).toBeDefined();
|
||||
expect(asRecord(asRecord(completedUpdate?.[1]).$set)).toMatchObject({
|
||||
status: 'completed',
|
||||
stage: 'completed',
|
||||
progress: {
|
||||
fetchedMatches: 149,
|
||||
totalMatches: 150,
|
||||
},
|
||||
});
|
||||
expect(
|
||||
asRecord(asRecord(completedUpdate?.[1]).$set).completedAt,
|
||||
).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
data: null,
|
||||
expectedError: 'Seed collection job data must be an object.',
|
||||
},
|
||||
{
|
||||
data: {
|
||||
...createSeedCollectionJobData(),
|
||||
platformRoute: 'invalid',
|
||||
},
|
||||
expectedError: 'Seed collection job platformRoute is unsupported.',
|
||||
},
|
||||
{
|
||||
data: {
|
||||
...createSeedCollectionJobData(),
|
||||
queueId: 430,
|
||||
},
|
||||
expectedError: 'Seed collection job queueId is unsupported.',
|
||||
},
|
||||
{
|
||||
data: {
|
||||
...createSeedCollectionJobData(),
|
||||
sourceLimits: {
|
||||
challenger: Number.NaN,
|
||||
},
|
||||
},
|
||||
expectedError:
|
||||
'Seed collection job sourceLimits.challenger must be a non-negative finite number.',
|
||||
},
|
||||
{
|
||||
data: {
|
||||
...createSeedCollectionJobData(),
|
||||
sourceLimits: {
|
||||
diamond: 50,
|
||||
},
|
||||
},
|
||||
expectedError: 'Seed collection job sourceLimits.diamond is unsupported.',
|
||||
},
|
||||
])(
|
||||
'rejects malformed seed collection payloads',
|
||||
async ({ data, expectedError }) => {
|
||||
const collectSeedAccounts = jest.fn();
|
||||
const updateOne = createUpdateOneMock();
|
||||
const connectionService = {
|
||||
getModel: jest.fn().mockResolvedValue({ updateOne }),
|
||||
} as unknown as ConnectionService;
|
||||
const processor = new SeedCollectionProcessor(
|
||||
{
|
||||
collectSeedAccounts,
|
||||
} as unknown as SeedCollectionService,
|
||||
connectionService,
|
||||
);
|
||||
|
||||
await expect(
|
||||
processor.process({
|
||||
attemptsMade: 0,
|
||||
id: 'seed-job-1',
|
||||
name: SEED_COLLECTION_JOB_NAME,
|
||||
opts: { attempts: 1 },
|
||||
data,
|
||||
} as Job<unknown>),
|
||||
).rejects.toThrow(expectedError);
|
||||
expect(collectSeedAccounts).not.toHaveBeenCalled();
|
||||
expect(updateOne).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('marks final-attempt failures as failed', async () => {
|
||||
const collectSeedAccounts = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('Riot unavailable'));
|
||||
const updateOne = createUpdateOneMock();
|
||||
const connectionService = {
|
||||
getModel: jest.fn().mockResolvedValue({ updateOne }),
|
||||
} as unknown as ConnectionService;
|
||||
const processor = new SeedCollectionProcessor(
|
||||
{
|
||||
collectSeedAccounts,
|
||||
} as unknown as SeedCollectionService,
|
||||
connectionService,
|
||||
);
|
||||
|
||||
await expect(
|
||||
processor.process({
|
||||
attemptsMade: 0,
|
||||
id: 'seed-job-1',
|
||||
name: SEED_COLLECTION_JOB_NAME,
|
||||
opts: { attempts: 1 },
|
||||
data: createSeedCollectionJobData(),
|
||||
} as Job<unknown>),
|
||||
).rejects.toThrow('Riot unavailable');
|
||||
|
||||
expect(updateOne).toHaveBeenCalledWith(
|
||||
{ jobId: 'seed-job-1' },
|
||||
{
|
||||
$set: {
|
||||
status: 'failed',
|
||||
error: 'Seed collection failed during processing.',
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects unsupported job names', async () => {
|
||||
const processor = new SeedCollectionProcessor(
|
||||
{
|
||||
collectSeedAccounts: jest.fn(),
|
||||
} as unknown as SeedCollectionService,
|
||||
{
|
||||
getModel: jest.fn(),
|
||||
} as unknown as ConnectionService,
|
||||
);
|
||||
|
||||
await expect(
|
||||
processor.process({
|
||||
id: 'seed-job-1',
|
||||
name: 'unexpected',
|
||||
data: createSeedCollectionJobData(),
|
||||
} as Job<unknown>),
|
||||
).rejects.toThrow('Unsupported seed collection job name "unexpected".');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
import { Processor, WorkerHost } from '@nestjs/bullmq';
|
||||
import type { Job } from 'bullmq';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { SEED_SOURCES, type SeedSource } from '../global/database/schemas';
|
||||
import {
|
||||
SEED_COLLECTION_JOB_NAME,
|
||||
SEED_COLLECTION_QUEUE_NAME,
|
||||
} from './analysis.constants';
|
||||
import type { SeedCollectionJobData } from './analysis.types';
|
||||
import {
|
||||
RIOT_PLATFORM_ROUTES,
|
||||
type RiotPlatformRoute,
|
||||
} from '../riot/riot.types';
|
||||
import { SeedCollectionService } from './seed-collection.service';
|
||||
|
||||
@Processor(SEED_COLLECTION_QUEUE_NAME, { concurrency: 1 })
|
||||
export class SeedCollectionProcessor extends WorkerHost {
|
||||
constructor(
|
||||
private readonly seedCollectionService: SeedCollectionService,
|
||||
private readonly connectionService: ConnectionService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public async process(job: Job<unknown>): Promise<void> {
|
||||
if (!job.id) {
|
||||
throw new Error('Seed collection job is missing a BullMQ job id.');
|
||||
}
|
||||
if (job.name !== SEED_COLLECTION_JOB_NAME) {
|
||||
throw new Error(`Unsupported seed collection job name "${job.name}".`);
|
||||
}
|
||||
|
||||
const data = this.parseSeedCollectionJobData(job.data);
|
||||
try {
|
||||
await this.updateJob(job.id, {
|
||||
status: 'processing',
|
||||
stage: 'collect-seed-accounts',
|
||||
progress: {
|
||||
fetchedMatches: 0,
|
||||
totalMatches: 0,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await this.seedCollectionService.collectSeedAccounts({
|
||||
platformRoute: data.platformRoute,
|
||||
...(data.sourceLimits ? { sourceLimits: data.sourceLimits } : {}),
|
||||
});
|
||||
|
||||
await this.updateJob(job.id, {
|
||||
status: 'completed',
|
||||
stage: 'completed',
|
||||
progress: {
|
||||
fetchedMatches: result.storedAccounts,
|
||||
totalMatches: result.selectedEntries,
|
||||
},
|
||||
completedAt: new Date(),
|
||||
});
|
||||
} catch (error) {
|
||||
if (this.isFinalAttempt(job)) {
|
||||
await this.updateJob(job.id, {
|
||||
status: 'failed',
|
||||
error: this.publicErrorMessage(),
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private parseSeedCollectionJobData(value: unknown): SeedCollectionJobData {
|
||||
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||
throw new Error('Seed collection job data must be an object.');
|
||||
}
|
||||
|
||||
const record = value as Record<string, unknown>;
|
||||
const platformRoute = this.parsePlatformRoute(record.platformRoute);
|
||||
const queueId = this.parseQueueId(record.queueId);
|
||||
const sourceLimits = this.parseSourceLimits(record.sourceLimits);
|
||||
|
||||
return {
|
||||
platformRoute,
|
||||
queueId,
|
||||
...(sourceLimits ? { sourceLimits } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
private parsePlatformRoute(value: unknown): RiotPlatformRoute {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!RIOT_PLATFORM_ROUTES.includes(value as RiotPlatformRoute)
|
||||
) {
|
||||
throw new Error('Seed collection job platformRoute is unsupported.');
|
||||
}
|
||||
return value as RiotPlatformRoute;
|
||||
}
|
||||
|
||||
private parseQueueId(value: unknown): number {
|
||||
if (value !== 420) {
|
||||
throw new Error('Seed collection job queueId is unsupported.');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private parseSourceLimits(
|
||||
value: unknown,
|
||||
): Partial<Record<SeedSource, number>> | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||
throw new Error('Seed collection job sourceLimits must be an object.');
|
||||
}
|
||||
|
||||
const sourceLimits = value as Record<string, unknown>;
|
||||
const parsedSourceLimits: Partial<Record<SeedSource, number>> = {};
|
||||
for (const [source, limit] of Object.entries(sourceLimits)) {
|
||||
if (!SEED_SOURCES.includes(source as SeedSource)) {
|
||||
throw new Error(
|
||||
`Seed collection job sourceLimits.${source} is unsupported.`,
|
||||
);
|
||||
}
|
||||
if (typeof limit !== 'number' || !Number.isFinite(limit) || limit < 0) {
|
||||
throw new Error(
|
||||
`Seed collection job sourceLimits.${source} must be a non-negative finite number.`,
|
||||
);
|
||||
}
|
||||
parsedSourceLimits[source as SeedSource] = Math.trunc(limit);
|
||||
}
|
||||
|
||||
return parsedSourceLimits;
|
||||
}
|
||||
|
||||
private async updateJob(
|
||||
jobId: string,
|
||||
updates: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
const analysisJobModel =
|
||||
await this.connectionService.getModel('analysisJob');
|
||||
await analysisJobModel
|
||||
.updateOne(
|
||||
{ jobId },
|
||||
{
|
||||
$set: updates,
|
||||
},
|
||||
)
|
||||
.exec();
|
||||
}
|
||||
|
||||
private isFinalAttempt(job: Job<unknown>): boolean {
|
||||
const attempts =
|
||||
typeof job.opts?.attempts === 'number' && job.opts.attempts > 0
|
||||
? job.opts.attempts
|
||||
: 1;
|
||||
return job.attemptsMade + 1 >= attempts;
|
||||
}
|
||||
|
||||
private publicErrorMessage(): string {
|
||||
return 'Seed collection failed during processing.';
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user