fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -4,10 +4,13 @@ import {
|
|||||||
OnModuleDestroy,
|
OnModuleDestroy,
|
||||||
OnModuleInit,
|
OnModuleInit,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import { DakaoOutboxService } from './dakao-outbox.service';
|
import { DakaoOutboxService } from './dakao-outbox.service';
|
||||||
|
|
||||||
const SWEEP_INTERVAL_MS = 60 * 1000;
|
const SWEEP_INTERVAL_MS = 60 * 1000;
|
||||||
const STALE_OUTBOX_MS = 5 * 60 * 1000;
|
const STALE_OUTBOX_MS = 5 * 60 * 1000;
|
||||||
|
const SWEEP_LOCK_KEY = 'distributed-lock:dakao-outbox:sweep-stale-publish-leases';
|
||||||
|
const SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS = 55_000;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class DakaoOutboxSweeperService
|
export class DakaoOutboxSweeperService
|
||||||
@@ -16,7 +19,10 @@ export class DakaoOutboxSweeperService
|
|||||||
private readonly logger = new Logger(DakaoOutboxSweeperService.name);
|
private readonly logger = new Logger(DakaoOutboxSweeperService.name);
|
||||||
private timer: NodeJS.Timeout | undefined;
|
private timer: NodeJS.Timeout | undefined;
|
||||||
|
|
||||||
constructor(private readonly outboxService: DakaoOutboxService) {}
|
constructor(
|
||||||
|
private readonly outboxService: DakaoOutboxService,
|
||||||
|
private readonly distributedLockService: DistributedLockService,
|
||||||
|
) {}
|
||||||
|
|
||||||
onModuleInit() {
|
onModuleInit() {
|
||||||
this.timer = setInterval(() => {
|
this.timer = setInterval(() => {
|
||||||
@@ -31,12 +37,23 @@ export class DakaoOutboxSweeperService
|
|||||||
|
|
||||||
private async sweep(): Promise<void> {
|
private async sweep(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const swept = await this.outboxService.sweepStale({
|
const swept = await this.distributedLockService.runWithLock(
|
||||||
staleMs: STALE_OUTBOX_MS,
|
{
|
||||||
limit: 20,
|
lockKey: SWEEP_LOCK_KEY,
|
||||||
});
|
lockExpirationDurationMilliseconds:
|
||||||
if (swept > 0) {
|
SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
this.logger.warn(`Re-published ${swept} stale Dakao outbox messages.`);
|
},
|
||||||
|
async () =>
|
||||||
|
this.outboxService.sweepStale({
|
||||||
|
staleMs: STALE_OUTBOX_MS,
|
||||||
|
limit: 20,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const sweptCount = swept ?? 0;
|
||||||
|
if (sweptCount > 0) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Re-published ${sweptCount} stale Dakao outbox messages.`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
|
|||||||
@@ -70,44 +70,55 @@ describe('DakaoOutboxService', () => {
|
|||||||
process.env.NODE_ENV = 'staging';
|
process.env.NODE_ENV = 'staging';
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const exec = jest.fn().mockResolvedValue([
|
const staleRow = {
|
||||||
{
|
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||||
eventId: 'lolympus:chat.send-message:chat-log-1',
|
type: 'chat.send-message',
|
||||||
type: 'chat.send-message',
|
eventSource: 'lolympus',
|
||||||
eventSource: 'lolympus',
|
subject: 'staging.dakao.chat.send-message.v1',
|
||||||
subject: 'staging.dakao.chat.send-message.v1',
|
correlationId: 'chat-log-1',
|
||||||
correlationId: 'chat-log-1',
|
payload: {
|
||||||
payload: {
|
chatroomId: '18474390111480678',
|
||||||
chatroomId: '18474390111480678',
|
segments: [{ kind: 'text', text: 'done' }],
|
||||||
segments: [{ kind: 'text', text: 'done' }],
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
]);
|
};
|
||||||
|
const exec = jest.fn().mockResolvedValue([staleRow]);
|
||||||
const lean = jest.fn().mockReturnValue({ exec });
|
const lean = jest.fn().mockReturnValue({ exec });
|
||||||
const limit = jest.fn().mockReturnValue({ lean });
|
const limit = jest.fn().mockReturnValue({ lean });
|
||||||
const sort = jest.fn().mockReturnValue({ limit });
|
const sort = jest.fn().mockReturnValue({ limit });
|
||||||
|
const claimExec = jest.fn().mockResolvedValue(staleRow);
|
||||||
|
const claimLean = jest.fn().mockReturnValue({ exec: claimExec });
|
||||||
const outboxModel = {
|
const outboxModel = {
|
||||||
find: jest.fn().mockReturnValue({ sort }),
|
find: jest.fn().mockReturnValue({ sort }),
|
||||||
|
findOneAndUpdate: jest.fn().mockReturnValue({ lean: claimLean }),
|
||||||
|
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
|
||||||
};
|
};
|
||||||
const { service } = makeService(outboxModel);
|
const { service, natsService } = makeService(outboxModel);
|
||||||
const enqueue = jest
|
|
||||||
.spyOn(service, 'enqueueAndPublish')
|
|
||||||
.mockResolvedValue(undefined);
|
|
||||||
|
|
||||||
await expect(service.sweepStale({ staleMs: 1, limit: 1 })).resolves.toBe(
|
await expect(service.sweepStale({ staleMs: 1, limit: 1 })).resolves.toBe(
|
||||||
1,
|
1,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(enqueue).toHaveBeenCalledWith(
|
expect(outboxModel.findOneAndUpdate).toHaveBeenCalledWith(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
envelope: expect.objectContaining({
|
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||||
resultSubjects: {
|
}),
|
||||||
succeeded:
|
expect.objectContaining({
|
||||||
'staging.lolympus.dakao.chat.send-message.succeeded.v1',
|
$set: expect.objectContaining({
|
||||||
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
|
publishLeaseOwnerIdentifier: expect.any(String),
|
||||||
},
|
publishLeaseExpiresAt: expect.any(Date),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
|
{ returnDocument: 'after' },
|
||||||
|
);
|
||||||
|
expect(natsService.publishJetStream).toHaveBeenCalledWith(
|
||||||
|
'staging.dakao.chat.send-message.v1',
|
||||||
|
expect.objectContaining({
|
||||||
|
resultSubjects: {
|
||||||
|
succeeded: 'staging.lolympus.dakao.chat.send-message.succeeded.v1',
|
||||||
|
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
{ messageId: 'lolympus:chat.send-message:chat-log-1' },
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
process.env.NODE_ENV = previousNodeEnv;
|
process.env.NODE_ENV = previousNodeEnv;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { Injectable, Logger } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
import { ConnectionService } from '../global/database/connection.service';
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
import { NatsService } from '../global/nats/nats.service';
|
import { NatsService } from '../global/nats/nats.service';
|
||||||
import { resolveDakaoNatsNames } from './dakao.nats-names';
|
import { resolveDakaoNatsNames } from './dakao.nats-names';
|
||||||
@@ -10,6 +11,7 @@ import {
|
|||||||
|
|
||||||
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
|
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
|
||||||
const TERMINAL_STATES = ['sent', 'failed'] as const;
|
const TERMINAL_STATES = ['sent', 'failed'] as const;
|
||||||
|
const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class DakaoOutboxService {
|
export class DakaoOutboxService {
|
||||||
@@ -50,35 +52,59 @@ export class DakaoOutboxService {
|
|||||||
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
|
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
|
||||||
).exec();
|
).exec();
|
||||||
|
|
||||||
|
await this.publishCommand({
|
||||||
|
subject: params.subject,
|
||||||
|
eventId: params.eventId,
|
||||||
|
envelope: params.envelope,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async publishCommand<TPayload>(params: {
|
||||||
|
subject: string;
|
||||||
|
eventId: string;
|
||||||
|
envelope: DakaoCommandEnvelope<TPayload>;
|
||||||
|
publishLeaseOwnerIdentifier?: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
const OutboxModel =
|
||||||
|
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||||
|
const updateFilter = {
|
||||||
|
eventId: params.eventId,
|
||||||
|
state: { $nin: TERMINAL_STATES },
|
||||||
|
...(params.publishLeaseOwnerIdentifier
|
||||||
|
? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier }
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
try {
|
try {
|
||||||
await this.natsService.publishJetStream(params.subject, params.envelope, {
|
await this.natsService.publishJetStream(params.subject, params.envelope, {
|
||||||
messageId: params.eventId,
|
messageId: params.eventId,
|
||||||
});
|
});
|
||||||
await OutboxModel.updateOne(
|
await OutboxModel.updateOne(
|
||||||
{
|
updateFilter,
|
||||||
eventId: params.eventId,
|
|
||||||
state: { $nin: TERMINAL_STATES },
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
$set: {
|
$set: {
|
||||||
state: 'published',
|
state: 'published',
|
||||||
publishedAt: new Date(),
|
publishedAt: new Date(),
|
||||||
},
|
},
|
||||||
$inc: { publishAttempts: 1 },
|
$inc: { publishAttempts: 1 },
|
||||||
$unset: { lastPublishError: '' },
|
$unset: {
|
||||||
|
lastPublishError: '',
|
||||||
|
publishLeaseOwnerIdentifier: '',
|
||||||
|
publishLeaseExpiresAt: '',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
).exec();
|
).exec();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
await OutboxModel.updateOne(
|
await OutboxModel.updateOne(
|
||||||
{
|
updateFilter,
|
||||||
eventId: params.eventId,
|
|
||||||
state: { $nin: TERMINAL_STATES },
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
$inc: { publishAttempts: 1 },
|
$inc: { publishAttempts: 1 },
|
||||||
$set: {
|
$set: {
|
||||||
lastPublishError: (err as Error)?.message ?? String(err),
|
lastPublishError: (err as Error)?.message ?? String(err),
|
||||||
},
|
},
|
||||||
|
$unset: {
|
||||||
|
publishLeaseOwnerIdentifier: '',
|
||||||
|
publishLeaseExpiresAt: '',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
).exec();
|
).exec();
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
@@ -146,40 +172,77 @@ export class DakaoOutboxService {
|
|||||||
|
|
||||||
public async sweepStale(params?: {
|
public async sweepStale(params?: {
|
||||||
staleMs?: number;
|
staleMs?: number;
|
||||||
|
publishLeaseDurationMilliseconds?: number;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
}): Promise<number> {
|
}): Promise<number> {
|
||||||
const OutboxModel =
|
const OutboxModel =
|
||||||
await this.connectionService.getModel('dakaoOutboxMessage');
|
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||||
|
const currentTimestamp = new Date();
|
||||||
const staleBefore = new Date(
|
const staleBefore = new Date(
|
||||||
Date.now() - (params?.staleMs ?? 5 * 60 * 1000),
|
currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000),
|
||||||
|
);
|
||||||
|
const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`;
|
||||||
|
const publishLeaseExpiresAt = new Date(
|
||||||
|
currentTimestamp.getTime() +
|
||||||
|
(params?.publishLeaseDurationMilliseconds ??
|
||||||
|
DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS),
|
||||||
);
|
);
|
||||||
const rows = await OutboxModel.find({
|
const rows = await OutboxModel.find({
|
||||||
state: { $in: ['pending', 'published'] },
|
state: { $in: ['pending', 'published'] },
|
||||||
updatedAt: { $lte: staleBefore },
|
updatedAt: { $lte: staleBefore },
|
||||||
|
$or: [
|
||||||
|
{ publishLeaseExpiresAt: { $exists: false } },
|
||||||
|
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||||
|
],
|
||||||
})
|
})
|
||||||
.sort({ updatedAt: 1 })
|
.sort({ updatedAt: 1 })
|
||||||
.limit(params?.limit ?? 20)
|
.limit(params?.limit ?? 20)
|
||||||
.lean()
|
.lean()
|
||||||
.exec();
|
.exec();
|
||||||
|
|
||||||
|
let swept = 0;
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
await this.enqueueAndPublish({
|
const claimedRow = await OutboxModel.findOneAndUpdate(
|
||||||
subject: row.subject,
|
{
|
||||||
eventId: row.eventId,
|
eventId: row.eventId,
|
||||||
|
state: { $in: ['pending', 'published'] },
|
||||||
|
$or: [
|
||||||
|
{ publishLeaseExpiresAt: { $exists: false } },
|
||||||
|
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
publishLeaseOwnerIdentifier,
|
||||||
|
publishLeaseExpiresAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ returnDocument: 'after' },
|
||||||
|
)
|
||||||
|
.lean()
|
||||||
|
.exec();
|
||||||
|
if (!claimedRow) continue;
|
||||||
|
|
||||||
|
await this.publishCommand({
|
||||||
|
subject: claimedRow.subject,
|
||||||
|
eventId: claimedRow.eventId,
|
||||||
|
publishLeaseOwnerIdentifier,
|
||||||
envelope: {
|
envelope: {
|
||||||
schema: 'dakao.command.v1',
|
schema: 'dakao.command.v1',
|
||||||
eventId: row.eventId,
|
eventId: claimedRow.eventId,
|
||||||
type: row.type,
|
type: claimedRow.type,
|
||||||
eventSource: row.eventSource,
|
eventSource: claimedRow.eventSource,
|
||||||
correlationId: row.correlationId,
|
correlationId: claimedRow.correlationId,
|
||||||
resultSubjects:
|
resultSubjects:
|
||||||
row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type),
|
claimedRow.resultSubjects ??
|
||||||
payload: row.payload,
|
this.resolveDefaultResultSubjects(claimedRow.type),
|
||||||
|
payload: claimedRow.payload,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
swept += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
return rows.length;
|
return swept;
|
||||||
}
|
}
|
||||||
|
|
||||||
private resolveDefaultResultSubjects(
|
private resolveDefaultResultSubjects(
|
||||||
|
|||||||
@@ -42,4 +42,21 @@ describe('resolveMongoConnectionOptions', () => {
|
|||||||
autoCreate: false,
|
autoCreate: false,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('enables automatic index creation when explicitly requested', () => {
|
||||||
|
expect(
|
||||||
|
resolveMongoConnectionOptions(
|
||||||
|
'lolympus-staging',
|
||||||
|
'lolympus-api:staging:1',
|
||||||
|
'true',
|
||||||
|
),
|
||||||
|
).toEqual({
|
||||||
|
dbName: 'lolympus-staging',
|
||||||
|
appName: 'lolympus-api:staging:1',
|
||||||
|
minPoolSize: 10,
|
||||||
|
maxPoolSize: 20,
|
||||||
|
autoIndex: true,
|
||||||
|
autoCreate: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -19,13 +19,17 @@ export function resolveMongoAppName(
|
|||||||
export function resolveMongoConnectionOptions(
|
export function resolveMongoConnectionOptions(
|
||||||
dbName: string,
|
dbName: string,
|
||||||
appName: string,
|
appName: string,
|
||||||
|
mongoAutoIndex = process.env.MONGODB_AUTO_INDEX,
|
||||||
): mongoose.ConnectOptions {
|
): mongoose.ConnectOptions {
|
||||||
|
const shouldEnableMongoAutoIndex =
|
||||||
|
mongoAutoIndex?.trim().toLowerCase() === 'true';
|
||||||
|
|
||||||
return {
|
return {
|
||||||
dbName,
|
dbName,
|
||||||
appName,
|
appName,
|
||||||
minPoolSize: 10,
|
minPoolSize: 10,
|
||||||
maxPoolSize: 20,
|
maxPoolSize: 20,
|
||||||
autoIndex: false,
|
autoIndex: shouldEnableMongoAutoIndex,
|
||||||
autoCreate: false,
|
autoCreate: false,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -392,6 +392,8 @@ export interface DakaoOutboxMessage extends MongooseTimestamps {
|
|||||||
payload: Record<string, unknown>;
|
payload: Record<string, unknown>;
|
||||||
state: DakaoOutboxMessageState;
|
state: DakaoOutboxMessageState;
|
||||||
publishAttempts: number;
|
publishAttempts: number;
|
||||||
|
publishLeaseOwnerIdentifier?: string;
|
||||||
|
publishLeaseExpiresAt?: Date;
|
||||||
publishedAt?: Date;
|
publishedAt?: Date;
|
||||||
terminalAt?: Date;
|
terminalAt?: Date;
|
||||||
resultEventId?: string;
|
resultEventId?: string;
|
||||||
@@ -733,6 +735,7 @@ export interface LegacyImportJob extends MongooseTimestamps {
|
|||||||
totalSkippedDuplicate: number;
|
totalSkippedDuplicate: number;
|
||||||
totalErrors: number;
|
totalErrors: number;
|
||||||
averageMemberSeconds?: number;
|
averageMemberSeconds?: number;
|
||||||
|
completedMemberJobIds: string[];
|
||||||
errors?: LegacyImportJobError[];
|
errors?: LegacyImportJobError[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1282,6 +1285,8 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
|
|||||||
default: 'pending',
|
default: 'pending',
|
||||||
},
|
},
|
||||||
publishAttempts: { type: Number, required: true, default: 0 },
|
publishAttempts: { type: Number, required: true, default: 0 },
|
||||||
|
publishLeaseOwnerIdentifier: { type: String },
|
||||||
|
publishLeaseExpiresAt: { type: Date },
|
||||||
publishedAt: { type: Date },
|
publishedAt: { type: Date },
|
||||||
terminalAt: { type: Date },
|
terminalAt: { type: Date },
|
||||||
resultEventId: { type: String },
|
resultEventId: { type: String },
|
||||||
@@ -1302,6 +1307,11 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
|
|||||||
);
|
);
|
||||||
|
|
||||||
DakaoOutboxMessageSchema.index({ state: 1, updatedAt: 1 });
|
DakaoOutboxMessageSchema.index({ state: 1, updatedAt: 1 });
|
||||||
|
DakaoOutboxMessageSchema.index({
|
||||||
|
state: 1,
|
||||||
|
publishLeaseExpiresAt: 1,
|
||||||
|
updatedAt: 1,
|
||||||
|
});
|
||||||
DakaoOutboxMessageSchema.index({ correlationId: 1 }, { sparse: true });
|
DakaoOutboxMessageSchema.index({ correlationId: 1 }, { sparse: true });
|
||||||
|
|
||||||
export const DakaoSendResultEventSchema = new Schema<DakaoSendResultEvent>(
|
export const DakaoSendResultEventSchema = new Schema<DakaoSendResultEvent>(
|
||||||
@@ -1988,6 +1998,11 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
|
|||||||
totalSkippedDuplicate: { type: Number, required: true, default: 0 },
|
totalSkippedDuplicate: { type: Number, required: true, default: 0 },
|
||||||
totalErrors: { type: Number, required: true, default: 0 },
|
totalErrors: { type: Number, required: true, default: 0 },
|
||||||
averageMemberSeconds: { type: Number },
|
averageMemberSeconds: { type: Number },
|
||||||
|
completedMemberJobIds: {
|
||||||
|
type: [String],
|
||||||
|
required: true,
|
||||||
|
default: [],
|
||||||
|
},
|
||||||
errors: {
|
errors: {
|
||||||
type: [LegacyImportJobErrorSubSchema],
|
type: [LegacyImportJobErrorSubSchema],
|
||||||
required: true,
|
required: true,
|
||||||
@@ -2003,6 +2018,7 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
|
|||||||
|
|
||||||
LegacyImportJobSchema.index({ communityId: 1, startedAt: -1 });
|
LegacyImportJobSchema.index({ communityId: 1, startedAt: -1 });
|
||||||
LegacyImportJobSchema.index({ communityId: 1, state: 1 });
|
LegacyImportJobSchema.index({ communityId: 1, state: 1 });
|
||||||
|
LegacyImportJobSchema.index({ jobId: 1, completedMemberJobIds: 1 });
|
||||||
|
|
||||||
const AuctionPlayerSnapshotSubSchema = new Schema<AuctionPlayerSnapshot>(
|
const AuctionPlayerSnapshotSubSchema = new Schema<AuctionPlayerSnapshot>(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -56,20 +56,25 @@ export interface JetStreamConsumeOptions {
|
|||||||
maxMessages?: number;
|
maxMessages?: number;
|
||||||
expiresMs?: number;
|
expiresMs?: number;
|
||||||
nakDelayMs?: number;
|
nakDelayMs?: number;
|
||||||
|
restartDelayMilliseconds?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS = 1_000;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class NatsService implements OnModuleDestroy {
|
export class NatsService implements OnModuleDestroy {
|
||||||
private readonly logger = new Logger(NatsService.name);
|
private readonly logger = new Logger(NatsService.name);
|
||||||
private readonly stringCodec = StringCodec();
|
private readonly stringCodec = StringCodec();
|
||||||
private readonly subscriptions = new Set<Subscription>();
|
private readonly subscriptions = new Set<Subscription>();
|
||||||
private readonly jetStreamConsumers = new Set<ConsumerMessages>();
|
private readonly jetStreamConsumers = new Set<ConsumerMessages>();
|
||||||
|
private isDestroying = false;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(NATS_CLIENT) private readonly natsClient: NatsConnection,
|
@Inject(NATS_CLIENT) private readonly natsClient: NatsConnection,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async onModuleDestroy() {
|
async onModuleDestroy() {
|
||||||
|
this.isDestroying = true;
|
||||||
for (const consumer of this.jetStreamConsumers) {
|
for (const consumer of this.jetStreamConsumers) {
|
||||||
try {
|
try {
|
||||||
await consumer.close();
|
await consumer.close();
|
||||||
@@ -199,28 +204,36 @@ export class NatsService implements OnModuleDestroy {
|
|||||||
options: JetStreamConsumeOptions,
|
options: JetStreamConsumeOptions,
|
||||||
handler: JetStreamMessageHandler<T>,
|
handler: JetStreamMessageHandler<T>,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
try {
|
while (!this.isDestroying) {
|
||||||
const consumer = await this.natsClient
|
|
||||||
.jetstream()
|
|
||||||
.consumers.get(options.stream, options.durable);
|
|
||||||
const messages = await consumer.consume({
|
|
||||||
max_messages: options.maxMessages ?? 10,
|
|
||||||
expires: options.expiresMs ?? 30000,
|
|
||||||
});
|
|
||||||
this.jetStreamConsumers.add(messages);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for await (const msg of messages) {
|
const consumer = await this.natsClient
|
||||||
await this.dispatchJetStream(msg, handler, options);
|
.jetstream()
|
||||||
|
.consumers.get(options.stream, options.durable);
|
||||||
|
const messages = await consumer.consume({
|
||||||
|
max_messages: options.maxMessages ?? 10,
|
||||||
|
expires: options.expiresMs ?? 30000,
|
||||||
|
});
|
||||||
|
this.jetStreamConsumers.add(messages);
|
||||||
|
|
||||||
|
try {
|
||||||
|
for await (const msg of messages) {
|
||||||
|
await this.dispatchJetStream(msg, handler, options);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
this.jetStreamConsumers.delete(messages);
|
||||||
}
|
}
|
||||||
} finally {
|
} catch (err) {
|
||||||
this.jetStreamConsumers.delete(messages);
|
if (this.isDestroying) return;
|
||||||
|
this.logger.error(
|
||||||
|
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
|
||||||
|
(err as Error)?.stack ?? err
|
||||||
|
}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
|
||||||
this.logger.error(
|
await this.sleep(
|
||||||
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
|
options.restartDelayMilliseconds ??
|
||||||
(err as Error)?.stack ?? err
|
JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS,
|
||||||
}`,
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -313,4 +326,8 @@ export class NatsService implements OnModuleDestroy {
|
|||||||
? RetentionPolicy.Workqueue
|
? RetentionPolicy.Workqueue
|
||||||
: RetentionPolicy.Limits;
|
: RetentionPolicy.Limits;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sleep(durationMilliseconds: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
import { DistributedLockService } from './distributed-lock.service';
|
||||||
|
import { RedisService } from './redis.service';
|
||||||
|
|
||||||
|
function createService(sendCommand: jest.Mock) {
|
||||||
|
const redisService = {
|
||||||
|
raw: jest.fn().mockReturnValue({ sendCommand }),
|
||||||
|
} as unknown as RedisService;
|
||||||
|
return new DistributedLockService(redisService);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DistributedLockService', () => {
|
||||||
|
it('acquires a lock with a token and millisecond expiration', async () => {
|
||||||
|
const sendCommand = jest.fn().mockResolvedValue('OK');
|
||||||
|
const service = createService(sendCommand);
|
||||||
|
|
||||||
|
const lockOwnership = await service.acquireLock({
|
||||||
|
lockKey: 'distributed-lock:test',
|
||||||
|
lockOwnershipToken: 'owner-token',
|
||||||
|
lockExpirationDurationMilliseconds: 30_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(lockOwnership).toEqual({
|
||||||
|
lockKey: 'distributed-lock:test',
|
||||||
|
lockOwnershipToken: 'owner-token',
|
||||||
|
lockExpirationDurationMilliseconds: 30_000,
|
||||||
|
});
|
||||||
|
expect(sendCommand).toHaveBeenCalledWith([
|
||||||
|
'SET',
|
||||||
|
'distributed-lock:test',
|
||||||
|
'owner-token',
|
||||||
|
'NX',
|
||||||
|
'PX',
|
||||||
|
'30000',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null when another owner already holds the lock', async () => {
|
||||||
|
const sendCommand = jest.fn().mockResolvedValue(null);
|
||||||
|
const service = createService(sendCommand);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.acquireLock({
|
||||||
|
lockKey: 'distributed-lock:test',
|
||||||
|
lockExpirationDurationMilliseconds: 30_000,
|
||||||
|
}),
|
||||||
|
).resolves.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('releases a lock only when the ownership token matches', async () => {
|
||||||
|
const sendCommand = jest.fn().mockResolvedValue(1);
|
||||||
|
const service = createService(sendCommand);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.releaseLockIfOwned({
|
||||||
|
lockKey: 'distributed-lock:test',
|
||||||
|
lockOwnershipToken: 'owner-token',
|
||||||
|
lockExpirationDurationMilliseconds: 30_000,
|
||||||
|
}),
|
||||||
|
).resolves.toBe(true);
|
||||||
|
|
||||||
|
expect(sendCommand).toHaveBeenCalledWith([
|
||||||
|
'EVAL',
|
||||||
|
expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'),
|
||||||
|
'1',
|
||||||
|
'distributed-lock:test',
|
||||||
|
'owner-token',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extends a lock only when the ownership token matches', async () => {
|
||||||
|
const sendCommand = jest.fn().mockResolvedValue(1);
|
||||||
|
const service = createService(sendCommand);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.extendLockIfOwned(
|
||||||
|
{
|
||||||
|
lockKey: 'distributed-lock:test',
|
||||||
|
lockOwnershipToken: 'owner-token',
|
||||||
|
lockExpirationDurationMilliseconds: 30_000,
|
||||||
|
},
|
||||||
|
45_000,
|
||||||
|
),
|
||||||
|
).resolves.toBe(true);
|
||||||
|
|
||||||
|
expect(sendCommand).toHaveBeenCalledWith([
|
||||||
|
'EVAL',
|
||||||
|
expect.stringContaining('PEXPIRE'),
|
||||||
|
'1',
|
||||||
|
'distributed-lock:test',
|
||||||
|
'owner-token',
|
||||||
|
'45000',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('runs the operation and releases the lock afterward', async () => {
|
||||||
|
const sendCommand = jest
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValueOnce('OK')
|
||||||
|
.mockResolvedValueOnce(1);
|
||||||
|
const service = createService(sendCommand);
|
||||||
|
const operation = jest.fn().mockResolvedValue('done');
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.runWithLock(
|
||||||
|
{
|
||||||
|
lockKey: 'distributed-lock:test',
|
||||||
|
lockExpirationDurationMilliseconds: 30_000,
|
||||||
|
},
|
||||||
|
operation,
|
||||||
|
),
|
||||||
|
).resolves.toBe('done');
|
||||||
|
|
||||||
|
expect(operation).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
lockKey: 'distributed-lock:test',
|
||||||
|
lockExpirationDurationMilliseconds: 30_000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(sendCommand).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { RedisService } from './redis.service';
|
||||||
|
|
||||||
|
const RELEASE_LOCK_IF_OWNED_SCRIPT = `
|
||||||
|
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||||
|
return redis.call("DEL", KEYS[1])
|
||||||
|
end
|
||||||
|
return 0
|
||||||
|
`;
|
||||||
|
|
||||||
|
const EXTEND_LOCK_IF_OWNED_SCRIPT = `
|
||||||
|
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||||
|
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
|
||||||
|
end
|
||||||
|
return 0
|
||||||
|
`;
|
||||||
|
|
||||||
|
export interface DistributedLockOwnership {
|
||||||
|
lockKey: string;
|
||||||
|
lockOwnershipToken: string;
|
||||||
|
lockExpirationDurationMilliseconds: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AcquireDistributedLockOptions {
|
||||||
|
lockKey: string;
|
||||||
|
lockExpirationDurationMilliseconds: number;
|
||||||
|
lockOwnershipToken?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RunWithDistributedLockOptions {
|
||||||
|
lockKey: string;
|
||||||
|
lockExpirationDurationMilliseconds: number;
|
||||||
|
lockAcquireTimeoutMilliseconds?: number;
|
||||||
|
lockRetryDelayMilliseconds?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RunWithRenewedDistributedLockOptions
|
||||||
|
extends RunWithDistributedLockOptions {
|
||||||
|
lockRenewalIntervalMilliseconds: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class DistributedLockService {
|
||||||
|
private readonly logger = new Logger(DistributedLockService.name);
|
||||||
|
|
||||||
|
constructor(private readonly redisService: RedisService) {}
|
||||||
|
|
||||||
|
public async acquireLock(
|
||||||
|
options: AcquireDistributedLockOptions,
|
||||||
|
): Promise<DistributedLockOwnership | null> {
|
||||||
|
this.assertPositiveDuration(
|
||||||
|
options.lockExpirationDurationMilliseconds,
|
||||||
|
'lockExpirationDurationMilliseconds',
|
||||||
|
);
|
||||||
|
|
||||||
|
const lockOwnershipToken = options.lockOwnershipToken ?? randomUUID();
|
||||||
|
const result = await this.redisService.raw().sendCommand([
|
||||||
|
'SET',
|
||||||
|
options.lockKey,
|
||||||
|
lockOwnershipToken,
|
||||||
|
'NX',
|
||||||
|
'PX',
|
||||||
|
String(options.lockExpirationDurationMilliseconds),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (String(result) !== 'OK') return null;
|
||||||
|
return {
|
||||||
|
lockKey: options.lockKey,
|
||||||
|
lockOwnershipToken,
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
options.lockExpirationDurationMilliseconds,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async releaseLockIfOwned(
|
||||||
|
lockOwnership: DistributedLockOwnership,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const result = await this.redisService.raw().sendCommand([
|
||||||
|
'EVAL',
|
||||||
|
RELEASE_LOCK_IF_OWNED_SCRIPT,
|
||||||
|
'1',
|
||||||
|
lockOwnership.lockKey,
|
||||||
|
lockOwnership.lockOwnershipToken,
|
||||||
|
]);
|
||||||
|
return Number(result) === 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async extendLockIfOwned(
|
||||||
|
lockOwnership: DistributedLockOwnership,
|
||||||
|
lockExpirationDurationMilliseconds =
|
||||||
|
lockOwnership.lockExpirationDurationMilliseconds,
|
||||||
|
): Promise<boolean> {
|
||||||
|
this.assertPositiveDuration(
|
||||||
|
lockExpirationDurationMilliseconds,
|
||||||
|
'lockExpirationDurationMilliseconds',
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await this.redisService.raw().sendCommand([
|
||||||
|
'EVAL',
|
||||||
|
EXTEND_LOCK_IF_OWNED_SCRIPT,
|
||||||
|
'1',
|
||||||
|
lockOwnership.lockKey,
|
||||||
|
lockOwnership.lockOwnershipToken,
|
||||||
|
String(lockExpirationDurationMilliseconds),
|
||||||
|
]);
|
||||||
|
return Number(result) === 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async runWithLock<TResult>(
|
||||||
|
options: RunWithDistributedLockOptions,
|
||||||
|
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||||
|
): Promise<TResult | null> {
|
||||||
|
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||||
|
if (!lockOwnership) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await operation(lockOwnership);
|
||||||
|
} finally {
|
||||||
|
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||||
|
this.logger.warn(
|
||||||
|
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async runWithRenewedLock<TResult>(
|
||||||
|
options: RunWithRenewedDistributedLockOptions,
|
||||||
|
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||||
|
): Promise<TResult | null> {
|
||||||
|
this.assertPositiveDuration(
|
||||||
|
options.lockRenewalIntervalMilliseconds,
|
||||||
|
'lockRenewalIntervalMilliseconds',
|
||||||
|
);
|
||||||
|
|
||||||
|
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||||
|
if (!lockOwnership) return null;
|
||||||
|
|
||||||
|
let lockRenewalTimer: NodeJS.Timeout | undefined;
|
||||||
|
lockRenewalTimer = setInterval(() => {
|
||||||
|
void this.extendLockIfOwned(lockOwnership).then((wasExtended) => {
|
||||||
|
if (wasExtended) return;
|
||||||
|
this.logger.warn(
|
||||||
|
`Lost distributed lock ownership before operation finished: ${lockOwnership.lockKey}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}, options.lockRenewalIntervalMilliseconds);
|
||||||
|
lockRenewalTimer.unref?.();
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await operation(lockOwnership);
|
||||||
|
} finally {
|
||||||
|
if (lockRenewalTimer) clearInterval(lockRenewalTimer);
|
||||||
|
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||||
|
this.logger.warn(
|
||||||
|
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async acquireLockWithRetry(
|
||||||
|
options: RunWithDistributedLockOptions,
|
||||||
|
): Promise<DistributedLockOwnership | null> {
|
||||||
|
const lockAcquireTimeoutMilliseconds =
|
||||||
|
options.lockAcquireTimeoutMilliseconds ?? 0;
|
||||||
|
const lockRetryDelayMilliseconds =
|
||||||
|
options.lockRetryDelayMilliseconds ?? 100;
|
||||||
|
const acquisitionStartedAtTimestampMilliseconds = Date.now();
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const lockOwnership = await this.acquireLock(options);
|
||||||
|
if (lockOwnership) return lockOwnership;
|
||||||
|
|
||||||
|
if (lockAcquireTimeoutMilliseconds <= 0) return null;
|
||||||
|
const elapsedMilliseconds =
|
||||||
|
Date.now() - acquisitionStartedAtTimestampMilliseconds;
|
||||||
|
if (elapsedMilliseconds >= lockAcquireTimeoutMilliseconds) return null;
|
||||||
|
|
||||||
|
await this.sleep(
|
||||||
|
Math.min(lockRetryDelayMilliseconds, lockAcquireTimeoutMilliseconds),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private assertPositiveDuration(value: number, fieldName: string): void {
|
||||||
|
if (Number.isFinite(value) && value > 0) return;
|
||||||
|
throw new Error(`${fieldName} must be a positive finite number.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sleep(durationMilliseconds: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import { SecretModule } from '../secret/secret.module';
|
|||||||
import { SecretService } from '../secret/secret.service';
|
import { SecretService } from '../secret/secret.service';
|
||||||
import { REDIS_CLIENT } from './redis.constants';
|
import { REDIS_CLIENT } from './redis.constants';
|
||||||
import { resolveRedisClientOptions } from './redis.config';
|
import { resolveRedisClientOptions } from './redis.config';
|
||||||
|
import { DistributedLockService } from './distributed-lock.service';
|
||||||
import { RedisService, type RedisClient } from './redis.service';
|
import { RedisService, type RedisClient } from './redis.service';
|
||||||
|
|
||||||
async function readFirstSecret(
|
async function readFirstSecret(
|
||||||
@@ -62,7 +63,8 @@ async function readFirstSecret(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
RedisService,
|
RedisService,
|
||||||
|
DistributedLockService,
|
||||||
],
|
],
|
||||||
exports: [RedisService, REDIS_CLIENT],
|
exports: [RedisService, DistributedLockService, REDIS_CLIENT],
|
||||||
})
|
})
|
||||||
export class RedisModule {}
|
export class RedisModule {}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
import { RedisService } from './redis.service';
|
||||||
|
|
||||||
|
function createRedisService(client: Record<string, unknown>) {
|
||||||
|
return new RedisService(client as never);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('RedisService stream pending recovery', () => {
|
||||||
|
it('normalizes XAUTOCLAIM array replies into stream entries', async () => {
|
||||||
|
const sendCommand = jest.fn().mockResolvedValue([
|
||||||
|
'0-0',
|
||||||
|
[
|
||||||
|
[
|
||||||
|
'1700000000000-0',
|
||||||
|
['job', '{"kind":"poll-tournament-result"}', 'source', 'test'],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
const service = createRedisService({ sendCommand });
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.claimStalePendingStreamMessages({
|
||||||
|
key: 'riot:queue:high',
|
||||||
|
group: 'riot-refresh-worker',
|
||||||
|
consumer: 'worker-1',
|
||||||
|
stalePendingMessageMinimumIdleMilliseconds: 1_800_000,
|
||||||
|
count: 1,
|
||||||
|
}),
|
||||||
|
).resolves.toEqual({
|
||||||
|
nextStartId: '0-0',
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
id: '1700000000000-0',
|
||||||
|
message: {
|
||||||
|
job: '{"kind":"poll-tournament-result"}',
|
||||||
|
source: 'test',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(sendCommand).toHaveBeenCalledWith([
|
||||||
|
'XAUTOCLAIM',
|
||||||
|
'riot:queue:high',
|
||||||
|
'riot-refresh-worker',
|
||||||
|
'worker-1',
|
||||||
|
'1800000',
|
||||||
|
'0-0',
|
||||||
|
'COUNT',
|
||||||
|
'1',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -16,6 +16,11 @@ export interface StreamReadResult {
|
|||||||
name: string;
|
name: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ClaimStalePendingStreamMessagesResult {
|
||||||
|
messages: StreamEntry[];
|
||||||
|
nextStartId: string;
|
||||||
|
}
|
||||||
|
|
||||||
interface RawStreamMessage {
|
interface RawStreamMessage {
|
||||||
deliveriesCounter?: number;
|
deliveriesCounter?: number;
|
||||||
id: string;
|
id: string;
|
||||||
@@ -29,6 +34,16 @@ interface RawStreamReadResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function normalizeStreamFields(fields: unknown): Record<string, string> {
|
function normalizeStreamFields(fields: unknown): Record<string, string> {
|
||||||
|
if (Array.isArray(fields)) {
|
||||||
|
const normalized: Record<string, string> = {};
|
||||||
|
for (let index = 0; index < fields.length; index += 2) {
|
||||||
|
const key = fields[index];
|
||||||
|
if (key === undefined) continue;
|
||||||
|
normalized[String(key)] = String(fields[index + 1] ?? '');
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
if (fields instanceof Map) {
|
if (fields instanceof Map) {
|
||||||
return Object.fromEntries(
|
return Object.fromEntries(
|
||||||
[...fields.entries()].map(([key, value]) => [String(key), String(value)]),
|
[...fields.entries()].map(([key, value]) => [String(key), String(value)]),
|
||||||
@@ -188,6 +203,38 @@ export class RedisService implements OnModuleDestroy {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async claimStalePendingStreamMessages(options: {
|
||||||
|
key: string;
|
||||||
|
group: string;
|
||||||
|
consumer: string;
|
||||||
|
stalePendingMessageMinimumIdleMilliseconds: number;
|
||||||
|
startId?: string;
|
||||||
|
count?: number;
|
||||||
|
}): Promise<ClaimStalePendingStreamMessagesResult> {
|
||||||
|
const response = (await this.client.sendCommand([
|
||||||
|
'XAUTOCLAIM',
|
||||||
|
options.key,
|
||||||
|
options.group,
|
||||||
|
options.consumer,
|
||||||
|
String(options.stalePendingMessageMinimumIdleMilliseconds),
|
||||||
|
options.startId ?? '0-0',
|
||||||
|
'COUNT',
|
||||||
|
String(options.count ?? 1),
|
||||||
|
])) as unknown;
|
||||||
|
|
||||||
|
const responseItems = Array.isArray(response) ? response : [];
|
||||||
|
const rawMessages = Array.isArray(responseItems[1])
|
||||||
|
? responseItems[1]
|
||||||
|
: [];
|
||||||
|
|
||||||
|
return {
|
||||||
|
nextStartId: String(responseItems[0] ?? '0-0'),
|
||||||
|
messages: rawMessages
|
||||||
|
.map((rawMessage) => this.normalizeClaimedStreamMessage(rawMessage))
|
||||||
|
.filter((message): message is StreamEntry => Boolean(message)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
public async acknowledgeStream(
|
public async acknowledgeStream(
|
||||||
key: string,
|
key: string,
|
||||||
group: string,
|
group: string,
|
||||||
@@ -210,4 +257,37 @@ export class RedisService implements OnModuleDestroy {
|
|||||||
public async publish(channel: string, message: string): Promise<number> {
|
public async publish(channel: string, message: string): Promise<number> {
|
||||||
return this.client.publish(channel, message);
|
return this.client.publish(channel, message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private normalizeClaimedStreamMessage(rawMessage: unknown): StreamEntry | null {
|
||||||
|
if (Array.isArray(rawMessage)) {
|
||||||
|
const [id, fields] = rawMessage;
|
||||||
|
if (id === undefined) return null;
|
||||||
|
return {
|
||||||
|
id: String(id),
|
||||||
|
message: normalizeStreamFields(fields),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
typeof rawMessage === 'object' &&
|
||||||
|
rawMessage !== null &&
|
||||||
|
'id' in rawMessage
|
||||||
|
) {
|
||||||
|
const message = rawMessage as RawStreamMessage;
|
||||||
|
return {
|
||||||
|
deliveriesCounter:
|
||||||
|
typeof message.deliveriesCounter === 'number'
|
||||||
|
? message.deliveriesCounter
|
||||||
|
: undefined,
|
||||||
|
id: String(message.id),
|
||||||
|
message: normalizeStreamFields(message.message),
|
||||||
|
millisElapsedFromDelivery:
|
||||||
|
typeof message.millisElapsedFromDelivery === 'number'
|
||||||
|
? message.millisElapsedFromDelivery
|
||||||
|
: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||||
|
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||||
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
|
import { RedisService } from '../global/redis/redis.service';
|
||||||
|
import { RiotMatchService } from '../riot/riot-match.service';
|
||||||
|
import { LegacyImportService } from './legacy-import.service';
|
||||||
|
import type { LegacyImportMemberJob } from './legacy-import.types';
|
||||||
|
|
||||||
|
function execResult<T>(value: T) {
|
||||||
|
return { exec: jest.fn().mockResolvedValue(value) };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('LegacyImportService member job completion', () => {
|
||||||
|
it('increments aggregate counters only once for the same member job id', async () => {
|
||||||
|
const legacyImportJobModel = {
|
||||||
|
findOne: jest
|
||||||
|
.fn()
|
||||||
|
.mockReturnValueOnce(
|
||||||
|
execResult({
|
||||||
|
jobId: 'legacy-job-1',
|
||||||
|
state: 'running',
|
||||||
|
processedMembers: 0,
|
||||||
|
totalMembers: 1,
|
||||||
|
averageMemberSeconds: 60,
|
||||||
|
completedMemberJobIds: [],
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.mockReturnValueOnce({
|
||||||
|
select: jest.fn().mockReturnValue({
|
||||||
|
lean: jest.fn().mockReturnValue(
|
||||||
|
execResult({
|
||||||
|
state: 'running',
|
||||||
|
processedMembers: 1,
|
||||||
|
totalMembers: 1,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.mockReturnValueOnce(
|
||||||
|
execResult({
|
||||||
|
jobId: 'legacy-job-1',
|
||||||
|
state: 'completed',
|
||||||
|
processedMembers: 1,
|
||||||
|
totalMembers: 1,
|
||||||
|
averageMemberSeconds: 60,
|
||||||
|
completedMemberJobIds: ['member-job-1'],
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
|
||||||
|
};
|
||||||
|
const connectionService = {
|
||||||
|
getModel: jest.fn().mockResolvedValue(legacyImportJobModel),
|
||||||
|
} as unknown as jest.Mocked<ConnectionService>;
|
||||||
|
const distributedLockService = {
|
||||||
|
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||||
|
} as unknown as jest.Mocked<DistributedLockService>;
|
||||||
|
const communityStatsService = {
|
||||||
|
invalidateCommunity: jest.fn().mockResolvedValue(undefined),
|
||||||
|
} as unknown as jest.Mocked<CommunityStatsService>;
|
||||||
|
const service = new LegacyImportService(
|
||||||
|
connectionService,
|
||||||
|
{} as RedisService,
|
||||||
|
{} as RiotMatchService,
|
||||||
|
communityStatsService,
|
||||||
|
{} as MatchStatsService,
|
||||||
|
distributedLockService,
|
||||||
|
);
|
||||||
|
const job: LegacyImportMemberJob = {
|
||||||
|
kind: 'legacy-import-member',
|
||||||
|
jobId: 'member-job-1',
|
||||||
|
parentJobId: 'legacy-job-1',
|
||||||
|
communityId: new mongoose.Types.ObjectId().toString(),
|
||||||
|
userId: new mongoose.Types.ObjectId().toString(),
|
||||||
|
puuid: 'puuid-1',
|
||||||
|
startTime: 1,
|
||||||
|
endTime: 2,
|
||||||
|
triggeredBy: new mongoose.Types.ObjectId().toString(),
|
||||||
|
region: 'asia',
|
||||||
|
};
|
||||||
|
|
||||||
|
await (
|
||||||
|
service as unknown as {
|
||||||
|
finishMemberJob: (
|
||||||
|
job: LegacyImportMemberJob,
|
||||||
|
stats: {
|
||||||
|
discovered: number;
|
||||||
|
imported: number;
|
||||||
|
duplicates: number;
|
||||||
|
errors: number;
|
||||||
|
elapsedSeconds: number;
|
||||||
|
},
|
||||||
|
) => Promise<void>;
|
||||||
|
}
|
||||||
|
).finishMemberJob(job, {
|
||||||
|
discovered: 3,
|
||||||
|
imported: 2,
|
||||||
|
duplicates: 1,
|
||||||
|
errors: 0,
|
||||||
|
elapsedSeconds: 30,
|
||||||
|
});
|
||||||
|
await (
|
||||||
|
service as unknown as {
|
||||||
|
finishMemberJob: (
|
||||||
|
job: LegacyImportMemberJob,
|
||||||
|
stats: {
|
||||||
|
discovered: number;
|
||||||
|
imported: number;
|
||||||
|
duplicates: number;
|
||||||
|
errors: number;
|
||||||
|
elapsedSeconds: number;
|
||||||
|
},
|
||||||
|
) => Promise<void>;
|
||||||
|
}
|
||||||
|
).finishMemberJob(job, {
|
||||||
|
discovered: 3,
|
||||||
|
imported: 2,
|
||||||
|
duplicates: 1,
|
||||||
|
errors: 0,
|
||||||
|
elapsedSeconds: 30,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(legacyImportJobModel.updateOne).toHaveBeenNthCalledWith(
|
||||||
|
1,
|
||||||
|
{
|
||||||
|
jobId: 'legacy-job-1',
|
||||||
|
completedMemberJobIds: { $ne: 'member-job-1' },
|
||||||
|
state: { $ne: 'cancelled' },
|
||||||
|
},
|
||||||
|
expect.objectContaining({
|
||||||
|
$addToSet: { completedMemberJobIds: 'member-job-1' },
|
||||||
|
$inc: expect.objectContaining({
|
||||||
|
processedMembers: 1,
|
||||||
|
totalDiscoveredMatchIds: 3,
|
||||||
|
totalImportedMatches: 2,
|
||||||
|
totalSkippedDuplicate: 1,
|
||||||
|
totalErrors: 0,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(legacyImportJobModel.updateOne).toHaveBeenCalledTimes(2);
|
||||||
|
expect(distributedLockService.releaseLockIfOwned).toHaveBeenCalledTimes(1);
|
||||||
|
expect(communityStatsService.invalidateCommunity).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -10,6 +10,10 @@ import mongoose from 'mongoose';
|
|||||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||||
import { ConnectionService } from '../global/database/connection.service';
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import {
|
||||||
|
type DistributedLockOwnership,
|
||||||
|
DistributedLockService,
|
||||||
|
} from '../global/redis/distributed-lock.service';
|
||||||
import type {
|
import type {
|
||||||
LegacyImportJobDocument,
|
LegacyImportJobDocument,
|
||||||
Match,
|
Match,
|
||||||
@@ -57,6 +61,7 @@ export class LegacyImportService {
|
|||||||
private readonly riotMatchService: RiotMatchService,
|
private readonly riotMatchService: RiotMatchService,
|
||||||
private readonly communityStatsService: CommunityStatsService,
|
private readonly communityStatsService: CommunityStatsService,
|
||||||
private readonly matchStatsService: MatchStatsService,
|
private readonly matchStatsService: MatchStatsService,
|
||||||
|
private readonly distributedLockService: DistributedLockService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
public async previewMembers(params: {
|
public async previewMembers(params: {
|
||||||
@@ -80,12 +85,14 @@ export class LegacyImportService {
|
|||||||
}): Promise<StartLegacyImportResponse> {
|
}): Promise<StartLegacyImportResponse> {
|
||||||
const lockKey = this.communityLockKey(params.communityId);
|
const lockKey = this.communityLockKey(params.communityId);
|
||||||
const jobId = randomUUID();
|
const jobId = randomUUID();
|
||||||
const acquired = await this.redisService.setStringIfAbsent(
|
const communityImportLockOwnership =
|
||||||
lockKey,
|
await this.distributedLockService.acquireLock({
|
||||||
jobId,
|
lockKey,
|
||||||
LEGACY_IMPORT_LOCK_TTL_SECONDS,
|
lockOwnershipToken: jobId,
|
||||||
);
|
lockExpirationDurationMilliseconds:
|
||||||
if (!acquired) {
|
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||||
|
});
|
||||||
|
if (!communityImportLockOwnership) {
|
||||||
throw new ConflictException('Legacy import is already running.');
|
throw new ConflictException('Legacy import is already running.');
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,6 +140,7 @@ export class LegacyImportService {
|
|||||||
totalSkippedDuplicate: 0,
|
totalSkippedDuplicate: 0,
|
||||||
totalErrors: 0,
|
totalErrors: 0,
|
||||||
averageMemberSeconds: LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
averageMemberSeconds: LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
||||||
|
completedMemberJobIds: [],
|
||||||
});
|
});
|
||||||
|
|
||||||
for (const window of windows) {
|
for (const window of windows) {
|
||||||
@@ -160,7 +168,7 @@ export class LegacyImportService {
|
|||||||
{ jobId },
|
{ jobId },
|
||||||
{ $set: { state: 'completed', completedAt: new Date() } },
|
{ $set: { state: 'completed', completedAt: new Date() } },
|
||||||
).exec();
|
).exec();
|
||||||
await this.redisService.delete(lockKey);
|
await this.releaseDistributedLockSafely(communityImportLockOwnership);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -169,7 +177,7 @@ export class LegacyImportService {
|
|||||||
etaSeconds: windows.length * LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
etaSeconds: windows.length * LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await this.redisService.delete(lockKey);
|
await this.releaseDistributedLockSafely(communityImportLockOwnership);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -220,7 +228,11 @@ export class LegacyImportService {
|
|||||||
).exec();
|
).exec();
|
||||||
if (!job)
|
if (!job)
|
||||||
throw new NotFoundException('Running legacy import job not found.');
|
throw new NotFoundException('Running legacy import job not found.');
|
||||||
await this.redisService.delete(this.communityLockKey(params.communityId));
|
await this.distributedLockService.releaseLockIfOwned({
|
||||||
|
lockKey: this.communityLockKey(params.communityId),
|
||||||
|
lockOwnershipToken: job.jobId,
|
||||||
|
lockExpirationDurationMilliseconds: LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||||
|
});
|
||||||
return job;
|
return job;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,6 +240,7 @@ export class LegacyImportService {
|
|||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
const legacyJob = await this.getRunnableJob(job);
|
const legacyJob = await this.getRunnableJob(job);
|
||||||
if (!legacyJob) return;
|
if (!legacyJob) return;
|
||||||
|
if (legacyJob.completedMemberJobIds?.includes(job.jobId)) return;
|
||||||
|
|
||||||
let discovered = 0;
|
let discovered = 0;
|
||||||
let imported = 0;
|
let imported = 0;
|
||||||
@@ -500,75 +513,81 @@ export class LegacyImportService {
|
|||||||
userByPuuid: UserByPuuidMap,
|
userByPuuid: UserByPuuidMap,
|
||||||
): Promise<ImportOutcome> {
|
): Promise<ImportOutcome> {
|
||||||
const lockKey = `legacy-import:match:${riotMatchId}`;
|
const lockKey = `legacy-import:match:${riotMatchId}`;
|
||||||
const acquired = await this.redisService.setStringIfAbsent(
|
|
||||||
lockKey,
|
|
||||||
job.parentJobId,
|
|
||||||
MATCH_IMPORT_LOCK_TTL_SECONDS,
|
|
||||||
);
|
|
||||||
if (!acquired) return 'duplicate';
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const MatchModel = await this.connectionService.getModel('match');
|
const importOutcome = await this.distributedLockService.runWithLock(
|
||||||
const existing = await MatchModel.exists({
|
{
|
||||||
'result.games.riotMatchId': riotMatchId,
|
lockKey,
|
||||||
});
|
lockExpirationDurationMilliseconds:
|
||||||
if (existing) return 'duplicate';
|
MATCH_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
const MatchModel = await this.connectionService.getModel('match');
|
||||||
|
const existing = await MatchModel.exists({
|
||||||
|
'result.games.riotMatchId': riotMatchId,
|
||||||
|
});
|
||||||
|
if (existing) return 'duplicate';
|
||||||
|
|
||||||
const detail = await this.ensureRiotMatchPersisted(
|
const detail = await this.ensureRiotMatchPersisted(
|
||||||
riotMatchId,
|
riotMatchId,
|
||||||
job.region,
|
job.region,
|
||||||
);
|
);
|
||||||
if (!detail.info.tournamentCode) return 'skipped';
|
if (!detail.info.tournamentCode) return 'skipped';
|
||||||
const counts = countKnownParticipants(detail, userByPuuid);
|
const counts = countKnownParticipants(detail, userByPuuid);
|
||||||
if (counts.total === 0) return 'skipped';
|
if (counts.total === 0) return 'skipped';
|
||||||
|
|
||||||
const gameResult = buildGameResult(detail, userByPuuid, 1);
|
const gameResult = buildGameResult(detail, userByPuuid, 1);
|
||||||
if (!gameResult || gameResult.participants.length === 0) return 'skipped';
|
if (!gameResult || gameResult.participants.length === 0) {
|
||||||
|
return 'skipped';
|
||||||
|
}
|
||||||
|
|
||||||
const winnerTeamIndex = gameResult.winnerTeamIndex;
|
const winnerTeamIndex = gameResult.winnerTeamIndex;
|
||||||
const matchPayload: Partial<Match> = {
|
const matchPayload: Partial<Match> = {
|
||||||
communityId: new mongoose.Types.ObjectId(job.communityId),
|
communityId: new mongoose.Types.ObjectId(job.communityId),
|
||||||
title: this.formatLegacyDate(detail.info.gameCreation),
|
title: this.formatLegacyDate(detail.info.gameCreation),
|
||||||
description: 'Legacy import (자동 생성)',
|
description: 'Legacy import (자동 생성)',
|
||||||
mapId: detail.info.mapId ?? 11,
|
mapId: detail.info.mapId ?? 11,
|
||||||
status: 'finished',
|
status: 'finished',
|
||||||
scheduledAt: new Date(detail.info.gameCreation),
|
scheduledAt: new Date(detail.info.gameCreation),
|
||||||
maxParticipants: 10,
|
maxParticipants: 10,
|
||||||
gameCount: 1,
|
gameCount: 1,
|
||||||
gameMode: 'pair',
|
gameMode: 'pair',
|
||||||
tournamentCodeIds: [],
|
tournamentCodeIds: [],
|
||||||
teamBuiltAt: new Date(detail.info.gameCreation),
|
teamBuiltAt: new Date(detail.info.gameCreation),
|
||||||
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
|
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||||
participants: buildParticipantsSnapshot(detail, userByPuuid),
|
participants: buildParticipantsSnapshot(detail, userByPuuid),
|
||||||
teams: buildTeams(detail, userByPuuid),
|
teams: buildTeams(detail, userByPuuid),
|
||||||
result: {
|
result: {
|
||||||
winnerTeamIndex,
|
|
||||||
riotMatchIds: [riotMatchId],
|
|
||||||
games: [gameResult],
|
|
||||||
seriesResults: [
|
|
||||||
{
|
|
||||||
teamAIndex: 1,
|
|
||||||
teamBIndex: 2,
|
|
||||||
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
|
|
||||||
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
|
|
||||||
winnerTeamIndex,
|
winnerTeamIndex,
|
||||||
|
riotMatchIds: [riotMatchId],
|
||||||
|
games: [gameResult],
|
||||||
|
seriesResults: [
|
||||||
|
{
|
||||||
|
teamAIndex: 1,
|
||||||
|
teamBIndex: 2,
|
||||||
|
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
|
||||||
|
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
|
||||||
|
winnerTeamIndex,
|
||||||
|
},
|
||||||
|
],
|
||||||
},
|
},
|
||||||
],
|
legacyImported: true,
|
||||||
|
legacyImportSource: {
|
||||||
|
importedAt: new Date(),
|
||||||
|
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||||
|
triggeringPuuid: job.puuid,
|
||||||
|
jobId: job.parentJobId,
|
||||||
|
},
|
||||||
|
prizes: [],
|
||||||
|
notices: [],
|
||||||
|
schedule: [],
|
||||||
|
};
|
||||||
|
const match = await MatchModel.create(matchPayload);
|
||||||
|
await this.matchStatsService.buildAndUpsert(match._id);
|
||||||
|
return 'imported';
|
||||||
},
|
},
|
||||||
legacyImported: true,
|
);
|
||||||
legacyImportSource: {
|
if (importOutcome === null) return 'duplicate';
|
||||||
importedAt: new Date(),
|
return importOutcome;
|
||||||
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
|
|
||||||
triggeringPuuid: job.puuid,
|
|
||||||
jobId: job.parentJobId,
|
|
||||||
},
|
|
||||||
prizes: [],
|
|
||||||
notices: [],
|
|
||||||
schedule: [],
|
|
||||||
};
|
|
||||||
const match = await MatchModel.create(matchPayload);
|
|
||||||
await this.matchStatsService.buildAndUpsert(match._id);
|
|
||||||
return 'imported';
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await this.recordError(job.parentJobId, {
|
await this.recordError(job.parentJobId, {
|
||||||
puuid: job.puuid,
|
puuid: job.puuid,
|
||||||
@@ -576,8 +595,6 @@ export class LegacyImportService {
|
|||||||
reason: error instanceof Error ? error.message : String(error),
|
reason: error instanceof Error ? error.message : String(error),
|
||||||
});
|
});
|
||||||
return 'error';
|
return 'error';
|
||||||
} finally {
|
|
||||||
await this.redisService.delete(lockKey);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -626,6 +643,7 @@ export class LegacyImportService {
|
|||||||
jobId: job.parentJobId,
|
jobId: job.parentJobId,
|
||||||
}).exec();
|
}).exec();
|
||||||
if (!current || current.state === 'cancelled') return;
|
if (!current || current.state === 'cancelled') return;
|
||||||
|
if (current.completedMemberJobIds?.includes(job.jobId)) return;
|
||||||
|
|
||||||
const average =
|
const average =
|
||||||
current.averageMemberSeconds === undefined
|
current.averageMemberSeconds === undefined
|
||||||
@@ -634,14 +652,14 @@ export class LegacyImportService {
|
|||||||
current.averageMemberSeconds * 0.7 + stats.elapsedSeconds * 0.3,
|
current.averageMemberSeconds * 0.7 + stats.elapsedSeconds * 0.3,
|
||||||
);
|
);
|
||||||
|
|
||||||
const nextProcessed = current.processedMembers + 1;
|
const updateResult = await LegacyImportJobModel.updateOne(
|
||||||
const completed = nextProcessed >= current.totalMembers;
|
{
|
||||||
await LegacyImportJobModel.updateOne(
|
jobId: job.parentJobId,
|
||||||
{ jobId: job.parentJobId },
|
completedMemberJobIds: { $ne: job.jobId },
|
||||||
|
state: { $ne: 'cancelled' },
|
||||||
|
},
|
||||||
{
|
{
|
||||||
$set: {
|
$set: {
|
||||||
state: completed ? 'completed' : current.state,
|
|
||||||
completedAt: completed ? new Date() : current.completedAt,
|
|
||||||
averageMemberSeconds: average,
|
averageMemberSeconds: average,
|
||||||
},
|
},
|
||||||
$inc: {
|
$inc: {
|
||||||
@@ -651,14 +669,49 @@ export class LegacyImportService {
|
|||||||
totalSkippedDuplicate: stats.duplicates,
|
totalSkippedDuplicate: stats.duplicates,
|
||||||
totalErrors: stats.errors,
|
totalErrors: stats.errors,
|
||||||
},
|
},
|
||||||
|
$addToSet: {
|
||||||
|
completedMemberJobIds: job.jobId,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
).exec();
|
).exec();
|
||||||
|
if (updateResult.modifiedCount === 0) return;
|
||||||
|
|
||||||
|
const updated = await LegacyImportJobModel.findOne({
|
||||||
|
jobId: job.parentJobId,
|
||||||
|
})
|
||||||
|
.select({ processedMembers: 1, totalMembers: 1, state: 1, completedAt: 1 })
|
||||||
|
.lean()
|
||||||
|
.exec();
|
||||||
|
const completed = Boolean(
|
||||||
|
updated &&
|
||||||
|
updated.state !== 'cancelled' &&
|
||||||
|
updated.processedMembers >= updated.totalMembers,
|
||||||
|
);
|
||||||
|
if (completed) {
|
||||||
|
await LegacyImportJobModel.updateOne(
|
||||||
|
{
|
||||||
|
jobId: job.parentJobId,
|
||||||
|
state: { $ne: 'cancelled' },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
state: 'completed',
|
||||||
|
completedAt: updated?.completedAt ?? new Date(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
).exec();
|
||||||
|
}
|
||||||
|
|
||||||
if (completed) {
|
if (completed) {
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
this.redisService.delete(
|
this.distributedLockService.releaseLockIfOwned({
|
||||||
this.communityLockKey(new mongoose.Types.ObjectId(job.communityId)),
|
lockKey: this.communityLockKey(
|
||||||
),
|
new mongoose.Types.ObjectId(job.communityId),
|
||||||
|
),
|
||||||
|
lockOwnershipToken: job.parentJobId,
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||||
|
}),
|
||||||
this.communityStatsService.invalidateCommunity(
|
this.communityStatsService.invalidateCommunity(
|
||||||
new mongoose.Types.ObjectId(job.communityId),
|
new mongoose.Types.ObjectId(job.communityId),
|
||||||
),
|
),
|
||||||
@@ -680,6 +733,20 @@ export class LegacyImportService {
|
|||||||
return legacyJob;
|
return legacyJob;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async releaseDistributedLockSafely(
|
||||||
|
lockOwnership: DistributedLockOwnership,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.distributedLockService
|
||||||
|
.releaseLockIfOwned(lockOwnership)
|
||||||
|
.catch((error: unknown) => {
|
||||||
|
this.logger.warn(
|
||||||
|
`Failed to release legacy import lock ${lockOwnership.lockKey}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private async isCancelled(parentJobId: string): Promise<boolean> {
|
private async isCancelled(parentJobId: string): Promise<boolean> {
|
||||||
const LegacyImportJobModel =
|
const LegacyImportJobModel =
|
||||||
await this.connectionService.getModel('legacyImportJob');
|
await this.connectionService.getModel('legacyImportJob');
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import mongoose from 'mongoose';
|
import mongoose from 'mongoose';
|
||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import { RedisService } from '../global/redis/redis.service';
|
import { RedisService } from '../global/redis/redis.service';
|
||||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||||
@@ -35,11 +36,18 @@ function makeScheduler() {
|
|||||||
};
|
};
|
||||||
const redis = {
|
const redis = {
|
||||||
getString: jest.fn().mockResolvedValue(null),
|
getString: jest.fn().mockResolvedValue(null),
|
||||||
setString: jest.fn().mockResolvedValue(undefined),
|
|
||||||
setStringIfAbsent: jest.fn().mockResolvedValue(true),
|
|
||||||
delete: jest.fn().mockResolvedValue(1),
|
delete: jest.fn().mockResolvedValue(1),
|
||||||
raw: jest.fn().mockReturnValue(rawClient),
|
raw: jest.fn().mockReturnValue(rawClient),
|
||||||
} as unknown as jest.Mocked<RedisService> & { raw: jest.Mock };
|
} as unknown as jest.Mocked<RedisService> & { raw: jest.Mock };
|
||||||
|
const distributedLockService = {
|
||||||
|
acquireLock: jest.fn().mockResolvedValue({
|
||||||
|
lockKey: 'distributed-lock:riot-background-refresh:leader',
|
||||||
|
lockOwnershipToken: 'lock-owner',
|
||||||
|
lockExpirationDurationMilliseconds: 90_000,
|
||||||
|
}),
|
||||||
|
extendLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||||
|
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||||
|
} as unknown as jest.Mocked<DistributedLockService>;
|
||||||
const rateLimiter = {
|
const rateLimiter = {
|
||||||
isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
|
isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
|
||||||
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
||||||
@@ -52,6 +60,7 @@ function makeScheduler() {
|
|||||||
} as unknown as jest.Mocked<StaleAccountRepository>;
|
} as unknown as jest.Mocked<StaleAccountRepository>;
|
||||||
const scheduler = new BackgroundRefreshScheduler(
|
const scheduler = new BackgroundRefreshScheduler(
|
||||||
redis,
|
redis,
|
||||||
|
distributedLockService,
|
||||||
rateLimiter,
|
rateLimiter,
|
||||||
enrichmentService,
|
enrichmentService,
|
||||||
staleAccountRepository,
|
staleAccountRepository,
|
||||||
@@ -60,6 +69,7 @@ function makeScheduler() {
|
|||||||
return {
|
return {
|
||||||
scheduler,
|
scheduler,
|
||||||
redis,
|
redis,
|
||||||
|
distributedLockService,
|
||||||
rawClient,
|
rawClient,
|
||||||
rateLimiter,
|
rateLimiter,
|
||||||
enrichmentService,
|
enrichmentService,
|
||||||
@@ -115,8 +125,9 @@ describe('BackgroundRefreshScheduler', () => {
|
|||||||
|
|
||||||
it('does not query stale accounts when another leader holds the lock', async () => {
|
it('does not query stale accounts when another leader holds the lock', async () => {
|
||||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||||
const { scheduler, redis, staleAccountRepository } = makeScheduler();
|
const { scheduler, distributedLockService, staleAccountRepository } =
|
||||||
redis.getString.mockResolvedValue('another-consumer');
|
makeScheduler();
|
||||||
|
distributedLockService.acquireLock.mockResolvedValueOnce(null);
|
||||||
|
|
||||||
await scheduler.tick();
|
await scheduler.tick();
|
||||||
|
|
||||||
|
|||||||
@@ -9,13 +9,17 @@ import {
|
|||||||
RiotAccountEnrichmentService,
|
RiotAccountEnrichmentService,
|
||||||
} from '../riot/riot-account-enrichment.service';
|
} from '../riot/riot-account-enrichment.service';
|
||||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||||
|
import {
|
||||||
|
type DistributedLockOwnership,
|
||||||
|
DistributedLockService,
|
||||||
|
} from '../global/redis/distributed-lock.service';
|
||||||
import { RedisService } from '../global/redis/redis.service';
|
import { RedisService } from '../global/redis/redis.service';
|
||||||
import { isInKstBackgroundWindow } from './kst-window.util';
|
import { isInKstBackgroundWindow } from './kst-window.util';
|
||||||
import { StaleAccountRepository } from './stale-account.repository';
|
import { StaleAccountRepository } from './stale-account.repository';
|
||||||
|
|
||||||
const TICK_INTERVAL_MS = 60_000;
|
const TICK_INTERVAL_MS = 60_000;
|
||||||
const LEADER_KEY = 'riot:bg-refresh:leader';
|
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
|
||||||
const LEADER_TTL_SECONDS = 90;
|
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
|
||||||
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
||||||
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
||||||
const PER_TICK_CANDIDATE_LIMIT = 20;
|
const PER_TICK_CANDIDATE_LIMIT = 20;
|
||||||
@@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler
|
|||||||
private tickTimer?: NodeJS.Timeout;
|
private tickTimer?: NodeJS.Timeout;
|
||||||
private leaderRenewTimer?: NodeJS.Timeout;
|
private leaderRenewTimer?: NodeJS.Timeout;
|
||||||
private tickInProgress = false;
|
private tickInProgress = false;
|
||||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly redis: RedisService,
|
private readonly redis: RedisService,
|
||||||
|
private readonly distributedLockService: DistributedLockService,
|
||||||
private readonly rateLimiter: RiotRateLimiterService,
|
private readonly rateLimiter: RiotRateLimiterService,
|
||||||
private readonly enrichmentService: RiotAccountEnrichmentService,
|
private readonly enrichmentService: RiotAccountEnrichmentService,
|
||||||
private readonly staleAccountRepository: StaleAccountRepository,
|
private readonly staleAccountRepository: StaleAccountRepository,
|
||||||
@@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler
|
|||||||
void this.safeTick();
|
void this.safeTick();
|
||||||
}
|
}
|
||||||
|
|
||||||
public onModuleDestroy(): void {
|
public async onModuleDestroy(): Promise<void> {
|
||||||
if (this.tickTimer) clearInterval(this.tickTimer);
|
if (this.tickTimer) clearInterval(this.tickTimer);
|
||||||
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
||||||
|
if (!this.leaderLockOwnership) return;
|
||||||
|
await this.distributedLockService
|
||||||
|
.releaseLockIfOwned(this.leaderLockOwnership)
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
this.logger.warn(
|
||||||
|
`Background refresh leader release failed: ${
|
||||||
|
err instanceof Error ? err.message : err
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
this.leaderLockOwnership = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async safeTick(): Promise<void> {
|
private async safeTick(): Promise<void> {
|
||||||
@@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async tryBecomeLeader(): Promise<boolean> {
|
private async tryBecomeLeader(): Promise<boolean> {
|
||||||
const current = await this.redis.getString(LEADER_KEY);
|
if (this.leaderLockOwnership) {
|
||||||
if (current === this.consumer) {
|
const didExtendLeaderLock =
|
||||||
await this.redis.setString(
|
await this.distributedLockService.extendLockIfOwned(
|
||||||
LEADER_KEY,
|
this.leaderLockOwnership,
|
||||||
this.consumer,
|
);
|
||||||
LEADER_TTL_SECONDS,
|
if (didExtendLeaderLock) return true;
|
||||||
);
|
this.leaderLockOwnership = undefined;
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
if (current) return false;
|
|
||||||
|
|
||||||
return this.redis.setStringIfAbsent(
|
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
||||||
LEADER_KEY,
|
lockKey: LEADER_LOCK_KEY,
|
||||||
this.consumer,
|
lockExpirationDurationMilliseconds:
|
||||||
LEADER_TTL_SECONDS,
|
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
);
|
});
|
||||||
|
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
||||||
|
return Boolean(leaderLockOwnership);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async renewLeaderIfHeld(): Promise<void> {
|
private async renewLeaderIfHeld(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const current = await this.redis.getString(LEADER_KEY);
|
if (!this.leaderLockOwnership) return;
|
||||||
if (current !== this.consumer) return;
|
const didExtendLeaderLock =
|
||||||
await this.redis.setString(
|
await this.distributedLockService.extendLockIfOwned(
|
||||||
LEADER_KEY,
|
this.leaderLockOwnership,
|
||||||
this.consumer,
|
);
|
||||||
LEADER_TTL_SECONDS,
|
if (didExtendLeaderLock) return;
|
||||||
);
|
this.leaderLockOwnership = undefined;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`Background refresh leader renew failed: ${
|
`Background refresh leader renew failed: ${
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import mongoose from 'mongoose';
|
import mongoose from 'mongoose';
|
||||||
import { ConnectionService } from '../global/database/connection.service';
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import { RedisService } from '../global/redis/redis.service';
|
import { RedisService } from '../global/redis/redis.service';
|
||||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||||
@@ -19,6 +20,16 @@ function makeWorker() {
|
|||||||
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
||||||
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
|
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
|
||||||
} as unknown as jest.Mocked<RedisService>;
|
} as unknown as jest.Mocked<RedisService>;
|
||||||
|
const distributedLockService = {
|
||||||
|
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||||
|
operation({
|
||||||
|
lockKey: options.lockKey,
|
||||||
|
lockOwnershipToken: 'lock-owner',
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
options.lockExpirationDurationMilliseconds,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
} as unknown as jest.Mocked<DistributedLockService>;
|
||||||
const jobQueueService = {
|
const jobQueueService = {
|
||||||
markProcessing: jest.fn().mockResolvedValue(undefined),
|
markProcessing: jest.fn().mockResolvedValue(undefined),
|
||||||
markCompleted: jest.fn().mockResolvedValue({
|
markCompleted: jest.fn().mockResolvedValue({
|
||||||
@@ -54,6 +65,7 @@ function makeWorker() {
|
|||||||
|
|
||||||
const worker = new RiotJobWorkerService(
|
const worker = new RiotJobWorkerService(
|
||||||
redisService,
|
redisService,
|
||||||
|
distributedLockService,
|
||||||
jobQueueService,
|
jobQueueService,
|
||||||
recentMatchesService,
|
recentMatchesService,
|
||||||
connectionService,
|
connectionService,
|
||||||
@@ -66,6 +78,7 @@ function makeWorker() {
|
|||||||
return {
|
return {
|
||||||
worker,
|
worker,
|
||||||
redisService,
|
redisService,
|
||||||
|
distributedLockService,
|
||||||
jobQueueService,
|
jobQueueService,
|
||||||
recentMatchesService,
|
recentMatchesService,
|
||||||
connectionService,
|
connectionService,
|
||||||
@@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => {
|
|||||||
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
|
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
|
||||||
|
const { worker, distributedLockService, redisService } = makeWorker();
|
||||||
|
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
|
||||||
|
const userId = new mongoose.Types.ObjectId().toString();
|
||||||
|
const job: RiotRefreshSubJob = {
|
||||||
|
jobId: 'job-1',
|
||||||
|
parentJobId: 'parent-1',
|
||||||
|
userId,
|
||||||
|
puuid: 'puuid-1',
|
||||||
|
region: 'asia',
|
||||||
|
platform: 'kr',
|
||||||
|
queueGroup: 'summoners-rift',
|
||||||
|
queueIds: [420],
|
||||||
|
requestedAt: Date.now(),
|
||||||
|
priority: 'high',
|
||||||
|
};
|
||||||
|
|
||||||
|
await (
|
||||||
|
worker as unknown as {
|
||||||
|
processEntry: (
|
||||||
|
stream: string,
|
||||||
|
entry: { id: string; message: Record<string, string> },
|
||||||
|
) => Promise<void>;
|
||||||
|
}
|
||||||
|
).processEntry('riot:queue:high', {
|
||||||
|
id: '1-0',
|
||||||
|
message: { job: JSON.stringify(job) },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
|
||||||
|
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -6,6 +6,10 @@ import {
|
|||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import mongoose from 'mongoose';
|
import mongoose from 'mongoose';
|
||||||
import { ConnectionService } from '../global/database/connection.service';
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import {
|
||||||
|
DistributedLockOwnership,
|
||||||
|
DistributedLockService,
|
||||||
|
} from '../global/redis/distributed-lock.service';
|
||||||
import { RedisService, StreamEntry } from '../global/redis/redis.service';
|
import { RedisService, StreamEntry } from '../global/redis/redis.service';
|
||||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||||
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
|
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
|
||||||
@@ -23,17 +27,24 @@ import {
|
|||||||
import { RiotJobQueueService } from './riot-job-queue.service';
|
import { RiotJobQueueService } from './riot-job-queue.service';
|
||||||
import { RiotRefreshGateway } from './riot-refresh.gateway';
|
import { RiotRefreshGateway } from './riot-refresh.gateway';
|
||||||
|
|
||||||
const LEADER_KEY = 'riot:worker:leader';
|
const RIOT_JOB_WORKER_LEADER_LOCK_KEY =
|
||||||
const LEADER_TTL_SECONDS = 60;
|
'distributed-lock:riot-job-worker:leader';
|
||||||
|
const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
|
||||||
|
const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000;
|
||||||
|
const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000;
|
||||||
|
const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000;
|
||||||
|
const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||||
private readonly logger = new Logger(RiotJobWorkerService.name);
|
private readonly logger = new Logger(RiotJobWorkerService.name);
|
||||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||||
|
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
||||||
private stopped = false;
|
private stopped = false;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly redisService: RedisService,
|
private readonly redisService: RedisService,
|
||||||
|
private readonly distributedLockService: DistributedLockService,
|
||||||
private readonly jobQueueService: RiotJobQueueService,
|
private readonly jobQueueService: RiotJobQueueService,
|
||||||
private readonly recentMatchesService: RiotRecentMatchesService,
|
private readonly recentMatchesService: RiotRecentMatchesService,
|
||||||
private readonly connectionService: ConnectionService,
|
private readonly connectionService: ConnectionService,
|
||||||
@@ -48,8 +59,19 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
void this.loop();
|
void this.loop();
|
||||||
}
|
}
|
||||||
|
|
||||||
public onModuleDestroy() {
|
public async onModuleDestroy() {
|
||||||
this.stopped = true;
|
this.stopped = true;
|
||||||
|
if (!this.leaderLockOwnership) return;
|
||||||
|
await this.distributedLockService
|
||||||
|
.releaseLockIfOwned(this.leaderLockOwnership)
|
||||||
|
.catch((error: unknown) => {
|
||||||
|
this.logger.warn(
|
||||||
|
`Failed to release Riot job worker leader lock: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
this.leaderLockOwnership = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async loop(): Promise<void> {
|
private async loop(): Promise<void> {
|
||||||
@@ -63,7 +85,9 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
|
|
||||||
await this.jobQueueService.promoteDueScheduledJobs();
|
await this.jobQueueService.promoteDueScheduledJobs();
|
||||||
const entry =
|
const entry =
|
||||||
|
(await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ??
|
||||||
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
|
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
|
||||||
|
(await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ??
|
||||||
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
|
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
|
||||||
if (!entry) continue;
|
if (!entry) continue;
|
||||||
|
|
||||||
@@ -89,6 +113,31 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const didProcessJob = await this.distributedLockService.runWithRenewedLock(
|
||||||
|
{
|
||||||
|
lockKey: this.jobProcessingLockKey(job),
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
|
lockRenewalIntervalMilliseconds:
|
||||||
|
RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
await this.processJob(job);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!didProcessJob) {
|
||||||
|
this.logger.debug(
|
||||||
|
`Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.acknowledgeAndDelete(stream, entry.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async processJob(job: RiotJobPayload): Promise<void> {
|
||||||
try {
|
try {
|
||||||
if (this.isTournamentResultJob(job)) {
|
if (this.isTournamentResultJob(job)) {
|
||||||
await this.tournamentResultService.processPollJob(job);
|
await this.tournamentResultService.processPollJob(job);
|
||||||
@@ -136,8 +185,6 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
error instanceof Error ? error.message : error
|
error instanceof Error ? error.message : error
|
||||||
}`,
|
}`,
|
||||||
);
|
);
|
||||||
} finally {
|
|
||||||
await this.acknowledgeAndDelete(stream, entry.id);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,15 +275,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
): Promise<{ stream: string; message: StreamEntry } | null> {
|
): Promise<{ stream: string; message: StreamEntry } | null> {
|
||||||
const result = await this.redisService.readGroup(
|
const result = await this.redisService.readGroup(
|
||||||
stream,
|
stream,
|
||||||
RIOT_JOB_GROUP,
|
RIOT_JOB_GROUP,
|
||||||
this.consumer,
|
this.consumerIdentifier,
|
||||||
'>',
|
'>',
|
||||||
{ count: 1, blockMs },
|
{ count: 1, blockMs },
|
||||||
);
|
);
|
||||||
const message = result?.[0]?.messages[0];
|
const message = result?.[0]?.messages[0];
|
||||||
return message ? { stream, message } : null;
|
return message ? { stream, message } : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async readOneStalePendingMessage(
|
||||||
|
stream: string,
|
||||||
|
): Promise<{ stream: string; message: StreamEntry } | null> {
|
||||||
|
const result = await this.redisService.claimStalePendingStreamMessages({
|
||||||
|
key: stream,
|
||||||
|
group: RIOT_JOB_GROUP,
|
||||||
|
consumer: this.consumerIdentifier,
|
||||||
|
stalePendingMessageMinimumIdleMilliseconds:
|
||||||
|
STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
|
||||||
|
count: STALE_PENDING_MESSAGE_CLAIM_COUNT,
|
||||||
|
});
|
||||||
|
const message = result.messages[0];
|
||||||
|
return message ? { stream, message } : null;
|
||||||
|
}
|
||||||
|
|
||||||
private async ensureConsumerGroups(): Promise<void> {
|
private async ensureConsumerGroups(): Promise<void> {
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
|
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
|
||||||
@@ -252,21 +314,26 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async tryBecomeLeader(): Promise<boolean> {
|
private async tryBecomeLeader(): Promise<boolean> {
|
||||||
const current = await this.redisService.getString(LEADER_KEY);
|
if (this.leaderLockOwnership) {
|
||||||
if (current === this.consumer) {
|
const didExtendLeaderLock =
|
||||||
await this.redisService.setString(
|
await this.distributedLockService.extendLockIfOwned(
|
||||||
LEADER_KEY,
|
this.leaderLockOwnership,
|
||||||
this.consumer,
|
);
|
||||||
LEADER_TTL_SECONDS,
|
if (didExtendLeaderLock) return true;
|
||||||
);
|
this.leaderLockOwnership = undefined;
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
if (current) return false;
|
|
||||||
return this.redisService.setStringIfAbsent(
|
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
||||||
LEADER_KEY,
|
lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY,
|
||||||
this.consumer,
|
lockExpirationDurationMilliseconds:
|
||||||
LEADER_TTL_SECONDS,
|
RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
);
|
});
|
||||||
|
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
||||||
|
return Boolean(leaderLockOwnership);
|
||||||
|
}
|
||||||
|
|
||||||
|
private jobProcessingLockKey(job: RiotJobPayload): string {
|
||||||
|
return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
private sleep(ms: number): Promise<void> {
|
private sleep(ms: number): Promise<void> {
|
||||||
|
|||||||
@@ -4,14 +4,14 @@ import type {
|
|||||||
RiotMatch as StoredRiotMatch,
|
RiotMatch as StoredRiotMatch,
|
||||||
} from '../global/database/schema';
|
} from '../global/database/schema';
|
||||||
import { ConnectionService } from '../global/database/connection.service';
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
import { RedisService } from '../global/redis/redis.service';
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import type { RiotRegionalRoute } from './riot.types';
|
import type { RiotRegionalRoute } from './riot.types';
|
||||||
import { RiotMatchService } from './riot-match.service';
|
import { RiotMatchService } from './riot-match.service';
|
||||||
|
|
||||||
export const RIOT_RECENT_MATCH_LIMIT = 20;
|
export const RIOT_RECENT_MATCH_LIMIT = 20;
|
||||||
export const RIOT_SOLO_QUEUE_ID = 420;
|
export const RIOT_SOLO_QUEUE_ID = 420;
|
||||||
|
|
||||||
const MATCH_ENRICHMENT_LOCK_TTL_SECONDS = 60;
|
const MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
|
||||||
const MATCH_DETAIL_CONCURRENCY = 2;
|
const MATCH_DETAIL_CONCURRENCY = 2;
|
||||||
const DEFAULT_REGION: RiotRegionalRoute = 'asia';
|
const DEFAULT_REGION: RiotRegionalRoute = 'asia';
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ export class RiotRecentMatchesService {
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly connectionService: ConnectionService,
|
private readonly connectionService: ConnectionService,
|
||||||
private readonly matchService: RiotMatchService,
|
private readonly matchService: RiotMatchService,
|
||||||
private readonly redisService: RedisService,
|
private readonly distributedLockService: DistributedLockService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
public async syncRecentMatches(params: {
|
public async syncRecentMatches(params: {
|
||||||
@@ -34,83 +34,84 @@ export class RiotRecentMatchesService {
|
|||||||
const region = params.region ?? DEFAULT_REGION;
|
const region = params.region ?? DEFAULT_REGION;
|
||||||
const limit = params.limit ?? RIOT_RECENT_MATCH_LIMIT;
|
const limit = params.limit ?? RIOT_RECENT_MATCH_LIMIT;
|
||||||
const queueId = params.queueId ?? RIOT_SOLO_QUEUE_ID;
|
const queueId = params.queueId ?? RIOT_SOLO_QUEUE_ID;
|
||||||
const lockKey = `riot:enrich:matches:${params.puuid}`;
|
const result = await this.distributedLockService.runWithLock(
|
||||||
const lockValue = `${process.pid}:${Date.now()}`;
|
{
|
||||||
|
lockKey: `distributed-lock:riot-recent-matches:sync:puuid:${params.puuid}`,
|
||||||
const acquired = await this.redisService.setStringIfAbsent(
|
lockExpirationDurationMilliseconds:
|
||||||
lockKey,
|
MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
lockValue,
|
},
|
||||||
MATCH_ENRICHMENT_LOCK_TTL_SECONDS,
|
async () => {
|
||||||
);
|
const matchIds = await this.matchService.getMatchIdsByPuuid(
|
||||||
if (!acquired) {
|
params.puuid,
|
||||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
{
|
||||||
}
|
region,
|
||||||
|
queue: queueId,
|
||||||
try {
|
count: limit,
|
||||||
const matchIds = await this.matchService.getMatchIdsByPuuid(params.puuid, {
|
},
|
||||||
region,
|
|
||||||
queue: queueId,
|
|
||||||
count: limit,
|
|
||||||
});
|
|
||||||
if (matchIds.length === 0) {
|
|
||||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
|
||||||
}
|
|
||||||
|
|
||||||
const RiotMatchModel = await this.connectionService.getModel('riotMatch');
|
|
||||||
const existingMatches = await RiotMatchModel.find({
|
|
||||||
matchId: { $in: matchIds },
|
|
||||||
})
|
|
||||||
.select({ matchId: 1 })
|
|
||||||
.lean()
|
|
||||||
.exec();
|
|
||||||
const existingMatchIds = new Set(existingMatches.map((match) => match.matchId));
|
|
||||||
const missingMatchIds = matchIds.filter((matchId) => !existingMatchIds.has(matchId));
|
|
||||||
|
|
||||||
await runWithConcurrency(missingMatchIds, MATCH_DETAIL_CONCURRENCY, async (matchId) => {
|
|
||||||
try {
|
|
||||||
const dto = await this.matchService.getMatch(matchId, { region });
|
|
||||||
if (dto.info.queueId !== queueId) {
|
|
||||||
this.logger.warn(
|
|
||||||
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const payload: Partial<StoredRiotMatch> = {
|
|
||||||
matchId: dto.metadata.matchId,
|
|
||||||
gameCreation: dto.info.gameCreation,
|
|
||||||
gameDuration: dto.info.gameDuration,
|
|
||||||
gameVersion: dto.info.gameVersion,
|
|
||||||
queueId: dto.info.queueId,
|
|
||||||
participantPuuids: dto.metadata.participants,
|
|
||||||
raw: dto as unknown as Record<string, unknown>,
|
|
||||||
fetchedAt: new Date(),
|
|
||||||
};
|
|
||||||
|
|
||||||
await RiotMatchModel.updateOne(
|
|
||||||
{ matchId: dto.metadata.matchId },
|
|
||||||
{ $setOnInsert: payload },
|
|
||||||
{ upsert: true },
|
|
||||||
).exec();
|
|
||||||
} catch (error: unknown) {
|
|
||||||
this.logger.warn(
|
|
||||||
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
|
|
||||||
error instanceof Error ? error.message : error
|
|
||||||
}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
|
||||||
} finally {
|
|
||||||
await this.redisService.delete(lockKey).catch((error: unknown) => {
|
|
||||||
this.logger.warn(
|
|
||||||
`Failed to release recent matches lock for ${params.puuid}: ${
|
|
||||||
(error as Error)?.message ?? error
|
|
||||||
}`,
|
|
||||||
);
|
);
|
||||||
});
|
if (matchIds.length === 0) {
|
||||||
}
|
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||||
|
}
|
||||||
|
|
||||||
|
const RiotMatchModel =
|
||||||
|
await this.connectionService.getModel('riotMatch');
|
||||||
|
const existingMatches = await RiotMatchModel.find({
|
||||||
|
matchId: { $in: matchIds },
|
||||||
|
})
|
||||||
|
.select({ matchId: 1 })
|
||||||
|
.lean()
|
||||||
|
.exec();
|
||||||
|
const existingMatchIds = new Set(
|
||||||
|
existingMatches.map((match) => match.matchId),
|
||||||
|
);
|
||||||
|
const missingMatchIds = matchIds.filter(
|
||||||
|
(matchId) => !existingMatchIds.has(matchId),
|
||||||
|
);
|
||||||
|
|
||||||
|
await runWithConcurrency(
|
||||||
|
missingMatchIds,
|
||||||
|
MATCH_DETAIL_CONCURRENCY,
|
||||||
|
async (matchId) => {
|
||||||
|
try {
|
||||||
|
const dto = await this.matchService.getMatch(matchId, { region });
|
||||||
|
if (dto.info.queueId !== queueId) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload: Partial<StoredRiotMatch> = {
|
||||||
|
matchId: dto.metadata.matchId,
|
||||||
|
gameCreation: dto.info.gameCreation,
|
||||||
|
gameDuration: dto.info.gameDuration,
|
||||||
|
gameVersion: dto.info.gameVersion,
|
||||||
|
queueId: dto.info.queueId,
|
||||||
|
participantPuuids: dto.metadata.participants,
|
||||||
|
raw: dto as unknown as Record<string, unknown>,
|
||||||
|
fetchedAt: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
await RiotMatchModel.updateOne(
|
||||||
|
{ matchId: dto.metadata.matchId },
|
||||||
|
{ $setOnInsert: payload },
|
||||||
|
{ upsert: true },
|
||||||
|
).exec();
|
||||||
|
} catch (error: unknown) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
|
||||||
|
error instanceof Error ? error.message : error
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return result ?? this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||||
}
|
}
|
||||||
|
|
||||||
public async ensureFreshForQueue(params: {
|
public async ensureFreshForQueue(params: {
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import { TournamentCodeScheduler } from './tournament-code.scheduler';
|
import { TournamentCodeScheduler } from './tournament-code.scheduler';
|
||||||
|
|
||||||
describe('TournamentCodeScheduler', () => {
|
describe('TournamentCodeScheduler', () => {
|
||||||
@@ -18,16 +19,28 @@ describe('TournamentCodeScheduler', () => {
|
|||||||
scheduleInitialPoll:
|
scheduleInitialPoll:
|
||||||
params?.scheduleInitialPoll ?? jest.fn().mockResolvedValue(null),
|
params?.scheduleInitialPoll ?? jest.fn().mockResolvedValue(null),
|
||||||
};
|
};
|
||||||
|
const distributedLockService = {
|
||||||
|
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||||
|
operation({
|
||||||
|
lockKey: options.lockKey,
|
||||||
|
lockOwnershipToken: 'lock-owner',
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
options.lockExpirationDurationMilliseconds,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
} as unknown as jest.Mocked<DistributedLockService>;
|
||||||
const scheduler = new TournamentCodeScheduler(
|
const scheduler = new TournamentCodeScheduler(
|
||||||
tournamentCodeService as never,
|
tournamentCodeService as never,
|
||||||
autoProvisionService as never,
|
autoProvisionService as never,
|
||||||
tournamentResultService as never,
|
tournamentResultService as never,
|
||||||
|
distributedLockService as never,
|
||||||
) as unknown as { attachPending(): Promise<void> };
|
) as unknown as { attachPending(): Promise<void> };
|
||||||
|
|
||||||
return {
|
return {
|
||||||
scheduler,
|
scheduler,
|
||||||
tournamentCodeService,
|
tournamentCodeService,
|
||||||
tournamentResultService,
|
tournamentResultService,
|
||||||
|
distributedLockService,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,4 +72,16 @@ describe('TournamentCodeScheduler', () => {
|
|||||||
|
|
||||||
expect(tournamentResultService.scheduleInitialPoll).not.toHaveBeenCalled();
|
expect(tournamentResultService.scheduleInitialPoll).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not attach pending tournament codes when another instance owns the scheduler lock', async () => {
|
||||||
|
const { scheduler, tournamentCodeService, distributedLockService } =
|
||||||
|
createScheduler();
|
||||||
|
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
|
||||||
|
|
||||||
|
await scheduler.attachPending();
|
||||||
|
|
||||||
|
expect(
|
||||||
|
tournamentCodeService.attachPendingMatchDocuments,
|
||||||
|
).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -1,8 +1,18 @@
|
|||||||
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||||
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
|
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
|
||||||
import { TournamentCodeService } from './tournament-code.service';
|
import { TournamentCodeService } from './tournament-code.service';
|
||||||
|
|
||||||
|
const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY =
|
||||||
|
'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches';
|
||||||
|
const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY =
|
||||||
|
'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes';
|
||||||
|
const ENSURE_VENDOR_CODE_POOL_LOCK_KEY =
|
||||||
|
'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool';
|
||||||
|
const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000;
|
||||||
|
const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||||
private readonly logger = new Logger(TournamentCodeScheduler.name);
|
private readonly logger = new Logger(TournamentCodeScheduler.name);
|
||||||
@@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
|||||||
private readonly tournamentCodeService: TournamentCodeService,
|
private readonly tournamentCodeService: TournamentCodeService,
|
||||||
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
|
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
|
||||||
private readonly tournamentResultService: TournamentResultService,
|
private readonly tournamentResultService: TournamentResultService,
|
||||||
|
private readonly distributedLockService: DistributedLockService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
public onModuleInit() {
|
public onModuleInit() {
|
||||||
@@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
|||||||
|
|
||||||
private async attachPending() {
|
private async attachPending() {
|
||||||
try {
|
try {
|
||||||
const matches =
|
await this.distributedLockService.runWithRenewedLock(
|
||||||
await this.tournamentCodeService.attachPendingMatchDocuments();
|
{
|
||||||
if (matches.length > 0) {
|
lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY,
|
||||||
this.logger.debug(
|
lockExpirationDurationMilliseconds:
|
||||||
`Attached tournament codes for ${matches.length} matches.`,
|
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
);
|
lockRenewalIntervalMilliseconds:
|
||||||
for (const match of matches) {
|
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||||
await this.tournamentResultService.scheduleInitialPoll(match);
|
},
|
||||||
}
|
async () => {
|
||||||
}
|
const matches =
|
||||||
|
await this.tournamentCodeService.attachPendingMatchDocuments();
|
||||||
|
if (matches.length > 0) {
|
||||||
|
this.logger.debug(
|
||||||
|
`Attached tournament codes for ${matches.length} matches.`,
|
||||||
|
);
|
||||||
|
for (const match of matches) {
|
||||||
|
await this.tournamentResultService.scheduleInitialPoll(match);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`Failed to attach pending tournament codes: ${
|
`Failed to attach pending tournament codes: ${
|
||||||
@@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
|||||||
|
|
||||||
private async markExpired() {
|
private async markExpired() {
|
||||||
try {
|
try {
|
||||||
await this.tournamentCodeService.markExpired();
|
await this.distributedLockService.runWithRenewedLock(
|
||||||
|
{
|
||||||
|
lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY,
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
|
lockRenewalIntervalMilliseconds:
|
||||||
|
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
await this.tournamentCodeService.markExpired();
|
||||||
|
},
|
||||||
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`Failed to mark expired tournament codes: ${
|
`Failed to mark expired tournament codes: ${
|
||||||
@@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
|||||||
|
|
||||||
private async ensureVendorPool() {
|
private async ensureVendorPool() {
|
||||||
try {
|
try {
|
||||||
await this.autoProvisionService.ensurePool();
|
await this.distributedLockService.runWithRenewedLock(
|
||||||
|
{
|
||||||
|
lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY,
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
|
lockRenewalIntervalMilliseconds:
|
||||||
|
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
await this.autoProvisionService.ensurePool();
|
||||||
|
},
|
||||||
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`Failed to auto-provision tournament codes: ${
|
`Failed to auto-provision tournament codes: ${
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import mongoose from 'mongoose';
|
import mongoose from 'mongoose';
|
||||||
import type { MatchDocument } from '../global/database/schema';
|
import type { MatchDocument } from '../global/database/schema';
|
||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import type { RiotMatchDto } from '../riot/riot.types';
|
import type { RiotMatchDto } from '../riot/riot.types';
|
||||||
import { TournamentResultService } from './tournament-result.service';
|
import { TournamentResultService } from './tournament-result.service';
|
||||||
|
|
||||||
@@ -16,15 +17,38 @@ type TournamentResultServiceInternals = TournamentResultService & {
|
|||||||
};
|
};
|
||||||
|
|
||||||
function createService() {
|
function createService() {
|
||||||
|
const distributedLockService = createDistributedLockService();
|
||||||
return new TournamentResultService(
|
return new TournamentResultService(
|
||||||
{} as never,
|
{} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
|
distributedLockService as never,
|
||||||
) as unknown as TournamentResultServiceInternals;
|
) as unknown as TournamentResultServiceInternals;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createDistributedLockService() {
|
||||||
|
return {
|
||||||
|
runWithLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||||
|
operation({
|
||||||
|
lockKey: options.lockKey,
|
||||||
|
lockOwnershipToken: 'lock-owner',
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
options.lockExpirationDurationMilliseconds,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||||
|
operation({
|
||||||
|
lockKey: options.lockKey,
|
||||||
|
lockOwnershipToken: 'lock-owner',
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
options.lockExpirationDurationMilliseconds,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
} as unknown as jest.Mocked<DistributedLockService>;
|
||||||
|
}
|
||||||
|
|
||||||
interface MatchOverrides {
|
interface MatchOverrides {
|
||||||
gameCount?: number;
|
gameCount?: number;
|
||||||
gameMode?: 'pair' | 'bracket' | 'round-robin';
|
gameMode?: 'pair' | 'bracket' | 'round-robin';
|
||||||
@@ -281,12 +305,14 @@ describe('TournamentResultService tournament code polling', () => {
|
|||||||
const riotJobQueueService = {
|
const riotJobQueueService = {
|
||||||
enqueueTournamentResultPoll: jest.fn().mockResolvedValue(undefined),
|
enqueueTournamentResultPoll: jest.fn().mockResolvedValue(undefined),
|
||||||
};
|
};
|
||||||
|
const distributedLockService = createDistributedLockService();
|
||||||
const service = new TournamentResultService(
|
const service = new TournamentResultService(
|
||||||
connectionService as never,
|
connectionService as never,
|
||||||
riotMatchService as never,
|
riotMatchService as never,
|
||||||
riotJobQueueService as never,
|
riotJobQueueService as never,
|
||||||
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
||||||
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
|
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
|
||||||
|
distributedLockService as never,
|
||||||
);
|
);
|
||||||
|
|
||||||
return { service, match, code, riotMatchService };
|
return { service, match, code, riotMatchService };
|
||||||
@@ -455,12 +481,14 @@ describe('TournamentResultService manual result collection', () => {
|
|||||||
getMatchIdsByPuuid: jest.fn().mockResolvedValue(['KR_1234567890']),
|
getMatchIdsByPuuid: jest.fn().mockResolvedValue(['KR_1234567890']),
|
||||||
getMatch: jest.fn().mockResolvedValue(detail),
|
getMatch: jest.fn().mockResolvedValue(detail),
|
||||||
};
|
};
|
||||||
|
const distributedLockService = createDistributedLockService();
|
||||||
const service = new TournamentResultService(
|
const service = new TournamentResultService(
|
||||||
connectionService as never,
|
connectionService as never,
|
||||||
riotMatchService as never,
|
riotMatchService as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
||||||
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
|
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
|
||||||
|
distributedLockService as never,
|
||||||
);
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||||
import { ConnectionService } from '../global/database/connection.service';
|
import { ConnectionService } from '../global/database/connection.service';
|
||||||
|
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||||
import type {
|
import type {
|
||||||
MatchDocument,
|
MatchDocument,
|
||||||
MatchGameResult,
|
MatchGameResult,
|
||||||
@@ -46,6 +47,12 @@ const POLL_EXPIRY_MS = 24 * 60 * 60_000;
|
|||||||
const EXPAND_AFTER_MS = 60 * 60_000;
|
const EXPAND_AFTER_MS = 60 * 60_000;
|
||||||
const EXPANDED_MEMBER_LIMIT_PER_TEAM = 2;
|
const EXPANDED_MEMBER_LIMIT_PER_TEAM = 2;
|
||||||
const TOURNAMENT_LOOKUP_START_MARGIN_SECONDS = 60;
|
const TOURNAMENT_LOOKUP_START_MARGIN_SECONDS = 60;
|
||||||
|
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS =
|
||||||
|
30 * 60_000;
|
||||||
|
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS =
|
||||||
|
5 * 60_000;
|
||||||
|
const TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS =
|
||||||
|
5 * 60_000;
|
||||||
|
|
||||||
interface ManualGameAssignment {
|
interface ManualGameAssignment {
|
||||||
gameNumber: number;
|
gameNumber: number;
|
||||||
@@ -64,11 +71,29 @@ export class TournamentResultService {
|
|||||||
private readonly riotJobQueueService: RiotJobQueueService,
|
private readonly riotJobQueueService: RiotJobQueueService,
|
||||||
private readonly communityStatsService: CommunityStatsService,
|
private readonly communityStatsService: CommunityStatsService,
|
||||||
private readonly matchStatsService: MatchStatsService,
|
private readonly matchStatsService: MatchStatsService,
|
||||||
|
private readonly distributedLockService: DistributedLockService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
public async scheduleInitialPoll(
|
public async scheduleInitialPoll(
|
||||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||||
priority: RiotJobPriority = 'high',
|
priority: RiotJobPriority = 'high',
|
||||||
|
): Promise<EnqueueTournamentResultPollResult | null> {
|
||||||
|
const matchIdentifier = this.resolveMatchIdentifier(matchOrId);
|
||||||
|
return this.distributedLockService.runWithLock(
|
||||||
|
{
|
||||||
|
lockKey: this.tournamentResultPollProcessingLockKey(matchIdentifier),
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
|
lockAcquireTimeoutMilliseconds: 1_000,
|
||||||
|
lockRetryDelayMilliseconds: 100,
|
||||||
|
},
|
||||||
|
async () => this.scheduleInitialPollWhileLockOwned(matchOrId, priority),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async scheduleInitialPollWhileLockOwned(
|
||||||
|
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||||
|
priority: RiotJobPriority,
|
||||||
): Promise<EnqueueTournamentResultPollResult | null> {
|
): Promise<EnqueueTournamentResultPollResult | null> {
|
||||||
const match = await this.loadMatch(matchOrId);
|
const match = await this.loadMatch(matchOrId);
|
||||||
if (match.status === 'finished') return null;
|
if (match.status === 'finished') return null;
|
||||||
@@ -195,6 +220,29 @@ export class TournamentResultService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
|
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
|
||||||
|
const didProcessPollJob = await this.distributedLockService.runWithRenewedLock(
|
||||||
|
{
|
||||||
|
lockKey: this.tournamentResultPollProcessingLockKey(job.matchId),
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
|
lockRenewalIntervalMilliseconds:
|
||||||
|
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
await this.processPollJobWhileLockOwned(job);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (didProcessPollJob) return;
|
||||||
|
this.logger.debug(
|
||||||
|
`Skipped tournament result poll job ${job.jobId}; another worker is processing match ${job.matchId}.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async processPollJobWhileLockOwned(
|
||||||
|
job: PollTournamentResultJob,
|
||||||
|
): Promise<void> {
|
||||||
const MatchModel = await this.connectionService.getModel('match');
|
const MatchModel = await this.connectionService.getModel('match');
|
||||||
const TournamentCodeModel =
|
const TournamentCodeModel =
|
||||||
await this.connectionService.getModel('tournamentCode');
|
await this.connectionService.getModel('tournamentCode');
|
||||||
@@ -818,12 +866,28 @@ export class TournamentResultService {
|
|||||||
|
|
||||||
private async reserveReadyDoubleEliminationCodes(
|
private async reserveReadyDoubleEliminationCodes(
|
||||||
match: MatchDocument,
|
match: MatchDocument,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.distributedLockService.runWithLock(
|
||||||
|
{
|
||||||
|
lockKey: this.tournamentResultReadyCodeReservationLockKey(
|
||||||
|
match._id.toString(),
|
||||||
|
),
|
||||||
|
lockExpirationDurationMilliseconds:
|
||||||
|
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||||
|
},
|
||||||
|
async () => this.reserveReadyDoubleEliminationCodesWhileLockOwned(match),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async reserveReadyDoubleEliminationCodesWhileLockOwned(
|
||||||
|
match: MatchDocument,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if ((match.gameMode ?? 'pair') !== 'double-elimination') return;
|
if ((match.gameMode ?? 'pair') !== 'double-elimination') return;
|
||||||
const readyBracketMatches = (match.doubleElimination?.bracketMatches ?? [])
|
const readyBracketMatches = (match.doubleElimination?.bracketMatches ?? [])
|
||||||
.filter((bracketMatch) => bracketMatch.state === 'ready');
|
.filter((bracketMatch) => bracketMatch.state === 'ready');
|
||||||
if (readyBracketMatches.length === 0) return;
|
if (readyBracketMatches.length === 0) return;
|
||||||
|
|
||||||
|
const MatchModel = await this.connectionService.getModel('match');
|
||||||
const TournamentCodeModel =
|
const TournamentCodeModel =
|
||||||
await this.connectionService.getModel('tournamentCode');
|
await this.connectionService.getModel('tournamentCode');
|
||||||
const existingCodes =
|
const existingCodes =
|
||||||
@@ -871,8 +935,19 @@ export class TournamentResultService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (reservedIds.length === 0) return;
|
if (reservedIds.length === 0) return;
|
||||||
match.tournamentCodeIds.push(...reservedIds);
|
await MatchModel.updateOne(
|
||||||
await match.save();
|
{ _id: match._id },
|
||||||
|
{ $addToSet: { tournamentCodeIds: { $each: reservedIds } } },
|
||||||
|
).exec();
|
||||||
|
const existingTournamentCodeIds = new Set(
|
||||||
|
match.tournamentCodeIds.map((tournamentCodeId) =>
|
||||||
|
tournamentCodeId.toString(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
for (const reservedId of reservedIds) {
|
||||||
|
if (existingTournamentCodeIds.has(reservedId.toString())) continue;
|
||||||
|
match.tournamentCodeIds.push(reservedId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async markTournamentCodeUsedForManualResult(
|
private async markTournamentCodeUsedForManualResult(
|
||||||
@@ -1126,6 +1201,24 @@ export class TournamentResultService {
|
|||||||
if (!match) throw new NotFoundException('Match not found.');
|
if (!match) throw new NotFoundException('Match not found.');
|
||||||
return match;
|
return match;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private resolveMatchIdentifier(
|
||||||
|
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||||
|
): string {
|
||||||
|
return matchOrId instanceof mongoose.Types.ObjectId
|
||||||
|
? matchOrId.toString()
|
||||||
|
: matchOrId._id.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private tournamentResultPollProcessingLockKey(matchIdentifier: string): string {
|
||||||
|
return `distributed-lock:tournament-result:process-poll-job:match:${matchIdentifier}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private tournamentResultReadyCodeReservationLockKey(
|
||||||
|
matchIdentifier: string,
|
||||||
|
): string {
|
||||||
|
return `distributed-lock:tournament-result:reserve-ready-double-elimination-codes:match:${matchIdentifier}`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function isRiotMatchDto(value: unknown): value is RiotMatchDto {
|
function isRiotMatchDto(value: unknown): value is RiotMatchDto {
|
||||||
|
|||||||
Reference in new issue
Block a user