fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -4,10 +4,13 @@ import {
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { DakaoOutboxService } from './dakao-outbox.service';
|
||||
|
||||
const SWEEP_INTERVAL_MS = 60 * 1000;
|
||||
const STALE_OUTBOX_MS = 5 * 60 * 1000;
|
||||
const SWEEP_LOCK_KEY = 'distributed-lock:dakao-outbox:sweep-stale-publish-leases';
|
||||
const SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS = 55_000;
|
||||
|
||||
@Injectable()
|
||||
export class DakaoOutboxSweeperService
|
||||
@@ -16,7 +19,10 @@ export class DakaoOutboxSweeperService
|
||||
private readonly logger = new Logger(DakaoOutboxSweeperService.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
|
||||
constructor(private readonly outboxService: DakaoOutboxService) {}
|
||||
constructor(
|
||||
private readonly outboxService: DakaoOutboxService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
this.timer = setInterval(() => {
|
||||
@@ -31,12 +37,23 @@ export class DakaoOutboxSweeperService
|
||||
|
||||
private async sweep(): Promise<void> {
|
||||
try {
|
||||
const swept = await this.outboxService.sweepStale({
|
||||
staleMs: STALE_OUTBOX_MS,
|
||||
limit: 20,
|
||||
});
|
||||
if (swept > 0) {
|
||||
this.logger.warn(`Re-published ${swept} stale Dakao outbox messages.`);
|
||||
const swept = await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: SWEEP_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SWEEP_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
},
|
||||
async () =>
|
||||
this.outboxService.sweepStale({
|
||||
staleMs: STALE_OUTBOX_MS,
|
||||
limit: 20,
|
||||
}),
|
||||
);
|
||||
const sweptCount = swept ?? 0;
|
||||
if (sweptCount > 0) {
|
||||
this.logger.warn(
|
||||
`Re-published ${sweptCount} stale Dakao outbox messages.`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
|
||||
@@ -70,44 +70,55 @@ describe('DakaoOutboxService', () => {
|
||||
process.env.NODE_ENV = 'staging';
|
||||
|
||||
try {
|
||||
const exec = jest.fn().mockResolvedValue([
|
||||
{
|
||||
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||
type: 'chat.send-message',
|
||||
eventSource: 'lolympus',
|
||||
subject: 'staging.dakao.chat.send-message.v1',
|
||||
correlationId: 'chat-log-1',
|
||||
payload: {
|
||||
chatroomId: '18474390111480678',
|
||||
segments: [{ kind: 'text', text: 'done' }],
|
||||
},
|
||||
const staleRow = {
|
||||
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||
type: 'chat.send-message',
|
||||
eventSource: 'lolympus',
|
||||
subject: 'staging.dakao.chat.send-message.v1',
|
||||
correlationId: 'chat-log-1',
|
||||
payload: {
|
||||
chatroomId: '18474390111480678',
|
||||
segments: [{ kind: 'text', text: 'done' }],
|
||||
},
|
||||
]);
|
||||
};
|
||||
const exec = jest.fn().mockResolvedValue([staleRow]);
|
||||
const lean = jest.fn().mockReturnValue({ exec });
|
||||
const limit = jest.fn().mockReturnValue({ lean });
|
||||
const sort = jest.fn().mockReturnValue({ limit });
|
||||
const claimExec = jest.fn().mockResolvedValue(staleRow);
|
||||
const claimLean = jest.fn().mockReturnValue({ exec: claimExec });
|
||||
const outboxModel = {
|
||||
find: jest.fn().mockReturnValue({ sort }),
|
||||
findOneAndUpdate: jest.fn().mockReturnValue({ lean: claimLean }),
|
||||
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
|
||||
};
|
||||
const { service } = makeService(outboxModel);
|
||||
const enqueue = jest
|
||||
.spyOn(service, 'enqueueAndPublish')
|
||||
.mockResolvedValue(undefined);
|
||||
const { service, natsService } = makeService(outboxModel);
|
||||
|
||||
await expect(service.sweepStale({ staleMs: 1, limit: 1 })).resolves.toBe(
|
||||
1,
|
||||
);
|
||||
|
||||
expect(enqueue).toHaveBeenCalledWith(
|
||||
expect(outboxModel.findOneAndUpdate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
envelope: expect.objectContaining({
|
||||
resultSubjects: {
|
||||
succeeded:
|
||||
'staging.lolympus.dakao.chat.send-message.succeeded.v1',
|
||||
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
|
||||
},
|
||||
eventId: 'lolympus:chat.send-message:chat-log-1',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
$set: expect.objectContaining({
|
||||
publishLeaseOwnerIdentifier: expect.any(String),
|
||||
publishLeaseExpiresAt: expect.any(Date),
|
||||
}),
|
||||
}),
|
||||
{ returnDocument: 'after' },
|
||||
);
|
||||
expect(natsService.publishJetStream).toHaveBeenCalledWith(
|
||||
'staging.dakao.chat.send-message.v1',
|
||||
expect.objectContaining({
|
||||
resultSubjects: {
|
||||
succeeded: 'staging.lolympus.dakao.chat.send-message.succeeded.v1',
|
||||
failed: 'staging.lolympus.dakao.chat.send-message.failed.v1',
|
||||
},
|
||||
}),
|
||||
{ messageId: 'lolympus:chat.send-message:chat-log-1' },
|
||||
);
|
||||
} finally {
|
||||
process.env.NODE_ENV = previousNodeEnv;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { NatsService } from '../global/nats/nats.service';
|
||||
import { resolveDakaoNatsNames } from './dakao.nats-names';
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
|
||||
const CHAT_SEND_MESSAGE_TYPE = 'chat.send-message';
|
||||
const TERMINAL_STATES = ['sent', 'failed'] as const;
|
||||
const DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS = 60_000;
|
||||
|
||||
@Injectable()
|
||||
export class DakaoOutboxService {
|
||||
@@ -50,35 +52,59 @@ export class DakaoOutboxService {
|
||||
{ upsert: true, returnDocument: 'after', setDefaultsOnInsert: true },
|
||||
).exec();
|
||||
|
||||
await this.publishCommand({
|
||||
subject: params.subject,
|
||||
eventId: params.eventId,
|
||||
envelope: params.envelope,
|
||||
});
|
||||
}
|
||||
|
||||
private async publishCommand<TPayload>(params: {
|
||||
subject: string;
|
||||
eventId: string;
|
||||
envelope: DakaoCommandEnvelope<TPayload>;
|
||||
publishLeaseOwnerIdentifier?: string;
|
||||
}): Promise<void> {
|
||||
const OutboxModel =
|
||||
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||
const updateFilter = {
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
...(params.publishLeaseOwnerIdentifier
|
||||
? { publishLeaseOwnerIdentifier: params.publishLeaseOwnerIdentifier }
|
||||
: {}),
|
||||
};
|
||||
try {
|
||||
await this.natsService.publishJetStream(params.subject, params.envelope, {
|
||||
messageId: params.eventId,
|
||||
});
|
||||
await OutboxModel.updateOne(
|
||||
{
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
},
|
||||
updateFilter,
|
||||
{
|
||||
$set: {
|
||||
state: 'published',
|
||||
publishedAt: new Date(),
|
||||
},
|
||||
$inc: { publishAttempts: 1 },
|
||||
$unset: { lastPublishError: '' },
|
||||
$unset: {
|
||||
lastPublishError: '',
|
||||
publishLeaseOwnerIdentifier: '',
|
||||
publishLeaseExpiresAt: '',
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
} catch (err) {
|
||||
await OutboxModel.updateOne(
|
||||
{
|
||||
eventId: params.eventId,
|
||||
state: { $nin: TERMINAL_STATES },
|
||||
},
|
||||
updateFilter,
|
||||
{
|
||||
$inc: { publishAttempts: 1 },
|
||||
$set: {
|
||||
lastPublishError: (err as Error)?.message ?? String(err),
|
||||
},
|
||||
$unset: {
|
||||
publishLeaseOwnerIdentifier: '',
|
||||
publishLeaseExpiresAt: '',
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
this.logger.error(
|
||||
@@ -146,40 +172,77 @@ export class DakaoOutboxService {
|
||||
|
||||
public async sweepStale(params?: {
|
||||
staleMs?: number;
|
||||
publishLeaseDurationMilliseconds?: number;
|
||||
limit?: number;
|
||||
}): Promise<number> {
|
||||
const OutboxModel =
|
||||
await this.connectionService.getModel('dakaoOutboxMessage');
|
||||
const currentTimestamp = new Date();
|
||||
const staleBefore = new Date(
|
||||
Date.now() - (params?.staleMs ?? 5 * 60 * 1000),
|
||||
currentTimestamp.getTime() - (params?.staleMs ?? 5 * 60 * 1000),
|
||||
);
|
||||
const publishLeaseOwnerIdentifier = `dakao-outbox-sweeper:${process.pid}:${randomUUID()}`;
|
||||
const publishLeaseExpiresAt = new Date(
|
||||
currentTimestamp.getTime() +
|
||||
(params?.publishLeaseDurationMilliseconds ??
|
||||
DEFAULT_PUBLISH_LEASE_DURATION_MILLISECONDS),
|
||||
);
|
||||
const rows = await OutboxModel.find({
|
||||
state: { $in: ['pending', 'published'] },
|
||||
updatedAt: { $lte: staleBefore },
|
||||
$or: [
|
||||
{ publishLeaseExpiresAt: { $exists: false } },
|
||||
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||
],
|
||||
})
|
||||
.sort({ updatedAt: 1 })
|
||||
.limit(params?.limit ?? 20)
|
||||
.lean()
|
||||
.exec();
|
||||
|
||||
let swept = 0;
|
||||
for (const row of rows) {
|
||||
await this.enqueueAndPublish({
|
||||
subject: row.subject,
|
||||
eventId: row.eventId,
|
||||
const claimedRow = await OutboxModel.findOneAndUpdate(
|
||||
{
|
||||
eventId: row.eventId,
|
||||
state: { $in: ['pending', 'published'] },
|
||||
$or: [
|
||||
{ publishLeaseExpiresAt: { $exists: false } },
|
||||
{ publishLeaseExpiresAt: { $lte: currentTimestamp } },
|
||||
],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
publishLeaseOwnerIdentifier,
|
||||
publishLeaseExpiresAt,
|
||||
},
|
||||
},
|
||||
{ returnDocument: 'after' },
|
||||
)
|
||||
.lean()
|
||||
.exec();
|
||||
if (!claimedRow) continue;
|
||||
|
||||
await this.publishCommand({
|
||||
subject: claimedRow.subject,
|
||||
eventId: claimedRow.eventId,
|
||||
publishLeaseOwnerIdentifier,
|
||||
envelope: {
|
||||
schema: 'dakao.command.v1',
|
||||
eventId: row.eventId,
|
||||
type: row.type,
|
||||
eventSource: row.eventSource,
|
||||
correlationId: row.correlationId,
|
||||
eventId: claimedRow.eventId,
|
||||
type: claimedRow.type,
|
||||
eventSource: claimedRow.eventSource,
|
||||
correlationId: claimedRow.correlationId,
|
||||
resultSubjects:
|
||||
row.resultSubjects ?? this.resolveDefaultResultSubjects(row.type),
|
||||
payload: row.payload,
|
||||
claimedRow.resultSubjects ??
|
||||
this.resolveDefaultResultSubjects(claimedRow.type),
|
||||
payload: claimedRow.payload,
|
||||
},
|
||||
});
|
||||
swept += 1;
|
||||
}
|
||||
|
||||
return rows.length;
|
||||
return swept;
|
||||
}
|
||||
|
||||
private resolveDefaultResultSubjects(
|
||||
|
||||
@@ -42,4 +42,21 @@ describe('resolveMongoConnectionOptions', () => {
|
||||
autoCreate: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('enables automatic index creation when explicitly requested', () => {
|
||||
expect(
|
||||
resolveMongoConnectionOptions(
|
||||
'lolympus-staging',
|
||||
'lolympus-api:staging:1',
|
||||
'true',
|
||||
),
|
||||
).toEqual({
|
||||
dbName: 'lolympus-staging',
|
||||
appName: 'lolympus-api:staging:1',
|
||||
minPoolSize: 10,
|
||||
maxPoolSize: 20,
|
||||
autoIndex: true,
|
||||
autoCreate: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -19,13 +19,17 @@ export function resolveMongoAppName(
|
||||
export function resolveMongoConnectionOptions(
|
||||
dbName: string,
|
||||
appName: string,
|
||||
mongoAutoIndex = process.env.MONGODB_AUTO_INDEX,
|
||||
): mongoose.ConnectOptions {
|
||||
const shouldEnableMongoAutoIndex =
|
||||
mongoAutoIndex?.trim().toLowerCase() === 'true';
|
||||
|
||||
return {
|
||||
dbName,
|
||||
appName,
|
||||
minPoolSize: 10,
|
||||
maxPoolSize: 20,
|
||||
autoIndex: false,
|
||||
autoIndex: shouldEnableMongoAutoIndex,
|
||||
autoCreate: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -392,6 +392,8 @@ export interface DakaoOutboxMessage extends MongooseTimestamps {
|
||||
payload: Record<string, unknown>;
|
||||
state: DakaoOutboxMessageState;
|
||||
publishAttempts: number;
|
||||
publishLeaseOwnerIdentifier?: string;
|
||||
publishLeaseExpiresAt?: Date;
|
||||
publishedAt?: Date;
|
||||
terminalAt?: Date;
|
||||
resultEventId?: string;
|
||||
@@ -733,6 +735,7 @@ export interface LegacyImportJob extends MongooseTimestamps {
|
||||
totalSkippedDuplicate: number;
|
||||
totalErrors: number;
|
||||
averageMemberSeconds?: number;
|
||||
completedMemberJobIds: string[];
|
||||
errors?: LegacyImportJobError[];
|
||||
}
|
||||
|
||||
@@ -1282,6 +1285,8 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
|
||||
default: 'pending',
|
||||
},
|
||||
publishAttempts: { type: Number, required: true, default: 0 },
|
||||
publishLeaseOwnerIdentifier: { type: String },
|
||||
publishLeaseExpiresAt: { type: Date },
|
||||
publishedAt: { type: Date },
|
||||
terminalAt: { type: Date },
|
||||
resultEventId: { type: String },
|
||||
@@ -1302,6 +1307,11 @@ export const DakaoOutboxMessageSchema = new Schema<DakaoOutboxMessage>(
|
||||
);
|
||||
|
||||
DakaoOutboxMessageSchema.index({ state: 1, updatedAt: 1 });
|
||||
DakaoOutboxMessageSchema.index({
|
||||
state: 1,
|
||||
publishLeaseExpiresAt: 1,
|
||||
updatedAt: 1,
|
||||
});
|
||||
DakaoOutboxMessageSchema.index({ correlationId: 1 }, { sparse: true });
|
||||
|
||||
export const DakaoSendResultEventSchema = new Schema<DakaoSendResultEvent>(
|
||||
@@ -1988,6 +1998,11 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
|
||||
totalSkippedDuplicate: { type: Number, required: true, default: 0 },
|
||||
totalErrors: { type: Number, required: true, default: 0 },
|
||||
averageMemberSeconds: { type: Number },
|
||||
completedMemberJobIds: {
|
||||
type: [String],
|
||||
required: true,
|
||||
default: [],
|
||||
},
|
||||
errors: {
|
||||
type: [LegacyImportJobErrorSubSchema],
|
||||
required: true,
|
||||
@@ -2003,6 +2018,7 @@ export const LegacyImportJobSchema = new Schema<LegacyImportJob>(
|
||||
|
||||
LegacyImportJobSchema.index({ communityId: 1, startedAt: -1 });
|
||||
LegacyImportJobSchema.index({ communityId: 1, state: 1 });
|
||||
LegacyImportJobSchema.index({ jobId: 1, completedMemberJobIds: 1 });
|
||||
|
||||
const AuctionPlayerSnapshotSubSchema = new Schema<AuctionPlayerSnapshot>(
|
||||
{
|
||||
|
||||
@@ -56,20 +56,25 @@ export interface JetStreamConsumeOptions {
|
||||
maxMessages?: number;
|
||||
expiresMs?: number;
|
||||
nakDelayMs?: number;
|
||||
restartDelayMilliseconds?: number;
|
||||
}
|
||||
|
||||
const JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS = 1_000;
|
||||
|
||||
@Injectable()
|
||||
export class NatsService implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(NatsService.name);
|
||||
private readonly stringCodec = StringCodec();
|
||||
private readonly subscriptions = new Set<Subscription>();
|
||||
private readonly jetStreamConsumers = new Set<ConsumerMessages>();
|
||||
private isDestroying = false;
|
||||
|
||||
constructor(
|
||||
@Inject(NATS_CLIENT) private readonly natsClient: NatsConnection,
|
||||
) {}
|
||||
|
||||
async onModuleDestroy() {
|
||||
this.isDestroying = true;
|
||||
for (const consumer of this.jetStreamConsumers) {
|
||||
try {
|
||||
await consumer.close();
|
||||
@@ -199,28 +204,36 @@ export class NatsService implements OnModuleDestroy {
|
||||
options: JetStreamConsumeOptions,
|
||||
handler: JetStreamMessageHandler<T>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const consumer = await this.natsClient
|
||||
.jetstream()
|
||||
.consumers.get(options.stream, options.durable);
|
||||
const messages = await consumer.consume({
|
||||
max_messages: options.maxMessages ?? 10,
|
||||
expires: options.expiresMs ?? 30000,
|
||||
});
|
||||
this.jetStreamConsumers.add(messages);
|
||||
|
||||
while (!this.isDestroying) {
|
||||
try {
|
||||
for await (const msg of messages) {
|
||||
await this.dispatchJetStream(msg, handler, options);
|
||||
const consumer = await this.natsClient
|
||||
.jetstream()
|
||||
.consumers.get(options.stream, options.durable);
|
||||
const messages = await consumer.consume({
|
||||
max_messages: options.maxMessages ?? 10,
|
||||
expires: options.expiresMs ?? 30000,
|
||||
});
|
||||
this.jetStreamConsumers.add(messages);
|
||||
|
||||
try {
|
||||
for await (const msg of messages) {
|
||||
await this.dispatchJetStream(msg, handler, options);
|
||||
}
|
||||
} finally {
|
||||
this.jetStreamConsumers.delete(messages);
|
||||
}
|
||||
} finally {
|
||||
this.jetStreamConsumers.delete(messages);
|
||||
} catch (err) {
|
||||
if (this.isDestroying) return;
|
||||
this.logger.error(
|
||||
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
|
||||
(err as Error)?.stack ?? err
|
||||
}`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`JetStream consumer stream="${options.stream}" durable="${options.durable}" terminated: ${
|
||||
(err as Error)?.stack ?? err
|
||||
}`,
|
||||
|
||||
await this.sleep(
|
||||
options.restartDelayMilliseconds ??
|
||||
JET_STREAM_CONSUMER_RESTART_DELAY_MILLISECONDS,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -313,4 +326,8 @@ export class NatsService implements OnModuleDestroy {
|
||||
? RetentionPolicy.Workqueue
|
||||
: RetentionPolicy.Limits;
|
||||
}
|
||||
|
||||
private sleep(durationMilliseconds: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { DistributedLockService } from './distributed-lock.service';
|
||||
import { RedisService } from './redis.service';
|
||||
|
||||
function createService(sendCommand: jest.Mock) {
|
||||
const redisService = {
|
||||
raw: jest.fn().mockReturnValue({ sendCommand }),
|
||||
} as unknown as RedisService;
|
||||
return new DistributedLockService(redisService);
|
||||
}
|
||||
|
||||
describe('DistributedLockService', () => {
|
||||
it('acquires a lock with a token and millisecond expiration', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue('OK');
|
||||
const service = createService(sendCommand);
|
||||
|
||||
const lockOwnership = await service.acquireLock({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
});
|
||||
|
||||
expect(lockOwnership).toEqual({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
});
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'SET',
|
||||
'distributed-lock:test',
|
||||
'owner-token',
|
||||
'NX',
|
||||
'PX',
|
||||
'30000',
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns null when another owner already holds the lock', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue(null);
|
||||
const service = createService(sendCommand);
|
||||
|
||||
await expect(
|
||||
service.acquireLock({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
}),
|
||||
).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('releases a lock only when the ownership token matches', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue(1);
|
||||
const service = createService(sendCommand);
|
||||
|
||||
await expect(
|
||||
service.releaseLockIfOwned({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'),
|
||||
'1',
|
||||
'distributed-lock:test',
|
||||
'owner-token',
|
||||
]);
|
||||
});
|
||||
|
||||
it('extends a lock only when the ownership token matches', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue(1);
|
||||
const service = createService(sendCommand);
|
||||
|
||||
await expect(
|
||||
service.extendLockIfOwned(
|
||||
{
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockOwnershipToken: 'owner-token',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
},
|
||||
45_000,
|
||||
),
|
||||
).resolves.toBe(true);
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'EVAL',
|
||||
expect.stringContaining('PEXPIRE'),
|
||||
'1',
|
||||
'distributed-lock:test',
|
||||
'owner-token',
|
||||
'45000',
|
||||
]);
|
||||
});
|
||||
|
||||
it('runs the operation and releases the lock afterward', async () => {
|
||||
const sendCommand = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce(1);
|
||||
const service = createService(sendCommand);
|
||||
const operation = jest.fn().mockResolvedValue('done');
|
||||
|
||||
await expect(
|
||||
service.runWithLock(
|
||||
{
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
},
|
||||
operation,
|
||||
),
|
||||
).resolves.toBe('done');
|
||||
|
||||
expect(operation).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
lockKey: 'distributed-lock:test',
|
||||
lockExpirationDurationMilliseconds: 30_000,
|
||||
}),
|
||||
);
|
||||
expect(sendCommand).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,199 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { RedisService } from './redis.service';
|
||||
|
||||
const RELEASE_LOCK_IF_OWNED_SCRIPT = `
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
const EXTEND_LOCK_IF_OWNED_SCRIPT = `
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
export interface DistributedLockOwnership {
|
||||
lockKey: string;
|
||||
lockOwnershipToken: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
}
|
||||
|
||||
export interface AcquireDistributedLockOptions {
|
||||
lockKey: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
lockOwnershipToken?: string;
|
||||
}
|
||||
|
||||
export interface RunWithDistributedLockOptions {
|
||||
lockKey: string;
|
||||
lockExpirationDurationMilliseconds: number;
|
||||
lockAcquireTimeoutMilliseconds?: number;
|
||||
lockRetryDelayMilliseconds?: number;
|
||||
}
|
||||
|
||||
export interface RunWithRenewedDistributedLockOptions
|
||||
extends RunWithDistributedLockOptions {
|
||||
lockRenewalIntervalMilliseconds: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DistributedLockService {
|
||||
private readonly logger = new Logger(DistributedLockService.name);
|
||||
|
||||
constructor(private readonly redisService: RedisService) {}
|
||||
|
||||
public async acquireLock(
|
||||
options: AcquireDistributedLockOptions,
|
||||
): Promise<DistributedLockOwnership | null> {
|
||||
this.assertPositiveDuration(
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
'lockExpirationDurationMilliseconds',
|
||||
);
|
||||
|
||||
const lockOwnershipToken = options.lockOwnershipToken ?? randomUUID();
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'SET',
|
||||
options.lockKey,
|
||||
lockOwnershipToken,
|
||||
'NX',
|
||||
'PX',
|
||||
String(options.lockExpirationDurationMilliseconds),
|
||||
]);
|
||||
|
||||
if (String(result) !== 'OK') return null;
|
||||
return {
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken,
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
};
|
||||
}
|
||||
|
||||
public async releaseLockIfOwned(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
): Promise<boolean> {
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'EVAL',
|
||||
RELEASE_LOCK_IF_OWNED_SCRIPT,
|
||||
'1',
|
||||
lockOwnership.lockKey,
|
||||
lockOwnership.lockOwnershipToken,
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
|
||||
public async extendLockIfOwned(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
lockExpirationDurationMilliseconds =
|
||||
lockOwnership.lockExpirationDurationMilliseconds,
|
||||
): Promise<boolean> {
|
||||
this.assertPositiveDuration(
|
||||
lockExpirationDurationMilliseconds,
|
||||
'lockExpirationDurationMilliseconds',
|
||||
);
|
||||
|
||||
const result = await this.redisService.raw().sendCommand([
|
||||
'EVAL',
|
||||
EXTEND_LOCK_IF_OWNED_SCRIPT,
|
||||
'1',
|
||||
lockOwnership.lockKey,
|
||||
lockOwnership.lockOwnershipToken,
|
||||
String(lockExpirationDurationMilliseconds),
|
||||
]);
|
||||
return Number(result) === 1;
|
||||
}
|
||||
|
||||
public async runWithLock<TResult>(
|
||||
options: RunWithDistributedLockOptions,
|
||||
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||
): Promise<TResult | null> {
|
||||
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||
if (!lockOwnership) return null;
|
||||
|
||||
try {
|
||||
return await operation(lockOwnership);
|
||||
} finally {
|
||||
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public async runWithRenewedLock<TResult>(
|
||||
options: RunWithRenewedDistributedLockOptions,
|
||||
operation: (lockOwnership: DistributedLockOwnership) => Promise<TResult>,
|
||||
): Promise<TResult | null> {
|
||||
this.assertPositiveDuration(
|
||||
options.lockRenewalIntervalMilliseconds,
|
||||
'lockRenewalIntervalMilliseconds',
|
||||
);
|
||||
|
||||
const lockOwnership = await this.acquireLockWithRetry(options);
|
||||
if (!lockOwnership) return null;
|
||||
|
||||
let lockRenewalTimer: NodeJS.Timeout | undefined;
|
||||
lockRenewalTimer = setInterval(() => {
|
||||
void this.extendLockIfOwned(lockOwnership).then((wasExtended) => {
|
||||
if (wasExtended) return;
|
||||
this.logger.warn(
|
||||
`Lost distributed lock ownership before operation finished: ${lockOwnership.lockKey}`,
|
||||
);
|
||||
});
|
||||
}, options.lockRenewalIntervalMilliseconds);
|
||||
lockRenewalTimer.unref?.();
|
||||
|
||||
try {
|
||||
return await operation(lockOwnership);
|
||||
} finally {
|
||||
if (lockRenewalTimer) clearInterval(lockRenewalTimer);
|
||||
await this.releaseLockIfOwned(lockOwnership).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release distributed lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async acquireLockWithRetry(
|
||||
options: RunWithDistributedLockOptions,
|
||||
): Promise<DistributedLockOwnership | null> {
|
||||
const lockAcquireTimeoutMilliseconds =
|
||||
options.lockAcquireTimeoutMilliseconds ?? 0;
|
||||
const lockRetryDelayMilliseconds =
|
||||
options.lockRetryDelayMilliseconds ?? 100;
|
||||
const acquisitionStartedAtTimestampMilliseconds = Date.now();
|
||||
|
||||
while (true) {
|
||||
const lockOwnership = await this.acquireLock(options);
|
||||
if (lockOwnership) return lockOwnership;
|
||||
|
||||
if (lockAcquireTimeoutMilliseconds <= 0) return null;
|
||||
const elapsedMilliseconds =
|
||||
Date.now() - acquisitionStartedAtTimestampMilliseconds;
|
||||
if (elapsedMilliseconds >= lockAcquireTimeoutMilliseconds) return null;
|
||||
|
||||
await this.sleep(
|
||||
Math.min(lockRetryDelayMilliseconds, lockAcquireTimeoutMilliseconds),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private assertPositiveDuration(value: number, fieldName: string): void {
|
||||
if (Number.isFinite(value) && value > 0) return;
|
||||
throw new Error(`${fieldName} must be a positive finite number.`);
|
||||
}
|
||||
|
||||
private sleep(durationMilliseconds: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, durationMilliseconds));
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { SecretModule } from '../secret/secret.module';
|
||||
import { SecretService } from '../secret/secret.service';
|
||||
import { REDIS_CLIENT } from './redis.constants';
|
||||
import { resolveRedisClientOptions } from './redis.config';
|
||||
import { DistributedLockService } from './distributed-lock.service';
|
||||
import { RedisService, type RedisClient } from './redis.service';
|
||||
|
||||
async function readFirstSecret(
|
||||
@@ -62,7 +63,8 @@ async function readFirstSecret(
|
||||
},
|
||||
},
|
||||
RedisService,
|
||||
DistributedLockService,
|
||||
],
|
||||
exports: [RedisService, REDIS_CLIENT],
|
||||
exports: [RedisService, DistributedLockService, REDIS_CLIENT],
|
||||
})
|
||||
export class RedisModule {}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { RedisService } from './redis.service';
|
||||
|
||||
function createRedisService(client: Record<string, unknown>) {
|
||||
return new RedisService(client as never);
|
||||
}
|
||||
|
||||
describe('RedisService stream pending recovery', () => {
|
||||
it('normalizes XAUTOCLAIM array replies into stream entries', async () => {
|
||||
const sendCommand = jest.fn().mockResolvedValue([
|
||||
'0-0',
|
||||
[
|
||||
[
|
||||
'1700000000000-0',
|
||||
['job', '{"kind":"poll-tournament-result"}', 'source', 'test'],
|
||||
],
|
||||
],
|
||||
]);
|
||||
const service = createRedisService({ sendCommand });
|
||||
|
||||
await expect(
|
||||
service.claimStalePendingStreamMessages({
|
||||
key: 'riot:queue:high',
|
||||
group: 'riot-refresh-worker',
|
||||
consumer: 'worker-1',
|
||||
stalePendingMessageMinimumIdleMilliseconds: 1_800_000,
|
||||
count: 1,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
nextStartId: '0-0',
|
||||
messages: [
|
||||
{
|
||||
id: '1700000000000-0',
|
||||
message: {
|
||||
job: '{"kind":"poll-tournament-result"}',
|
||||
source: 'test',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(sendCommand).toHaveBeenCalledWith([
|
||||
'XAUTOCLAIM',
|
||||
'riot:queue:high',
|
||||
'riot-refresh-worker',
|
||||
'worker-1',
|
||||
'1800000',
|
||||
'0-0',
|
||||
'COUNT',
|
||||
'1',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,11 @@ export interface StreamReadResult {
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface ClaimStalePendingStreamMessagesResult {
|
||||
messages: StreamEntry[];
|
||||
nextStartId: string;
|
||||
}
|
||||
|
||||
interface RawStreamMessage {
|
||||
deliveriesCounter?: number;
|
||||
id: string;
|
||||
@@ -29,6 +34,16 @@ interface RawStreamReadResult {
|
||||
}
|
||||
|
||||
function normalizeStreamFields(fields: unknown): Record<string, string> {
|
||||
if (Array.isArray(fields)) {
|
||||
const normalized: Record<string, string> = {};
|
||||
for (let index = 0; index < fields.length; index += 2) {
|
||||
const key = fields[index];
|
||||
if (key === undefined) continue;
|
||||
normalized[String(key)] = String(fields[index + 1] ?? '');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (fields instanceof Map) {
|
||||
return Object.fromEntries(
|
||||
[...fields.entries()].map(([key, value]) => [String(key), String(value)]),
|
||||
@@ -188,6 +203,38 @@ export class RedisService implements OnModuleDestroy {
|
||||
}));
|
||||
}
|
||||
|
||||
public async claimStalePendingStreamMessages(options: {
|
||||
key: string;
|
||||
group: string;
|
||||
consumer: string;
|
||||
stalePendingMessageMinimumIdleMilliseconds: number;
|
||||
startId?: string;
|
||||
count?: number;
|
||||
}): Promise<ClaimStalePendingStreamMessagesResult> {
|
||||
const response = (await this.client.sendCommand([
|
||||
'XAUTOCLAIM',
|
||||
options.key,
|
||||
options.group,
|
||||
options.consumer,
|
||||
String(options.stalePendingMessageMinimumIdleMilliseconds),
|
||||
options.startId ?? '0-0',
|
||||
'COUNT',
|
||||
String(options.count ?? 1),
|
||||
])) as unknown;
|
||||
|
||||
const responseItems = Array.isArray(response) ? response : [];
|
||||
const rawMessages = Array.isArray(responseItems[1])
|
||||
? responseItems[1]
|
||||
: [];
|
||||
|
||||
return {
|
||||
nextStartId: String(responseItems[0] ?? '0-0'),
|
||||
messages: rawMessages
|
||||
.map((rawMessage) => this.normalizeClaimedStreamMessage(rawMessage))
|
||||
.filter((message): message is StreamEntry => Boolean(message)),
|
||||
};
|
||||
}
|
||||
|
||||
public async acknowledgeStream(
|
||||
key: string,
|
||||
group: string,
|
||||
@@ -210,4 +257,37 @@ export class RedisService implements OnModuleDestroy {
|
||||
public async publish(channel: string, message: string): Promise<number> {
|
||||
return this.client.publish(channel, message);
|
||||
}
|
||||
|
||||
private normalizeClaimedStreamMessage(rawMessage: unknown): StreamEntry | null {
|
||||
if (Array.isArray(rawMessage)) {
|
||||
const [id, fields] = rawMessage;
|
||||
if (id === undefined) return null;
|
||||
return {
|
||||
id: String(id),
|
||||
message: normalizeStreamFields(fields),
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
typeof rawMessage === 'object' &&
|
||||
rawMessage !== null &&
|
||||
'id' in rawMessage
|
||||
) {
|
||||
const message = rawMessage as RawStreamMessage;
|
||||
return {
|
||||
deliveriesCounter:
|
||||
typeof message.deliveriesCounter === 'number'
|
||||
? message.deliveriesCounter
|
||||
: undefined,
|
||||
id: String(message.id),
|
||||
message: normalizeStreamFields(message.message),
|
||||
millisElapsedFromDelivery:
|
||||
typeof message.millisElapsedFromDelivery === 'number'
|
||||
? message.millisElapsedFromDelivery
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { RiotMatchService } from '../riot/riot-match.service';
|
||||
import { LegacyImportService } from './legacy-import.service';
|
||||
import type { LegacyImportMemberJob } from './legacy-import.types';
|
||||
|
||||
function execResult<T>(value: T) {
|
||||
return { exec: jest.fn().mockResolvedValue(value) };
|
||||
}
|
||||
|
||||
describe('LegacyImportService member job completion', () => {
|
||||
it('increments aggregate counters only once for the same member job id', async () => {
|
||||
const legacyImportJobModel = {
|
||||
findOne: jest
|
||||
.fn()
|
||||
.mockReturnValueOnce(
|
||||
execResult({
|
||||
jobId: 'legacy-job-1',
|
||||
state: 'running',
|
||||
processedMembers: 0,
|
||||
totalMembers: 1,
|
||||
averageMemberSeconds: 60,
|
||||
completedMemberJobIds: [],
|
||||
}),
|
||||
)
|
||||
.mockReturnValueOnce({
|
||||
select: jest.fn().mockReturnValue({
|
||||
lean: jest.fn().mockReturnValue(
|
||||
execResult({
|
||||
state: 'running',
|
||||
processedMembers: 1,
|
||||
totalMembers: 1,
|
||||
}),
|
||||
),
|
||||
}),
|
||||
})
|
||||
.mockReturnValueOnce(
|
||||
execResult({
|
||||
jobId: 'legacy-job-1',
|
||||
state: 'completed',
|
||||
processedMembers: 1,
|
||||
totalMembers: 1,
|
||||
averageMemberSeconds: 60,
|
||||
completedMemberJobIds: ['member-job-1'],
|
||||
}),
|
||||
),
|
||||
updateOne: jest.fn().mockReturnValue(execResult({ modifiedCount: 1 })),
|
||||
};
|
||||
const connectionService = {
|
||||
getModel: jest.fn().mockResolvedValue(legacyImportJobModel),
|
||||
} as unknown as jest.Mocked<ConnectionService>;
|
||||
const distributedLockService = {
|
||||
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const communityStatsService = {
|
||||
invalidateCommunity: jest.fn().mockResolvedValue(undefined),
|
||||
} as unknown as jest.Mocked<CommunityStatsService>;
|
||||
const service = new LegacyImportService(
|
||||
connectionService,
|
||||
{} as RedisService,
|
||||
{} as RiotMatchService,
|
||||
communityStatsService,
|
||||
{} as MatchStatsService,
|
||||
distributedLockService,
|
||||
);
|
||||
const job: LegacyImportMemberJob = {
|
||||
kind: 'legacy-import-member',
|
||||
jobId: 'member-job-1',
|
||||
parentJobId: 'legacy-job-1',
|
||||
communityId: new mongoose.Types.ObjectId().toString(),
|
||||
userId: new mongoose.Types.ObjectId().toString(),
|
||||
puuid: 'puuid-1',
|
||||
startTime: 1,
|
||||
endTime: 2,
|
||||
triggeredBy: new mongoose.Types.ObjectId().toString(),
|
||||
region: 'asia',
|
||||
};
|
||||
|
||||
await (
|
||||
service as unknown as {
|
||||
finishMemberJob: (
|
||||
job: LegacyImportMemberJob,
|
||||
stats: {
|
||||
discovered: number;
|
||||
imported: number;
|
||||
duplicates: number;
|
||||
errors: number;
|
||||
elapsedSeconds: number;
|
||||
},
|
||||
) => Promise<void>;
|
||||
}
|
||||
).finishMemberJob(job, {
|
||||
discovered: 3,
|
||||
imported: 2,
|
||||
duplicates: 1,
|
||||
errors: 0,
|
||||
elapsedSeconds: 30,
|
||||
});
|
||||
await (
|
||||
service as unknown as {
|
||||
finishMemberJob: (
|
||||
job: LegacyImportMemberJob,
|
||||
stats: {
|
||||
discovered: number;
|
||||
imported: number;
|
||||
duplicates: number;
|
||||
errors: number;
|
||||
elapsedSeconds: number;
|
||||
},
|
||||
) => Promise<void>;
|
||||
}
|
||||
).finishMemberJob(job, {
|
||||
discovered: 3,
|
||||
imported: 2,
|
||||
duplicates: 1,
|
||||
errors: 0,
|
||||
elapsedSeconds: 30,
|
||||
});
|
||||
|
||||
expect(legacyImportJobModel.updateOne).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
{
|
||||
jobId: 'legacy-job-1',
|
||||
completedMemberJobIds: { $ne: 'member-job-1' },
|
||||
state: { $ne: 'cancelled' },
|
||||
},
|
||||
expect.objectContaining({
|
||||
$addToSet: { completedMemberJobIds: 'member-job-1' },
|
||||
$inc: expect.objectContaining({
|
||||
processedMembers: 1,
|
||||
totalDiscoveredMatchIds: 3,
|
||||
totalImportedMatches: 2,
|
||||
totalSkippedDuplicate: 1,
|
||||
totalErrors: 0,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(legacyImportJobModel.updateOne).toHaveBeenCalledTimes(2);
|
||||
expect(distributedLockService.releaseLockIfOwned).toHaveBeenCalledTimes(1);
|
||||
expect(communityStatsService.invalidateCommunity).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,10 @@ import mongoose from 'mongoose';
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import {
|
||||
type DistributedLockOwnership,
|
||||
DistributedLockService,
|
||||
} from '../global/redis/distributed-lock.service';
|
||||
import type {
|
||||
LegacyImportJobDocument,
|
||||
Match,
|
||||
@@ -57,6 +61,7 @@ export class LegacyImportService {
|
||||
private readonly riotMatchService: RiotMatchService,
|
||||
private readonly communityStatsService: CommunityStatsService,
|
||||
private readonly matchStatsService: MatchStatsService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public async previewMembers(params: {
|
||||
@@ -80,12 +85,14 @@ export class LegacyImportService {
|
||||
}): Promise<StartLegacyImportResponse> {
|
||||
const lockKey = this.communityLockKey(params.communityId);
|
||||
const jobId = randomUUID();
|
||||
const acquired = await this.redisService.setStringIfAbsent(
|
||||
lockKey,
|
||||
jobId,
|
||||
LEGACY_IMPORT_LOCK_TTL_SECONDS,
|
||||
);
|
||||
if (!acquired) {
|
||||
const communityImportLockOwnership =
|
||||
await this.distributedLockService.acquireLock({
|
||||
lockKey,
|
||||
lockOwnershipToken: jobId,
|
||||
lockExpirationDurationMilliseconds:
|
||||
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
});
|
||||
if (!communityImportLockOwnership) {
|
||||
throw new ConflictException('Legacy import is already running.');
|
||||
}
|
||||
|
||||
@@ -133,6 +140,7 @@ export class LegacyImportService {
|
||||
totalSkippedDuplicate: 0,
|
||||
totalErrors: 0,
|
||||
averageMemberSeconds: LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
||||
completedMemberJobIds: [],
|
||||
});
|
||||
|
||||
for (const window of windows) {
|
||||
@@ -160,7 +168,7 @@ export class LegacyImportService {
|
||||
{ jobId },
|
||||
{ $set: { state: 'completed', completedAt: new Date() } },
|
||||
).exec();
|
||||
await this.redisService.delete(lockKey);
|
||||
await this.releaseDistributedLockSafely(communityImportLockOwnership);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -169,7 +177,7 @@ export class LegacyImportService {
|
||||
etaSeconds: windows.length * LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
||||
};
|
||||
} catch (error) {
|
||||
await this.redisService.delete(lockKey);
|
||||
await this.releaseDistributedLockSafely(communityImportLockOwnership);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -220,7 +228,11 @@ export class LegacyImportService {
|
||||
).exec();
|
||||
if (!job)
|
||||
throw new NotFoundException('Running legacy import job not found.');
|
||||
await this.redisService.delete(this.communityLockKey(params.communityId));
|
||||
await this.distributedLockService.releaseLockIfOwned({
|
||||
lockKey: this.communityLockKey(params.communityId),
|
||||
lockOwnershipToken: job.jobId,
|
||||
lockExpirationDurationMilliseconds: LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
});
|
||||
return job;
|
||||
}
|
||||
|
||||
@@ -228,6 +240,7 @@ export class LegacyImportService {
|
||||
const started = Date.now();
|
||||
const legacyJob = await this.getRunnableJob(job);
|
||||
if (!legacyJob) return;
|
||||
if (legacyJob.completedMemberJobIds?.includes(job.jobId)) return;
|
||||
|
||||
let discovered = 0;
|
||||
let imported = 0;
|
||||
@@ -500,75 +513,81 @@ export class LegacyImportService {
|
||||
userByPuuid: UserByPuuidMap,
|
||||
): Promise<ImportOutcome> {
|
||||
const lockKey = `legacy-import:match:${riotMatchId}`;
|
||||
const acquired = await this.redisService.setStringIfAbsent(
|
||||
lockKey,
|
||||
job.parentJobId,
|
||||
MATCH_IMPORT_LOCK_TTL_SECONDS,
|
||||
);
|
||||
if (!acquired) return 'duplicate';
|
||||
|
||||
try {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const existing = await MatchModel.exists({
|
||||
'result.games.riotMatchId': riotMatchId,
|
||||
});
|
||||
if (existing) return 'duplicate';
|
||||
const importOutcome = await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey,
|
||||
lockExpirationDurationMilliseconds:
|
||||
MATCH_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
},
|
||||
async () => {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const existing = await MatchModel.exists({
|
||||
'result.games.riotMatchId': riotMatchId,
|
||||
});
|
||||
if (existing) return 'duplicate';
|
||||
|
||||
const detail = await this.ensureRiotMatchPersisted(
|
||||
riotMatchId,
|
||||
job.region,
|
||||
);
|
||||
if (!detail.info.tournamentCode) return 'skipped';
|
||||
const counts = countKnownParticipants(detail, userByPuuid);
|
||||
if (counts.total === 0) return 'skipped';
|
||||
const detail = await this.ensureRiotMatchPersisted(
|
||||
riotMatchId,
|
||||
job.region,
|
||||
);
|
||||
if (!detail.info.tournamentCode) return 'skipped';
|
||||
const counts = countKnownParticipants(detail, userByPuuid);
|
||||
if (counts.total === 0) return 'skipped';
|
||||
|
||||
const gameResult = buildGameResult(detail, userByPuuid, 1);
|
||||
if (!gameResult || gameResult.participants.length === 0) return 'skipped';
|
||||
const gameResult = buildGameResult(detail, userByPuuid, 1);
|
||||
if (!gameResult || gameResult.participants.length === 0) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
const winnerTeamIndex = gameResult.winnerTeamIndex;
|
||||
const matchPayload: Partial<Match> = {
|
||||
communityId: new mongoose.Types.ObjectId(job.communityId),
|
||||
title: this.formatLegacyDate(detail.info.gameCreation),
|
||||
description: 'Legacy import (자동 생성)',
|
||||
mapId: detail.info.mapId ?? 11,
|
||||
status: 'finished',
|
||||
scheduledAt: new Date(detail.info.gameCreation),
|
||||
maxParticipants: 10,
|
||||
gameCount: 1,
|
||||
gameMode: 'pair',
|
||||
tournamentCodeIds: [],
|
||||
teamBuiltAt: new Date(detail.info.gameCreation),
|
||||
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
participants: buildParticipantsSnapshot(detail, userByPuuid),
|
||||
teams: buildTeams(detail, userByPuuid),
|
||||
result: {
|
||||
winnerTeamIndex,
|
||||
riotMatchIds: [riotMatchId],
|
||||
games: [gameResult],
|
||||
seriesResults: [
|
||||
{
|
||||
teamAIndex: 1,
|
||||
teamBIndex: 2,
|
||||
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
|
||||
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
|
||||
const winnerTeamIndex = gameResult.winnerTeamIndex;
|
||||
const matchPayload: Partial<Match> = {
|
||||
communityId: new mongoose.Types.ObjectId(job.communityId),
|
||||
title: this.formatLegacyDate(detail.info.gameCreation),
|
||||
description: 'Legacy import (자동 생성)',
|
||||
mapId: detail.info.mapId ?? 11,
|
||||
status: 'finished',
|
||||
scheduledAt: new Date(detail.info.gameCreation),
|
||||
maxParticipants: 10,
|
||||
gameCount: 1,
|
||||
gameMode: 'pair',
|
||||
tournamentCodeIds: [],
|
||||
teamBuiltAt: new Date(detail.info.gameCreation),
|
||||
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
participants: buildParticipantsSnapshot(detail, userByPuuid),
|
||||
teams: buildTeams(detail, userByPuuid),
|
||||
result: {
|
||||
winnerTeamIndex,
|
||||
riotMatchIds: [riotMatchId],
|
||||
games: [gameResult],
|
||||
seriesResults: [
|
||||
{
|
||||
teamAIndex: 1,
|
||||
teamBIndex: 2,
|
||||
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
|
||||
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
|
||||
winnerTeamIndex,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
legacyImported: true,
|
||||
legacyImportSource: {
|
||||
importedAt: new Date(),
|
||||
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
triggeringPuuid: job.puuid,
|
||||
jobId: job.parentJobId,
|
||||
},
|
||||
prizes: [],
|
||||
notices: [],
|
||||
schedule: [],
|
||||
};
|
||||
const match = await MatchModel.create(matchPayload);
|
||||
await this.matchStatsService.buildAndUpsert(match._id);
|
||||
return 'imported';
|
||||
},
|
||||
legacyImported: true,
|
||||
legacyImportSource: {
|
||||
importedAt: new Date(),
|
||||
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
triggeringPuuid: job.puuid,
|
||||
jobId: job.parentJobId,
|
||||
},
|
||||
prizes: [],
|
||||
notices: [],
|
||||
schedule: [],
|
||||
};
|
||||
const match = await MatchModel.create(matchPayload);
|
||||
await this.matchStatsService.buildAndUpsert(match._id);
|
||||
return 'imported';
|
||||
);
|
||||
if (importOutcome === null) return 'duplicate';
|
||||
return importOutcome;
|
||||
} catch (error) {
|
||||
await this.recordError(job.parentJobId, {
|
||||
puuid: job.puuid,
|
||||
@@ -576,8 +595,6 @@ export class LegacyImportService {
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return 'error';
|
||||
} finally {
|
||||
await this.redisService.delete(lockKey);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,6 +643,7 @@ export class LegacyImportService {
|
||||
jobId: job.parentJobId,
|
||||
}).exec();
|
||||
if (!current || current.state === 'cancelled') return;
|
||||
if (current.completedMemberJobIds?.includes(job.jobId)) return;
|
||||
|
||||
const average =
|
||||
current.averageMemberSeconds === undefined
|
||||
@@ -634,14 +652,14 @@ export class LegacyImportService {
|
||||
current.averageMemberSeconds * 0.7 + stats.elapsedSeconds * 0.3,
|
||||
);
|
||||
|
||||
const nextProcessed = current.processedMembers + 1;
|
||||
const completed = nextProcessed >= current.totalMembers;
|
||||
await LegacyImportJobModel.updateOne(
|
||||
{ jobId: job.parentJobId },
|
||||
const updateResult = await LegacyImportJobModel.updateOne(
|
||||
{
|
||||
jobId: job.parentJobId,
|
||||
completedMemberJobIds: { $ne: job.jobId },
|
||||
state: { $ne: 'cancelled' },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
state: completed ? 'completed' : current.state,
|
||||
completedAt: completed ? new Date() : current.completedAt,
|
||||
averageMemberSeconds: average,
|
||||
},
|
||||
$inc: {
|
||||
@@ -651,14 +669,49 @@ export class LegacyImportService {
|
||||
totalSkippedDuplicate: stats.duplicates,
|
||||
totalErrors: stats.errors,
|
||||
},
|
||||
$addToSet: {
|
||||
completedMemberJobIds: job.jobId,
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
if (updateResult.modifiedCount === 0) return;
|
||||
|
||||
const updated = await LegacyImportJobModel.findOne({
|
||||
jobId: job.parentJobId,
|
||||
})
|
||||
.select({ processedMembers: 1, totalMembers: 1, state: 1, completedAt: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
const completed = Boolean(
|
||||
updated &&
|
||||
updated.state !== 'cancelled' &&
|
||||
updated.processedMembers >= updated.totalMembers,
|
||||
);
|
||||
if (completed) {
|
||||
await LegacyImportJobModel.updateOne(
|
||||
{
|
||||
jobId: job.parentJobId,
|
||||
state: { $ne: 'cancelled' },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
state: 'completed',
|
||||
completedAt: updated?.completedAt ?? new Date(),
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
}
|
||||
|
||||
if (completed) {
|
||||
await Promise.all([
|
||||
this.redisService.delete(
|
||||
this.communityLockKey(new mongoose.Types.ObjectId(job.communityId)),
|
||||
),
|
||||
this.distributedLockService.releaseLockIfOwned({
|
||||
lockKey: this.communityLockKey(
|
||||
new mongoose.Types.ObjectId(job.communityId),
|
||||
),
|
||||
lockOwnershipToken: job.parentJobId,
|
||||
lockExpirationDurationMilliseconds:
|
||||
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
}),
|
||||
this.communityStatsService.invalidateCommunity(
|
||||
new mongoose.Types.ObjectId(job.communityId),
|
||||
),
|
||||
@@ -680,6 +733,20 @@ export class LegacyImportService {
|
||||
return legacyJob;
|
||||
}
|
||||
|
||||
private async releaseDistributedLockSafely(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
): Promise<void> {
|
||||
await this.distributedLockService
|
||||
.releaseLockIfOwned(lockOwnership)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release legacy import lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async isCancelled(parentJobId: string): Promise<boolean> {
|
||||
const LegacyImportJobModel =
|
||||
await this.connectionService.getModel('legacyImportJob');
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
@@ -35,11 +36,18 @@ function makeScheduler() {
|
||||
};
|
||||
const redis = {
|
||||
getString: jest.fn().mockResolvedValue(null),
|
||||
setString: jest.fn().mockResolvedValue(undefined),
|
||||
setStringIfAbsent: jest.fn().mockResolvedValue(true),
|
||||
delete: jest.fn().mockResolvedValue(1),
|
||||
raw: jest.fn().mockReturnValue(rawClient),
|
||||
} as unknown as jest.Mocked<RedisService> & { raw: jest.Mock };
|
||||
const distributedLockService = {
|
||||
acquireLock: jest.fn().mockResolvedValue({
|
||||
lockKey: 'distributed-lock:riot-background-refresh:leader',
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds: 90_000,
|
||||
}),
|
||||
extendLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||
releaseLockIfOwned: jest.fn().mockResolvedValue(true),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const rateLimiter = {
|
||||
isRefreshWindowOpen: jest.fn().mockResolvedValue(true),
|
||||
} as unknown as jest.Mocked<RiotRateLimiterService>;
|
||||
@@ -52,6 +60,7 @@ function makeScheduler() {
|
||||
} as unknown as jest.Mocked<StaleAccountRepository>;
|
||||
const scheduler = new BackgroundRefreshScheduler(
|
||||
redis,
|
||||
distributedLockService,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
staleAccountRepository,
|
||||
@@ -60,6 +69,7 @@ function makeScheduler() {
|
||||
return {
|
||||
scheduler,
|
||||
redis,
|
||||
distributedLockService,
|
||||
rawClient,
|
||||
rateLimiter,
|
||||
enrichmentService,
|
||||
@@ -115,8 +125,9 @@ describe('BackgroundRefreshScheduler', () => {
|
||||
|
||||
it('does not query stale accounts when another leader holds the lock', async () => {
|
||||
jest.useFakeTimers().setSystemTime(new Date('2026-05-02T18:00:00.000Z'));
|
||||
const { scheduler, redis, staleAccountRepository } = makeScheduler();
|
||||
redis.getString.mockResolvedValue('another-consumer');
|
||||
const { scheduler, distributedLockService, staleAccountRepository } =
|
||||
makeScheduler();
|
||||
distributedLockService.acquireLock.mockResolvedValueOnce(null);
|
||||
|
||||
await scheduler.tick();
|
||||
|
||||
|
||||
@@ -9,13 +9,17 @@ import {
|
||||
RiotAccountEnrichmentService,
|
||||
} from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRateLimiterService } from '../riot/riot-rate-limiter.service';
|
||||
import {
|
||||
type DistributedLockOwnership,
|
||||
DistributedLockService,
|
||||
} from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { isInKstBackgroundWindow } from './kst-window.util';
|
||||
import { StaleAccountRepository } from './stale-account.repository';
|
||||
|
||||
const TICK_INTERVAL_MS = 60_000;
|
||||
const LEADER_KEY = 'riot:bg-refresh:leader';
|
||||
const LEADER_TTL_SECONDS = 90;
|
||||
const LEADER_LOCK_KEY = 'distributed-lock:riot-background-refresh:leader';
|
||||
const LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 90_000;
|
||||
const LEADER_RENEW_INTERVAL_MS = 30_000;
|
||||
const STALE_THRESHOLD_MS = 6 * 60 * 60 * 1000;
|
||||
const PER_TICK_CANDIDATE_LIMIT = 20;
|
||||
@@ -38,10 +42,11 @@ export class BackgroundRefreshScheduler
|
||||
private tickTimer?: NodeJS.Timeout;
|
||||
private leaderRenewTimer?: NodeJS.Timeout;
|
||||
private tickInProgress = false;
|
||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly redis: RedisService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
private readonly rateLimiter: RiotRateLimiterService,
|
||||
private readonly enrichmentService: RiotAccountEnrichmentService,
|
||||
private readonly staleAccountRepository: StaleAccountRepository,
|
||||
@@ -59,9 +64,20 @@ export class BackgroundRefreshScheduler
|
||||
void this.safeTick();
|
||||
}
|
||||
|
||||
public onModuleDestroy(): void {
|
||||
public async onModuleDestroy(): Promise<void> {
|
||||
if (this.tickTimer) clearInterval(this.tickTimer);
|
||||
if (this.leaderRenewTimer) clearInterval(this.leaderRenewTimer);
|
||||
if (!this.leaderLockOwnership) return;
|
||||
await this.distributedLockService
|
||||
.releaseLockIfOwned(this.leaderLockOwnership)
|
||||
.catch((err: unknown) => {
|
||||
this.logger.warn(
|
||||
`Background refresh leader release failed: ${
|
||||
err instanceof Error ? err.message : err
|
||||
}`,
|
||||
);
|
||||
});
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
|
||||
private async safeTick(): Promise<void> {
|
||||
@@ -158,33 +174,33 @@ export class BackgroundRefreshScheduler
|
||||
}
|
||||
|
||||
private async tryBecomeLeader(): Promise<boolean> {
|
||||
const current = await this.redis.getString(LEADER_KEY);
|
||||
if (current === this.consumer) {
|
||||
await this.redis.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
return true;
|
||||
if (this.leaderLockOwnership) {
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return true;
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
if (current) return false;
|
||||
|
||||
return this.redis.setStringIfAbsent(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
||||
lockKey: LEADER_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
});
|
||||
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
||||
return Boolean(leaderLockOwnership);
|
||||
}
|
||||
|
||||
private async renewLeaderIfHeld(): Promise<void> {
|
||||
try {
|
||||
const current = await this.redis.getString(LEADER_KEY);
|
||||
if (current !== this.consumer) return;
|
||||
await this.redis.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
if (!this.leaderLockOwnership) return;
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return;
|
||||
this.leaderLockOwnership = undefined;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Background refresh leader renew failed: ${
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||
@@ -19,6 +20,16 @@ function makeWorker() {
|
||||
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
||||
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
|
||||
} as unknown as jest.Mocked<RedisService>;
|
||||
const distributedLockService = {
|
||||
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||
operation({
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
}),
|
||||
),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const jobQueueService = {
|
||||
markProcessing: jest.fn().mockResolvedValue(undefined),
|
||||
markCompleted: jest.fn().mockResolvedValue({
|
||||
@@ -54,6 +65,7 @@ function makeWorker() {
|
||||
|
||||
const worker = new RiotJobWorkerService(
|
||||
redisService,
|
||||
distributedLockService,
|
||||
jobQueueService,
|
||||
recentMatchesService,
|
||||
connectionService,
|
||||
@@ -66,6 +78,7 @@ function makeWorker() {
|
||||
return {
|
||||
worker,
|
||||
redisService,
|
||||
distributedLockService,
|
||||
jobQueueService,
|
||||
recentMatchesService,
|
||||
connectionService,
|
||||
@@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => {
|
||||
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
|
||||
const { worker, distributedLockService, redisService } = makeWorker();
|
||||
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
const job: RiotRefreshSubJob = {
|
||||
jobId: 'job-1',
|
||||
parentJobId: 'parent-1',
|
||||
userId,
|
||||
puuid: 'puuid-1',
|
||||
region: 'asia',
|
||||
platform: 'kr',
|
||||
queueGroup: 'summoners-rift',
|
||||
queueIds: [420],
|
||||
requestedAt: Date.now(),
|
||||
priority: 'high',
|
||||
};
|
||||
|
||||
await (
|
||||
worker as unknown as {
|
||||
processEntry: (
|
||||
stream: string,
|
||||
entry: { id: string; message: Record<string, string> },
|
||||
) => Promise<void>;
|
||||
}
|
||||
).processEntry('riot:queue:high', {
|
||||
id: '1-0',
|
||||
message: { job: JSON.stringify(job) },
|
||||
});
|
||||
|
||||
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
|
||||
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,10 @@ import {
|
||||
} from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import {
|
||||
DistributedLockOwnership,
|
||||
DistributedLockService,
|
||||
} from '../global/redis/distributed-lock.service';
|
||||
import { RedisService, StreamEntry } from '../global/redis/redis.service';
|
||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
|
||||
@@ -23,17 +27,24 @@ import {
|
||||
import { RiotJobQueueService } from './riot-job-queue.service';
|
||||
import { RiotRefreshGateway } from './riot-refresh.gateway';
|
||||
|
||||
const LEADER_KEY = 'riot:worker:leader';
|
||||
const LEADER_TTL_SECONDS = 60;
|
||||
const RIOT_JOB_WORKER_LEADER_LOCK_KEY =
|
||||
'distributed-lock:riot-job-worker:leader';
|
||||
const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
|
||||
const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000;
|
||||
const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000;
|
||||
const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000;
|
||||
const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1;
|
||||
|
||||
@Injectable()
|
||||
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(RiotJobWorkerService.name);
|
||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
||||
private stopped = false;
|
||||
|
||||
constructor(
|
||||
private readonly redisService: RedisService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
private readonly jobQueueService: RiotJobQueueService,
|
||||
private readonly recentMatchesService: RiotRecentMatchesService,
|
||||
private readonly connectionService: ConnectionService,
|
||||
@@ -48,8 +59,19 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
void this.loop();
|
||||
}
|
||||
|
||||
public onModuleDestroy() {
|
||||
public async onModuleDestroy() {
|
||||
this.stopped = true;
|
||||
if (!this.leaderLockOwnership) return;
|
||||
await this.distributedLockService
|
||||
.releaseLockIfOwned(this.leaderLockOwnership)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release Riot job worker leader lock: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
|
||||
private async loop(): Promise<void> {
|
||||
@@ -63,7 +85,9 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
await this.jobQueueService.promoteDueScheduledJobs();
|
||||
const entry =
|
||||
(await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ??
|
||||
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
|
||||
(await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ??
|
||||
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
|
||||
if (!entry) continue;
|
||||
|
||||
@@ -89,6 +113,31 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
return;
|
||||
}
|
||||
|
||||
const didProcessJob = await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: this.jobProcessingLockKey(job),
|
||||
lockExpirationDurationMilliseconds:
|
||||
RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.processJob(job);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
if (!didProcessJob) {
|
||||
this.logger.debug(
|
||||
`Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.acknowledgeAndDelete(stream, entry.id);
|
||||
}
|
||||
|
||||
private async processJob(job: RiotJobPayload): Promise<void> {
|
||||
try {
|
||||
if (this.isTournamentResultJob(job)) {
|
||||
await this.tournamentResultService.processPollJob(job);
|
||||
@@ -136,8 +185,6 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
} finally {
|
||||
await this.acknowledgeAndDelete(stream, entry.id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,15 +275,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
): Promise<{ stream: string; message: StreamEntry } | null> {
|
||||
const result = await this.redisService.readGroup(
|
||||
stream,
|
||||
RIOT_JOB_GROUP,
|
||||
this.consumer,
|
||||
'>',
|
||||
{ count: 1, blockMs },
|
||||
RIOT_JOB_GROUP,
|
||||
this.consumerIdentifier,
|
||||
'>',
|
||||
{ count: 1, blockMs },
|
||||
);
|
||||
const message = result?.[0]?.messages[0];
|
||||
return message ? { stream, message } : null;
|
||||
}
|
||||
|
||||
private async readOneStalePendingMessage(
|
||||
stream: string,
|
||||
): Promise<{ stream: string; message: StreamEntry } | null> {
|
||||
const result = await this.redisService.claimStalePendingStreamMessages({
|
||||
key: stream,
|
||||
group: RIOT_JOB_GROUP,
|
||||
consumer: this.consumerIdentifier,
|
||||
stalePendingMessageMinimumIdleMilliseconds:
|
||||
STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
|
||||
count: STALE_PENDING_MESSAGE_CLAIM_COUNT,
|
||||
});
|
||||
const message = result.messages[0];
|
||||
return message ? { stream, message } : null;
|
||||
}
|
||||
|
||||
private async ensureConsumerGroups(): Promise<void> {
|
||||
await Promise.all(
|
||||
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
|
||||
@@ -252,21 +314,26 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
private async tryBecomeLeader(): Promise<boolean> {
|
||||
const current = await this.redisService.getString(LEADER_KEY);
|
||||
if (current === this.consumer) {
|
||||
await this.redisService.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
return true;
|
||||
if (this.leaderLockOwnership) {
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return true;
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
if (current) return false;
|
||||
return this.redisService.setStringIfAbsent(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
|
||||
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
||||
lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
});
|
||||
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
||||
return Boolean(leaderLockOwnership);
|
||||
}
|
||||
|
||||
private jobProcessingLockKey(job: RiotJobPayload): string {
|
||||
return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`;
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
|
||||
@@ -4,14 +4,14 @@ import type {
|
||||
RiotMatch as StoredRiotMatch,
|
||||
} from '../global/database/schema';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import type { RiotRegionalRoute } from './riot.types';
|
||||
import { RiotMatchService } from './riot-match.service';
|
||||
|
||||
export const RIOT_RECENT_MATCH_LIMIT = 20;
|
||||
export const RIOT_SOLO_QUEUE_ID = 420;
|
||||
|
||||
const MATCH_ENRICHMENT_LOCK_TTL_SECONDS = 60;
|
||||
const MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
|
||||
const MATCH_DETAIL_CONCURRENCY = 2;
|
||||
const DEFAULT_REGION: RiotRegionalRoute = 'asia';
|
||||
|
||||
@@ -22,7 +22,7 @@ export class RiotRecentMatchesService {
|
||||
constructor(
|
||||
private readonly connectionService: ConnectionService,
|
||||
private readonly matchService: RiotMatchService,
|
||||
private readonly redisService: RedisService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public async syncRecentMatches(params: {
|
||||
@@ -34,83 +34,84 @@ export class RiotRecentMatchesService {
|
||||
const region = params.region ?? DEFAULT_REGION;
|
||||
const limit = params.limit ?? RIOT_RECENT_MATCH_LIMIT;
|
||||
const queueId = params.queueId ?? RIOT_SOLO_QUEUE_ID;
|
||||
const lockKey = `riot:enrich:matches:${params.puuid}`;
|
||||
const lockValue = `${process.pid}:${Date.now()}`;
|
||||
|
||||
const acquired = await this.redisService.setStringIfAbsent(
|
||||
lockKey,
|
||||
lockValue,
|
||||
MATCH_ENRICHMENT_LOCK_TTL_SECONDS,
|
||||
);
|
||||
if (!acquired) {
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
try {
|
||||
const matchIds = await this.matchService.getMatchIdsByPuuid(params.puuid, {
|
||||
region,
|
||||
queue: queueId,
|
||||
count: limit,
|
||||
});
|
||||
if (matchIds.length === 0) {
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
const RiotMatchModel = await this.connectionService.getModel('riotMatch');
|
||||
const existingMatches = await RiotMatchModel.find({
|
||||
matchId: { $in: matchIds },
|
||||
})
|
||||
.select({ matchId: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
const existingMatchIds = new Set(existingMatches.map((match) => match.matchId));
|
||||
const missingMatchIds = matchIds.filter((matchId) => !existingMatchIds.has(matchId));
|
||||
|
||||
await runWithConcurrency(missingMatchIds, MATCH_DETAIL_CONCURRENCY, async (matchId) => {
|
||||
try {
|
||||
const dto = await this.matchService.getMatch(matchId, { region });
|
||||
if (dto.info.queueId !== queueId) {
|
||||
this.logger.warn(
|
||||
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: Partial<StoredRiotMatch> = {
|
||||
matchId: dto.metadata.matchId,
|
||||
gameCreation: dto.info.gameCreation,
|
||||
gameDuration: dto.info.gameDuration,
|
||||
gameVersion: dto.info.gameVersion,
|
||||
queueId: dto.info.queueId,
|
||||
participantPuuids: dto.metadata.participants,
|
||||
raw: dto as unknown as Record<string, unknown>,
|
||||
fetchedAt: new Date(),
|
||||
};
|
||||
|
||||
await RiotMatchModel.updateOne(
|
||||
{ matchId: dto.metadata.matchId },
|
||||
{ $setOnInsert: payload },
|
||||
{ upsert: true },
|
||||
).exec();
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn(
|
||||
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
} finally {
|
||||
await this.redisService.delete(lockKey).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release recent matches lock for ${params.puuid}: ${
|
||||
(error as Error)?.message ?? error
|
||||
}`,
|
||||
const result = await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: `distributed-lock:riot-recent-matches:sync:puuid:${params.puuid}`,
|
||||
lockExpirationDurationMilliseconds:
|
||||
MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
const matchIds = await this.matchService.getMatchIdsByPuuid(
|
||||
params.puuid,
|
||||
{
|
||||
region,
|
||||
queue: queueId,
|
||||
count: limit,
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
if (matchIds.length === 0) {
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
const RiotMatchModel =
|
||||
await this.connectionService.getModel('riotMatch');
|
||||
const existingMatches = await RiotMatchModel.find({
|
||||
matchId: { $in: matchIds },
|
||||
})
|
||||
.select({ matchId: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
const existingMatchIds = new Set(
|
||||
existingMatches.map((match) => match.matchId),
|
||||
);
|
||||
const missingMatchIds = matchIds.filter(
|
||||
(matchId) => !existingMatchIds.has(matchId),
|
||||
);
|
||||
|
||||
await runWithConcurrency(
|
||||
missingMatchIds,
|
||||
MATCH_DETAIL_CONCURRENCY,
|
||||
async (matchId) => {
|
||||
try {
|
||||
const dto = await this.matchService.getMatch(matchId, { region });
|
||||
if (dto.info.queueId !== queueId) {
|
||||
this.logger.warn(
|
||||
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: Partial<StoredRiotMatch> = {
|
||||
matchId: dto.metadata.matchId,
|
||||
gameCreation: dto.info.gameCreation,
|
||||
gameDuration: dto.info.gameDuration,
|
||||
gameVersion: dto.info.gameVersion,
|
||||
queueId: dto.info.queueId,
|
||||
participantPuuids: dto.metadata.participants,
|
||||
raw: dto as unknown as Record<string, unknown>,
|
||||
fetchedAt: new Date(),
|
||||
};
|
||||
|
||||
await RiotMatchModel.updateOne(
|
||||
{ matchId: dto.metadata.matchId },
|
||||
{ $setOnInsert: payload },
|
||||
{ upsert: true },
|
||||
).exec();
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn(
|
||||
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
},
|
||||
);
|
||||
|
||||
return result ?? this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
public async ensureFreshForQueue(params: {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { TournamentCodeScheduler } from './tournament-code.scheduler';
|
||||
|
||||
describe('TournamentCodeScheduler', () => {
|
||||
@@ -18,16 +19,28 @@ describe('TournamentCodeScheduler', () => {
|
||||
scheduleInitialPoll:
|
||||
params?.scheduleInitialPoll ?? jest.fn().mockResolvedValue(null),
|
||||
};
|
||||
const distributedLockService = {
|
||||
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||
operation({
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
}),
|
||||
),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const scheduler = new TournamentCodeScheduler(
|
||||
tournamentCodeService as never,
|
||||
autoProvisionService as never,
|
||||
tournamentResultService as never,
|
||||
distributedLockService as never,
|
||||
) as unknown as { attachPending(): Promise<void> };
|
||||
|
||||
return {
|
||||
scheduler,
|
||||
tournamentCodeService,
|
||||
tournamentResultService,
|
||||
distributedLockService,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -59,4 +72,16 @@ describe('TournamentCodeScheduler', () => {
|
||||
|
||||
expect(tournamentResultService.scheduleInitialPoll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not attach pending tournament codes when another instance owns the scheduler lock', async () => {
|
||||
const { scheduler, tournamentCodeService, distributedLockService } =
|
||||
createScheduler();
|
||||
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
|
||||
|
||||
await scheduler.attachPending();
|
||||
|
||||
expect(
|
||||
tournamentCodeService.attachPendingMatchDocuments,
|
||||
).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,18 @@
|
||||
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||
import { TournamentCodeAutoProvisionService } from './tournament-code-auto-provision.service';
|
||||
import { TournamentCodeService } from './tournament-code.service';
|
||||
|
||||
const ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:attach-pending-codes-to-matches';
|
||||
const MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:mark-expired-assigned-codes';
|
||||
const ENSURE_VENDOR_CODE_POOL_LOCK_KEY =
|
||||
'distributed-lock:tournament-code:scheduler:ensure-vendor-code-pool';
|
||||
const SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 10 * 60_000;
|
||||
const SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 2 * 60_000;
|
||||
|
||||
@Injectable()
|
||||
export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(TournamentCodeScheduler.name);
|
||||
@@ -14,6 +24,7 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly tournamentCodeService: TournamentCodeService,
|
||||
private readonly autoProvisionService: TournamentCodeAutoProvisionService,
|
||||
private readonly tournamentResultService: TournamentResultService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public onModuleInit() {
|
||||
@@ -47,16 +58,27 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async attachPending() {
|
||||
try {
|
||||
const matches =
|
||||
await this.tournamentCodeService.attachPendingMatchDocuments();
|
||||
if (matches.length > 0) {
|
||||
this.logger.debug(
|
||||
`Attached tournament codes for ${matches.length} matches.`,
|
||||
);
|
||||
for (const match of matches) {
|
||||
await this.tournamentResultService.scheduleInitialPoll(match);
|
||||
}
|
||||
}
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: ATTACH_PENDING_CODES_TO_MATCHES_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
const matches =
|
||||
await this.tournamentCodeService.attachPendingMatchDocuments();
|
||||
if (matches.length > 0) {
|
||||
this.logger.debug(
|
||||
`Attached tournament codes for ${matches.length} matches.`,
|
||||
);
|
||||
for (const match of matches) {
|
||||
await this.tournamentResultService.scheduleInitialPoll(match);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to attach pending tournament codes: ${
|
||||
@@ -68,7 +90,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async markExpired() {
|
||||
try {
|
||||
await this.tournamentCodeService.markExpired();
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: MARK_EXPIRED_ASSIGNED_CODES_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.tournamentCodeService.markExpired();
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to mark expired tournament codes: ${
|
||||
@@ -80,7 +113,18 @@ export class TournamentCodeScheduler implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async ensureVendorPool() {
|
||||
try {
|
||||
await this.autoProvisionService.ensurePool();
|
||||
await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: ENSURE_VENDOR_CODE_POOL_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
SCHEDULER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
SCHEDULER_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.autoProvisionService.ensurePool();
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to auto-provision tournament codes: ${
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import mongoose from 'mongoose';
|
||||
import type { MatchDocument } from '../global/database/schema';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import type { RiotMatchDto } from '../riot/riot.types';
|
||||
import { TournamentResultService } from './tournament-result.service';
|
||||
|
||||
@@ -16,15 +17,38 @@ type TournamentResultServiceInternals = TournamentResultService & {
|
||||
};
|
||||
|
||||
function createService() {
|
||||
const distributedLockService = createDistributedLockService();
|
||||
return new TournamentResultService(
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
distributedLockService as never,
|
||||
) as unknown as TournamentResultServiceInternals;
|
||||
}
|
||||
|
||||
function createDistributedLockService() {
|
||||
return {
|
||||
runWithLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||
operation({
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
}),
|
||||
),
|
||||
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||
operation({
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
}),
|
||||
),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
}
|
||||
|
||||
interface MatchOverrides {
|
||||
gameCount?: number;
|
||||
gameMode?: 'pair' | 'bracket' | 'round-robin';
|
||||
@@ -281,12 +305,14 @@ describe('TournamentResultService tournament code polling', () => {
|
||||
const riotJobQueueService = {
|
||||
enqueueTournamentResultPoll: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const distributedLockService = createDistributedLockService();
|
||||
const service = new TournamentResultService(
|
||||
connectionService as never,
|
||||
riotMatchService as never,
|
||||
riotJobQueueService as never,
|
||||
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
distributedLockService as never,
|
||||
);
|
||||
|
||||
return { service, match, code, riotMatchService };
|
||||
@@ -455,12 +481,14 @@ describe('TournamentResultService manual result collection', () => {
|
||||
getMatchIdsByPuuid: jest.fn().mockResolvedValue(['KR_1234567890']),
|
||||
getMatch: jest.fn().mockResolvedValue(detail),
|
||||
};
|
||||
const distributedLockService = createDistributedLockService();
|
||||
const service = new TournamentResultService(
|
||||
connectionService as never,
|
||||
riotMatchService as never,
|
||||
{} as never,
|
||||
{ invalidateCommunity: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
{ buildAndUpsert: jest.fn().mockResolvedValue(undefined) } as never,
|
||||
distributedLockService as never,
|
||||
);
|
||||
|
||||
return {
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import type {
|
||||
MatchDocument,
|
||||
MatchGameResult,
|
||||
@@ -46,6 +47,12 @@ const POLL_EXPIRY_MS = 24 * 60 * 60_000;
|
||||
const EXPAND_AFTER_MS = 60 * 60_000;
|
||||
const EXPANDED_MEMBER_LIMIT_PER_TEAM = 2;
|
||||
const TOURNAMENT_LOOKUP_START_MARGIN_SECONDS = 60;
|
||||
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS =
|
||||
30 * 60_000;
|
||||
const TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS =
|
||||
5 * 60_000;
|
||||
const TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS =
|
||||
5 * 60_000;
|
||||
|
||||
interface ManualGameAssignment {
|
||||
gameNumber: number;
|
||||
@@ -64,11 +71,29 @@ export class TournamentResultService {
|
||||
private readonly riotJobQueueService: RiotJobQueueService,
|
||||
private readonly communityStatsService: CommunityStatsService,
|
||||
private readonly matchStatsService: MatchStatsService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public async scheduleInitialPoll(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
priority: RiotJobPriority = 'high',
|
||||
): Promise<EnqueueTournamentResultPollResult | null> {
|
||||
const matchIdentifier = this.resolveMatchIdentifier(matchOrId);
|
||||
return this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: this.tournamentResultPollProcessingLockKey(matchIdentifier),
|
||||
lockExpirationDurationMilliseconds:
|
||||
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockAcquireTimeoutMilliseconds: 1_000,
|
||||
lockRetryDelayMilliseconds: 100,
|
||||
},
|
||||
async () => this.scheduleInitialPollWhileLockOwned(matchOrId, priority),
|
||||
);
|
||||
}
|
||||
|
||||
private async scheduleInitialPollWhileLockOwned(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
priority: RiotJobPriority,
|
||||
): Promise<EnqueueTournamentResultPollResult | null> {
|
||||
const match = await this.loadMatch(matchOrId);
|
||||
if (match.status === 'finished') return null;
|
||||
@@ -195,6 +220,29 @@ export class TournamentResultService {
|
||||
}
|
||||
|
||||
public async processPollJob(job: PollTournamentResultJob): Promise<void> {
|
||||
const didProcessPollJob = await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: this.tournamentResultPollProcessingLockKey(job.matchId),
|
||||
lockExpirationDurationMilliseconds:
|
||||
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
TOURNAMENT_RESULT_POLL_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.processPollJobWhileLockOwned(job);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
if (didProcessPollJob) return;
|
||||
this.logger.debug(
|
||||
`Skipped tournament result poll job ${job.jobId}; another worker is processing match ${job.matchId}.`,
|
||||
);
|
||||
}
|
||||
|
||||
private async processPollJobWhileLockOwned(
|
||||
job: PollTournamentResultJob,
|
||||
): Promise<void> {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
@@ -818,12 +866,28 @@ export class TournamentResultService {
|
||||
|
||||
private async reserveReadyDoubleEliminationCodes(
|
||||
match: MatchDocument,
|
||||
): Promise<void> {
|
||||
await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: this.tournamentResultReadyCodeReservationLockKey(
|
||||
match._id.toString(),
|
||||
),
|
||||
lockExpirationDurationMilliseconds:
|
||||
TOURNAMENT_RESULT_READY_CODE_RESERVATION_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
},
|
||||
async () => this.reserveReadyDoubleEliminationCodesWhileLockOwned(match),
|
||||
);
|
||||
}
|
||||
|
||||
private async reserveReadyDoubleEliminationCodesWhileLockOwned(
|
||||
match: MatchDocument,
|
||||
): Promise<void> {
|
||||
if ((match.gameMode ?? 'pair') !== 'double-elimination') return;
|
||||
const readyBracketMatches = (match.doubleElimination?.bracketMatches ?? [])
|
||||
.filter((bracketMatch) => bracketMatch.state === 'ready');
|
||||
if (readyBracketMatches.length === 0) return;
|
||||
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const TournamentCodeModel =
|
||||
await this.connectionService.getModel('tournamentCode');
|
||||
const existingCodes =
|
||||
@@ -871,8 +935,19 @@ export class TournamentResultService {
|
||||
}
|
||||
|
||||
if (reservedIds.length === 0) return;
|
||||
match.tournamentCodeIds.push(...reservedIds);
|
||||
await match.save();
|
||||
await MatchModel.updateOne(
|
||||
{ _id: match._id },
|
||||
{ $addToSet: { tournamentCodeIds: { $each: reservedIds } } },
|
||||
).exec();
|
||||
const existingTournamentCodeIds = new Set(
|
||||
match.tournamentCodeIds.map((tournamentCodeId) =>
|
||||
tournamentCodeId.toString(),
|
||||
),
|
||||
);
|
||||
for (const reservedId of reservedIds) {
|
||||
if (existingTournamentCodeIds.has(reservedId.toString())) continue;
|
||||
match.tournamentCodeIds.push(reservedId);
|
||||
}
|
||||
}
|
||||
|
||||
private async markTournamentCodeUsedForManualResult(
|
||||
@@ -1126,6 +1201,24 @@ export class TournamentResultService {
|
||||
if (!match) throw new NotFoundException('Match not found.');
|
||||
return match;
|
||||
}
|
||||
|
||||
private resolveMatchIdentifier(
|
||||
matchOrId: MatchDocument | mongoose.Types.ObjectId,
|
||||
): string {
|
||||
return matchOrId instanceof mongoose.Types.ObjectId
|
||||
? matchOrId.toString()
|
||||
: matchOrId._id.toString();
|
||||
}
|
||||
|
||||
private tournamentResultPollProcessingLockKey(matchIdentifier: string): string {
|
||||
return `distributed-lock:tournament-result:process-poll-job:match:${matchIdentifier}`;
|
||||
}
|
||||
|
||||
private tournamentResultReadyCodeReservationLockKey(
|
||||
matchIdentifier: string,
|
||||
): string {
|
||||
return `distributed-lock:tournament-result:reserve-ready-double-elimination-codes:match:${matchIdentifier}`;
|
||||
}
|
||||
}
|
||||
|
||||
function isRiotMatchDto(value: unknown): value is RiotMatchDto {
|
||||
|
||||
Reference in new issue
Block a user