fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||
@@ -19,6 +20,16 @@ function makeWorker() {
|
||||
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
||||
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
|
||||
} as unknown as jest.Mocked<RedisService>;
|
||||
const distributedLockService = {
|
||||
runWithRenewedLock: jest.fn().mockImplementation(async (options, operation) =>
|
||||
operation({
|
||||
lockKey: options.lockKey,
|
||||
lockOwnershipToken: 'lock-owner',
|
||||
lockExpirationDurationMilliseconds:
|
||||
options.lockExpirationDurationMilliseconds,
|
||||
}),
|
||||
),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const jobQueueService = {
|
||||
markProcessing: jest.fn().mockResolvedValue(undefined),
|
||||
markCompleted: jest.fn().mockResolvedValue({
|
||||
@@ -54,6 +65,7 @@ function makeWorker() {
|
||||
|
||||
const worker = new RiotJobWorkerService(
|
||||
redisService,
|
||||
distributedLockService,
|
||||
jobQueueService,
|
||||
recentMatchesService,
|
||||
connectionService,
|
||||
@@ -66,6 +78,7 @@ function makeWorker() {
|
||||
return {
|
||||
worker,
|
||||
redisService,
|
||||
distributedLockService,
|
||||
jobQueueService,
|
||||
recentMatchesService,
|
||||
connectionService,
|
||||
@@ -151,4 +164,37 @@ describe('RiotJobWorkerService', () => {
|
||||
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves a stream entry pending when another worker owns the job processing lock', async () => {
|
||||
const { worker, distributedLockService, redisService } = makeWorker();
|
||||
distributedLockService.runWithRenewedLock.mockResolvedValueOnce(null);
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
const job: RiotRefreshSubJob = {
|
||||
jobId: 'job-1',
|
||||
parentJobId: 'parent-1',
|
||||
userId,
|
||||
puuid: 'puuid-1',
|
||||
region: 'asia',
|
||||
platform: 'kr',
|
||||
queueGroup: 'summoners-rift',
|
||||
queueIds: [420],
|
||||
requestedAt: Date.now(),
|
||||
priority: 'high',
|
||||
};
|
||||
|
||||
await (
|
||||
worker as unknown as {
|
||||
processEntry: (
|
||||
stream: string,
|
||||
entry: { id: string; message: Record<string, string> },
|
||||
) => Promise<void>;
|
||||
}
|
||||
).processEntry('riot:queue:high', {
|
||||
id: '1-0',
|
||||
message: { job: JSON.stringify(job) },
|
||||
});
|
||||
|
||||
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
|
||||
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,10 @@ import {
|
||||
} from '@nestjs/common';
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import {
|
||||
DistributedLockOwnership,
|
||||
DistributedLockService,
|
||||
} from '../global/redis/distributed-lock.service';
|
||||
import { RedisService, StreamEntry } from '../global/redis/redis.service';
|
||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||
import type { LegacyImportMemberJob } from '../legacy-import/legacy-import.types';
|
||||
@@ -23,17 +27,24 @@ import {
|
||||
import { RiotJobQueueService } from './riot-job-queue.service';
|
||||
import { RiotRefreshGateway } from './riot-refresh.gateway';
|
||||
|
||||
const LEADER_KEY = 'riot:worker:leader';
|
||||
const LEADER_TTL_SECONDS = 60;
|
||||
const RIOT_JOB_WORKER_LEADER_LOCK_KEY =
|
||||
'distributed-lock:riot-job-worker:leader';
|
||||
const RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
|
||||
const RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS = 30 * 60_000;
|
||||
const RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS = 5 * 60_000;
|
||||
const STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS = 30 * 60_000;
|
||||
const STALE_PENDING_MESSAGE_CLAIM_COUNT = 1;
|
||||
|
||||
@Injectable()
|
||||
export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(RiotJobWorkerService.name);
|
||||
private readonly consumer = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private readonly consumerIdentifier = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
private leaderLockOwnership: DistributedLockOwnership | undefined;
|
||||
private stopped = false;
|
||||
|
||||
constructor(
|
||||
private readonly redisService: RedisService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
private readonly jobQueueService: RiotJobQueueService,
|
||||
private readonly recentMatchesService: RiotRecentMatchesService,
|
||||
private readonly connectionService: ConnectionService,
|
||||
@@ -48,8 +59,19 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
void this.loop();
|
||||
}
|
||||
|
||||
public onModuleDestroy() {
|
||||
public async onModuleDestroy() {
|
||||
this.stopped = true;
|
||||
if (!this.leaderLockOwnership) return;
|
||||
await this.distributedLockService
|
||||
.releaseLockIfOwned(this.leaderLockOwnership)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release Riot job worker leader lock: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
|
||||
private async loop(): Promise<void> {
|
||||
@@ -63,7 +85,9 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
await this.jobQueueService.promoteDueScheduledJobs();
|
||||
const entry =
|
||||
(await this.readOneStalePendingMessage(RIOT_JOB_HIGH_STREAM)) ??
|
||||
(await this.readOne(RIOT_JOB_HIGH_STREAM, 100)) ??
|
||||
(await this.readOneStalePendingMessage(RIOT_JOB_LOW_STREAM)) ??
|
||||
(await this.readOne(RIOT_JOB_LOW_STREAM, 1_000));
|
||||
if (!entry) continue;
|
||||
|
||||
@@ -89,6 +113,31 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
return;
|
||||
}
|
||||
|
||||
const didProcessJob = await this.distributedLockService.runWithRenewedLock(
|
||||
{
|
||||
lockKey: this.jobProcessingLockKey(job),
|
||||
lockExpirationDurationMilliseconds:
|
||||
RIOT_JOB_PROCESSING_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
lockRenewalIntervalMilliseconds:
|
||||
RIOT_JOB_PROCESSING_LOCK_RENEWAL_INTERVAL_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
await this.processJob(job);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
if (!didProcessJob) {
|
||||
this.logger.debug(
|
||||
`Skipped Riot job ${job.jobId}; another worker still owns the processing lock.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.acknowledgeAndDelete(stream, entry.id);
|
||||
}
|
||||
|
||||
private async processJob(job: RiotJobPayload): Promise<void> {
|
||||
try {
|
||||
if (this.isTournamentResultJob(job)) {
|
||||
await this.tournamentResultService.processPollJob(job);
|
||||
@@ -136,8 +185,6 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
} finally {
|
||||
await this.acknowledgeAndDelete(stream, entry.id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,15 +275,30 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
): Promise<{ stream: string; message: StreamEntry } | null> {
|
||||
const result = await this.redisService.readGroup(
|
||||
stream,
|
||||
RIOT_JOB_GROUP,
|
||||
this.consumer,
|
||||
'>',
|
||||
{ count: 1, blockMs },
|
||||
RIOT_JOB_GROUP,
|
||||
this.consumerIdentifier,
|
||||
'>',
|
||||
{ count: 1, blockMs },
|
||||
);
|
||||
const message = result?.[0]?.messages[0];
|
||||
return message ? { stream, message } : null;
|
||||
}
|
||||
|
||||
private async readOneStalePendingMessage(
|
||||
stream: string,
|
||||
): Promise<{ stream: string; message: StreamEntry } | null> {
|
||||
const result = await this.redisService.claimStalePendingStreamMessages({
|
||||
key: stream,
|
||||
group: RIOT_JOB_GROUP,
|
||||
consumer: this.consumerIdentifier,
|
||||
stalePendingMessageMinimumIdleMilliseconds:
|
||||
STALE_PENDING_MESSAGE_MINIMUM_IDLE_MILLISECONDS,
|
||||
count: STALE_PENDING_MESSAGE_CLAIM_COUNT,
|
||||
});
|
||||
const message = result.messages[0];
|
||||
return message ? { stream, message } : null;
|
||||
}
|
||||
|
||||
private async ensureConsumerGroups(): Promise<void> {
|
||||
await Promise.all(
|
||||
[RIOT_JOB_HIGH_STREAM, RIOT_JOB_LOW_STREAM].map(async (stream) => {
|
||||
@@ -252,21 +314,26 @@ export class RiotJobWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
private async tryBecomeLeader(): Promise<boolean> {
|
||||
const current = await this.redisService.getString(LEADER_KEY);
|
||||
if (current === this.consumer) {
|
||||
await this.redisService.setString(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
return true;
|
||||
if (this.leaderLockOwnership) {
|
||||
const didExtendLeaderLock =
|
||||
await this.distributedLockService.extendLockIfOwned(
|
||||
this.leaderLockOwnership,
|
||||
);
|
||||
if (didExtendLeaderLock) return true;
|
||||
this.leaderLockOwnership = undefined;
|
||||
}
|
||||
if (current) return false;
|
||||
return this.redisService.setStringIfAbsent(
|
||||
LEADER_KEY,
|
||||
this.consumer,
|
||||
LEADER_TTL_SECONDS,
|
||||
);
|
||||
|
||||
const leaderLockOwnership = await this.distributedLockService.acquireLock({
|
||||
lockKey: RIOT_JOB_WORKER_LEADER_LOCK_KEY,
|
||||
lockExpirationDurationMilliseconds:
|
||||
RIOT_JOB_WORKER_LEADER_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
});
|
||||
this.leaderLockOwnership = leaderLockOwnership ?? undefined;
|
||||
return Boolean(leaderLockOwnership);
|
||||
}
|
||||
|
||||
private jobProcessingLockKey(job: RiotJobPayload): string {
|
||||
return `distributed-lock:riot-job-worker:job:${job.kind ?? 'refresh-account'}:${job.jobId}`;
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
|
||||
Reference in new issue
Block a user