fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -10,6 +10,10 @@ import mongoose from 'mongoose';
import { MatchStatsService } from '../community-stats/match-stats.service';
import { CommunityStatsService } from '../community-stats/community-stats.service';
import { ConnectionService } from '../global/database/connection.service';
import {
type DistributedLockOwnership,
DistributedLockService,
} from '../global/redis/distributed-lock.service';
import type {
LegacyImportJobDocument,
Match,
@@ -57,6 +61,7 @@ export class LegacyImportService {
private readonly riotMatchService: RiotMatchService,
private readonly communityStatsService: CommunityStatsService,
private readonly matchStatsService: MatchStatsService,
private readonly distributedLockService: DistributedLockService,
) {}
public async previewMembers(params: {
@@ -80,12 +85,14 @@ export class LegacyImportService {
}): Promise<StartLegacyImportResponse> {
const lockKey = this.communityLockKey(params.communityId);
const jobId = randomUUID();
const acquired = await this.redisService.setStringIfAbsent(
lockKey,
jobId,
LEGACY_IMPORT_LOCK_TTL_SECONDS,
);
if (!acquired) {
const communityImportLockOwnership =
await this.distributedLockService.acquireLock({
lockKey,
lockOwnershipToken: jobId,
lockExpirationDurationMilliseconds:
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
});
if (!communityImportLockOwnership) {
throw new ConflictException('Legacy import is already running.');
}
@@ -133,6 +140,7 @@ export class LegacyImportService {
totalSkippedDuplicate: 0,
totalErrors: 0,
averageMemberSeconds: LEGACY_IMPORT_MEMBER_AVG_SECONDS,
completedMemberJobIds: [],
});
for (const window of windows) {
@@ -160,7 +168,7 @@ export class LegacyImportService {
{ jobId },
{ $set: { state: 'completed', completedAt: new Date() } },
).exec();
await this.redisService.delete(lockKey);
await this.releaseDistributedLockSafely(communityImportLockOwnership);
}
return {
@@ -169,7 +177,7 @@ export class LegacyImportService {
etaSeconds: windows.length * LEGACY_IMPORT_MEMBER_AVG_SECONDS,
};
} catch (error) {
await this.redisService.delete(lockKey);
await this.releaseDistributedLockSafely(communityImportLockOwnership);
throw error;
}
}
@@ -220,7 +228,11 @@ export class LegacyImportService {
).exec();
if (!job)
throw new NotFoundException('Running legacy import job not found.');
await this.redisService.delete(this.communityLockKey(params.communityId));
await this.distributedLockService.releaseLockIfOwned({
lockKey: this.communityLockKey(params.communityId),
lockOwnershipToken: job.jobId,
lockExpirationDurationMilliseconds: LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
});
return job;
}
@@ -228,6 +240,7 @@ export class LegacyImportService {
const started = Date.now();
const legacyJob = await this.getRunnableJob(job);
if (!legacyJob) return;
if (legacyJob.completedMemberJobIds?.includes(job.jobId)) return;
let discovered = 0;
let imported = 0;
@@ -500,75 +513,81 @@ export class LegacyImportService {
userByPuuid: UserByPuuidMap,
): Promise<ImportOutcome> {
const lockKey = `legacy-import:match:${riotMatchId}`;
const acquired = await this.redisService.setStringIfAbsent(
lockKey,
job.parentJobId,
MATCH_IMPORT_LOCK_TTL_SECONDS,
);
if (!acquired) return 'duplicate';
try {
const MatchModel = await this.connectionService.getModel('match');
const existing = await MatchModel.exists({
'result.games.riotMatchId': riotMatchId,
});
if (existing) return 'duplicate';
const importOutcome = await this.distributedLockService.runWithLock(
{
lockKey,
lockExpirationDurationMilliseconds:
MATCH_IMPORT_LOCK_TTL_SECONDS * 1000,
},
async () => {
const MatchModel = await this.connectionService.getModel('match');
const existing = await MatchModel.exists({
'result.games.riotMatchId': riotMatchId,
});
if (existing) return 'duplicate';
const detail = await this.ensureRiotMatchPersisted(
riotMatchId,
job.region,
);
if (!detail.info.tournamentCode) return 'skipped';
const counts = countKnownParticipants(detail, userByPuuid);
if (counts.total === 0) return 'skipped';
const detail = await this.ensureRiotMatchPersisted(
riotMatchId,
job.region,
);
if (!detail.info.tournamentCode) return 'skipped';
const counts = countKnownParticipants(detail, userByPuuid);
if (counts.total === 0) return 'skipped';
const gameResult = buildGameResult(detail, userByPuuid, 1);
if (!gameResult || gameResult.participants.length === 0) return 'skipped';
const gameResult = buildGameResult(detail, userByPuuid, 1);
if (!gameResult || gameResult.participants.length === 0) {
return 'skipped';
}
const winnerTeamIndex = gameResult.winnerTeamIndex;
const matchPayload: Partial<Match> = {
communityId: new mongoose.Types.ObjectId(job.communityId),
title: this.formatLegacyDate(detail.info.gameCreation),
description: 'Legacy import (자동 생성)',
mapId: detail.info.mapId ?? 11,
status: 'finished',
scheduledAt: new Date(detail.info.gameCreation),
maxParticipants: 10,
gameCount: 1,
gameMode: 'pair',
tournamentCodeIds: [],
teamBuiltAt: new Date(detail.info.gameCreation),
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
participants: buildParticipantsSnapshot(detail, userByPuuid),
teams: buildTeams(detail, userByPuuid),
result: {
winnerTeamIndex,
riotMatchIds: [riotMatchId],
games: [gameResult],
seriesResults: [
{
teamAIndex: 1,
teamBIndex: 2,
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
const winnerTeamIndex = gameResult.winnerTeamIndex;
const matchPayload: Partial<Match> = {
communityId: new mongoose.Types.ObjectId(job.communityId),
title: this.formatLegacyDate(detail.info.gameCreation),
description: 'Legacy import (자동 생성)',
mapId: detail.info.mapId ?? 11,
status: 'finished',
scheduledAt: new Date(detail.info.gameCreation),
maxParticipants: 10,
gameCount: 1,
gameMode: 'pair',
tournamentCodeIds: [],
teamBuiltAt: new Date(detail.info.gameCreation),
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
participants: buildParticipantsSnapshot(detail, userByPuuid),
teams: buildTeams(detail, userByPuuid),
result: {
winnerTeamIndex,
riotMatchIds: [riotMatchId],
games: [gameResult],
seriesResults: [
{
teamAIndex: 1,
teamBIndex: 2,
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
winnerTeamIndex,
},
],
},
],
legacyImported: true,
legacyImportSource: {
importedAt: new Date(),
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
triggeringPuuid: job.puuid,
jobId: job.parentJobId,
},
prizes: [],
notices: [],
schedule: [],
};
const match = await MatchModel.create(matchPayload);
await this.matchStatsService.buildAndUpsert(match._id);
return 'imported';
},
legacyImported: true,
legacyImportSource: {
importedAt: new Date(),
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
triggeringPuuid: job.puuid,
jobId: job.parentJobId,
},
prizes: [],
notices: [],
schedule: [],
};
const match = await MatchModel.create(matchPayload);
await this.matchStatsService.buildAndUpsert(match._id);
return 'imported';
);
if (importOutcome === null) return 'duplicate';
return importOutcome;
} catch (error) {
await this.recordError(job.parentJobId, {
puuid: job.puuid,
@@ -576,8 +595,6 @@ export class LegacyImportService {
reason: error instanceof Error ? error.message : String(error),
});
return 'error';
} finally {
await this.redisService.delete(lockKey);
}
}
@@ -626,6 +643,7 @@ export class LegacyImportService {
jobId: job.parentJobId,
}).exec();
if (!current || current.state === 'cancelled') return;
if (current.completedMemberJobIds?.includes(job.jobId)) return;
const average =
current.averageMemberSeconds === undefined
@@ -634,14 +652,14 @@ export class LegacyImportService {
current.averageMemberSeconds * 0.7 + stats.elapsedSeconds * 0.3,
);
const nextProcessed = current.processedMembers + 1;
const completed = nextProcessed >= current.totalMembers;
await LegacyImportJobModel.updateOne(
{ jobId: job.parentJobId },
const updateResult = await LegacyImportJobModel.updateOne(
{
jobId: job.parentJobId,
completedMemberJobIds: { $ne: job.jobId },
state: { $ne: 'cancelled' },
},
{
$set: {
state: completed ? 'completed' : current.state,
completedAt: completed ? new Date() : current.completedAt,
averageMemberSeconds: average,
},
$inc: {
@@ -651,14 +669,49 @@ export class LegacyImportService {
totalSkippedDuplicate: stats.duplicates,
totalErrors: stats.errors,
},
$addToSet: {
completedMemberJobIds: job.jobId,
},
},
).exec();
if (updateResult.modifiedCount === 0) return;
const updated = await LegacyImportJobModel.findOne({
jobId: job.parentJobId,
})
.select({ processedMembers: 1, totalMembers: 1, state: 1, completedAt: 1 })
.lean()
.exec();
const completed = Boolean(
updated &&
updated.state !== 'cancelled' &&
updated.processedMembers >= updated.totalMembers,
);
if (completed) {
await LegacyImportJobModel.updateOne(
{
jobId: job.parentJobId,
state: { $ne: 'cancelled' },
},
{
$set: {
state: 'completed',
completedAt: updated?.completedAt ?? new Date(),
},
},
).exec();
}
if (completed) {
await Promise.all([
this.redisService.delete(
this.communityLockKey(new mongoose.Types.ObjectId(job.communityId)),
),
this.distributedLockService.releaseLockIfOwned({
lockKey: this.communityLockKey(
new mongoose.Types.ObjectId(job.communityId),
),
lockOwnershipToken: job.parentJobId,
lockExpirationDurationMilliseconds:
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
}),
this.communityStatsService.invalidateCommunity(
new mongoose.Types.ObjectId(job.communityId),
),
@@ -680,6 +733,20 @@ export class LegacyImportService {
return legacyJob;
}
private async releaseDistributedLockSafely(
lockOwnership: DistributedLockOwnership,
): Promise<void> {
await this.distributedLockService
.releaseLockIfOwned(lockOwnership)
.catch((error: unknown) => {
this.logger.warn(
`Failed to release legacy import lock ${lockOwnership.lockKey}: ${
error instanceof Error ? error.message : error
}`,
);
});
}
private async isCancelled(parentJobId: string): Promise<boolean> {
const LegacyImportJobModel =
await this.connectionService.getModel('legacyImportJob');