fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -10,6 +10,10 @@ import mongoose from 'mongoose';
|
||||
import { MatchStatsService } from '../community-stats/match-stats.service';
|
||||
import { CommunityStatsService } from '../community-stats/community-stats.service';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import {
|
||||
type DistributedLockOwnership,
|
||||
DistributedLockService,
|
||||
} from '../global/redis/distributed-lock.service';
|
||||
import type {
|
||||
LegacyImportJobDocument,
|
||||
Match,
|
||||
@@ -57,6 +61,7 @@ export class LegacyImportService {
|
||||
private readonly riotMatchService: RiotMatchService,
|
||||
private readonly communityStatsService: CommunityStatsService,
|
||||
private readonly matchStatsService: MatchStatsService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public async previewMembers(params: {
|
||||
@@ -80,12 +85,14 @@ export class LegacyImportService {
|
||||
}): Promise<StartLegacyImportResponse> {
|
||||
const lockKey = this.communityLockKey(params.communityId);
|
||||
const jobId = randomUUID();
|
||||
const acquired = await this.redisService.setStringIfAbsent(
|
||||
lockKey,
|
||||
jobId,
|
||||
LEGACY_IMPORT_LOCK_TTL_SECONDS,
|
||||
);
|
||||
if (!acquired) {
|
||||
const communityImportLockOwnership =
|
||||
await this.distributedLockService.acquireLock({
|
||||
lockKey,
|
||||
lockOwnershipToken: jobId,
|
||||
lockExpirationDurationMilliseconds:
|
||||
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
});
|
||||
if (!communityImportLockOwnership) {
|
||||
throw new ConflictException('Legacy import is already running.');
|
||||
}
|
||||
|
||||
@@ -133,6 +140,7 @@ export class LegacyImportService {
|
||||
totalSkippedDuplicate: 0,
|
||||
totalErrors: 0,
|
||||
averageMemberSeconds: LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
||||
completedMemberJobIds: [],
|
||||
});
|
||||
|
||||
for (const window of windows) {
|
||||
@@ -160,7 +168,7 @@ export class LegacyImportService {
|
||||
{ jobId },
|
||||
{ $set: { state: 'completed', completedAt: new Date() } },
|
||||
).exec();
|
||||
await this.redisService.delete(lockKey);
|
||||
await this.releaseDistributedLockSafely(communityImportLockOwnership);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -169,7 +177,7 @@ export class LegacyImportService {
|
||||
etaSeconds: windows.length * LEGACY_IMPORT_MEMBER_AVG_SECONDS,
|
||||
};
|
||||
} catch (error) {
|
||||
await this.redisService.delete(lockKey);
|
||||
await this.releaseDistributedLockSafely(communityImportLockOwnership);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -220,7 +228,11 @@ export class LegacyImportService {
|
||||
).exec();
|
||||
if (!job)
|
||||
throw new NotFoundException('Running legacy import job not found.');
|
||||
await this.redisService.delete(this.communityLockKey(params.communityId));
|
||||
await this.distributedLockService.releaseLockIfOwned({
|
||||
lockKey: this.communityLockKey(params.communityId),
|
||||
lockOwnershipToken: job.jobId,
|
||||
lockExpirationDurationMilliseconds: LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
});
|
||||
return job;
|
||||
}
|
||||
|
||||
@@ -228,6 +240,7 @@ export class LegacyImportService {
|
||||
const started = Date.now();
|
||||
const legacyJob = await this.getRunnableJob(job);
|
||||
if (!legacyJob) return;
|
||||
if (legacyJob.completedMemberJobIds?.includes(job.jobId)) return;
|
||||
|
||||
let discovered = 0;
|
||||
let imported = 0;
|
||||
@@ -500,75 +513,81 @@ export class LegacyImportService {
|
||||
userByPuuid: UserByPuuidMap,
|
||||
): Promise<ImportOutcome> {
|
||||
const lockKey = `legacy-import:match:${riotMatchId}`;
|
||||
const acquired = await this.redisService.setStringIfAbsent(
|
||||
lockKey,
|
||||
job.parentJobId,
|
||||
MATCH_IMPORT_LOCK_TTL_SECONDS,
|
||||
);
|
||||
if (!acquired) return 'duplicate';
|
||||
|
||||
try {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const existing = await MatchModel.exists({
|
||||
'result.games.riotMatchId': riotMatchId,
|
||||
});
|
||||
if (existing) return 'duplicate';
|
||||
const importOutcome = await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey,
|
||||
lockExpirationDurationMilliseconds:
|
||||
MATCH_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
},
|
||||
async () => {
|
||||
const MatchModel = await this.connectionService.getModel('match');
|
||||
const existing = await MatchModel.exists({
|
||||
'result.games.riotMatchId': riotMatchId,
|
||||
});
|
||||
if (existing) return 'duplicate';
|
||||
|
||||
const detail = await this.ensureRiotMatchPersisted(
|
||||
riotMatchId,
|
||||
job.region,
|
||||
);
|
||||
if (!detail.info.tournamentCode) return 'skipped';
|
||||
const counts = countKnownParticipants(detail, userByPuuid);
|
||||
if (counts.total === 0) return 'skipped';
|
||||
const detail = await this.ensureRiotMatchPersisted(
|
||||
riotMatchId,
|
||||
job.region,
|
||||
);
|
||||
if (!detail.info.tournamentCode) return 'skipped';
|
||||
const counts = countKnownParticipants(detail, userByPuuid);
|
||||
if (counts.total === 0) return 'skipped';
|
||||
|
||||
const gameResult = buildGameResult(detail, userByPuuid, 1);
|
||||
if (!gameResult || gameResult.participants.length === 0) return 'skipped';
|
||||
const gameResult = buildGameResult(detail, userByPuuid, 1);
|
||||
if (!gameResult || gameResult.participants.length === 0) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
const winnerTeamIndex = gameResult.winnerTeamIndex;
|
||||
const matchPayload: Partial<Match> = {
|
||||
communityId: new mongoose.Types.ObjectId(job.communityId),
|
||||
title: this.formatLegacyDate(detail.info.gameCreation),
|
||||
description: 'Legacy import (자동 생성)',
|
||||
mapId: detail.info.mapId ?? 11,
|
||||
status: 'finished',
|
||||
scheduledAt: new Date(detail.info.gameCreation),
|
||||
maxParticipants: 10,
|
||||
gameCount: 1,
|
||||
gameMode: 'pair',
|
||||
tournamentCodeIds: [],
|
||||
teamBuiltAt: new Date(detail.info.gameCreation),
|
||||
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
participants: buildParticipantsSnapshot(detail, userByPuuid),
|
||||
teams: buildTeams(detail, userByPuuid),
|
||||
result: {
|
||||
winnerTeamIndex,
|
||||
riotMatchIds: [riotMatchId],
|
||||
games: [gameResult],
|
||||
seriesResults: [
|
||||
{
|
||||
teamAIndex: 1,
|
||||
teamBIndex: 2,
|
||||
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
|
||||
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
|
||||
const winnerTeamIndex = gameResult.winnerTeamIndex;
|
||||
const matchPayload: Partial<Match> = {
|
||||
communityId: new mongoose.Types.ObjectId(job.communityId),
|
||||
title: this.formatLegacyDate(detail.info.gameCreation),
|
||||
description: 'Legacy import (자동 생성)',
|
||||
mapId: detail.info.mapId ?? 11,
|
||||
status: 'finished',
|
||||
scheduledAt: new Date(detail.info.gameCreation),
|
||||
maxParticipants: 10,
|
||||
gameCount: 1,
|
||||
gameMode: 'pair',
|
||||
tournamentCodeIds: [],
|
||||
teamBuiltAt: new Date(detail.info.gameCreation),
|
||||
hostUserId: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
participants: buildParticipantsSnapshot(detail, userByPuuid),
|
||||
teams: buildTeams(detail, userByPuuid),
|
||||
result: {
|
||||
winnerTeamIndex,
|
||||
riotMatchIds: [riotMatchId],
|
||||
games: [gameResult],
|
||||
seriesResults: [
|
||||
{
|
||||
teamAIndex: 1,
|
||||
teamBIndex: 2,
|
||||
teamAWins: winnerTeamIndex === 1 ? 1 : 0,
|
||||
teamBWins: winnerTeamIndex === 2 ? 1 : 0,
|
||||
winnerTeamIndex,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
legacyImported: true,
|
||||
legacyImportSource: {
|
||||
importedAt: new Date(),
|
||||
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
triggeringPuuid: job.puuid,
|
||||
jobId: job.parentJobId,
|
||||
},
|
||||
prizes: [],
|
||||
notices: [],
|
||||
schedule: [],
|
||||
};
|
||||
const match = await MatchModel.create(matchPayload);
|
||||
await this.matchStatsService.buildAndUpsert(match._id);
|
||||
return 'imported';
|
||||
},
|
||||
legacyImported: true,
|
||||
legacyImportSource: {
|
||||
importedAt: new Date(),
|
||||
importedBy: new mongoose.Types.ObjectId(job.triggeredBy),
|
||||
triggeringPuuid: job.puuid,
|
||||
jobId: job.parentJobId,
|
||||
},
|
||||
prizes: [],
|
||||
notices: [],
|
||||
schedule: [],
|
||||
};
|
||||
const match = await MatchModel.create(matchPayload);
|
||||
await this.matchStatsService.buildAndUpsert(match._id);
|
||||
return 'imported';
|
||||
);
|
||||
if (importOutcome === null) return 'duplicate';
|
||||
return importOutcome;
|
||||
} catch (error) {
|
||||
await this.recordError(job.parentJobId, {
|
||||
puuid: job.puuid,
|
||||
@@ -576,8 +595,6 @@ export class LegacyImportService {
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return 'error';
|
||||
} finally {
|
||||
await this.redisService.delete(lockKey);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,6 +643,7 @@ export class LegacyImportService {
|
||||
jobId: job.parentJobId,
|
||||
}).exec();
|
||||
if (!current || current.state === 'cancelled') return;
|
||||
if (current.completedMemberJobIds?.includes(job.jobId)) return;
|
||||
|
||||
const average =
|
||||
current.averageMemberSeconds === undefined
|
||||
@@ -634,14 +652,14 @@ export class LegacyImportService {
|
||||
current.averageMemberSeconds * 0.7 + stats.elapsedSeconds * 0.3,
|
||||
);
|
||||
|
||||
const nextProcessed = current.processedMembers + 1;
|
||||
const completed = nextProcessed >= current.totalMembers;
|
||||
await LegacyImportJobModel.updateOne(
|
||||
{ jobId: job.parentJobId },
|
||||
const updateResult = await LegacyImportJobModel.updateOne(
|
||||
{
|
||||
jobId: job.parentJobId,
|
||||
completedMemberJobIds: { $ne: job.jobId },
|
||||
state: { $ne: 'cancelled' },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
state: completed ? 'completed' : current.state,
|
||||
completedAt: completed ? new Date() : current.completedAt,
|
||||
averageMemberSeconds: average,
|
||||
},
|
||||
$inc: {
|
||||
@@ -651,14 +669,49 @@ export class LegacyImportService {
|
||||
totalSkippedDuplicate: stats.duplicates,
|
||||
totalErrors: stats.errors,
|
||||
},
|
||||
$addToSet: {
|
||||
completedMemberJobIds: job.jobId,
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
if (updateResult.modifiedCount === 0) return;
|
||||
|
||||
const updated = await LegacyImportJobModel.findOne({
|
||||
jobId: job.parentJobId,
|
||||
})
|
||||
.select({ processedMembers: 1, totalMembers: 1, state: 1, completedAt: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
const completed = Boolean(
|
||||
updated &&
|
||||
updated.state !== 'cancelled' &&
|
||||
updated.processedMembers >= updated.totalMembers,
|
||||
);
|
||||
if (completed) {
|
||||
await LegacyImportJobModel.updateOne(
|
||||
{
|
||||
jobId: job.parentJobId,
|
||||
state: { $ne: 'cancelled' },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
state: 'completed',
|
||||
completedAt: updated?.completedAt ?? new Date(),
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
}
|
||||
|
||||
if (completed) {
|
||||
await Promise.all([
|
||||
this.redisService.delete(
|
||||
this.communityLockKey(new mongoose.Types.ObjectId(job.communityId)),
|
||||
),
|
||||
this.distributedLockService.releaseLockIfOwned({
|
||||
lockKey: this.communityLockKey(
|
||||
new mongoose.Types.ObjectId(job.communityId),
|
||||
),
|
||||
lockOwnershipToken: job.parentJobId,
|
||||
lockExpirationDurationMilliseconds:
|
||||
LEGACY_IMPORT_LOCK_TTL_SECONDS * 1000,
|
||||
}),
|
||||
this.communityStatsService.invalidateCommunity(
|
||||
new mongoose.Types.ObjectId(job.communityId),
|
||||
),
|
||||
@@ -680,6 +733,20 @@ export class LegacyImportService {
|
||||
return legacyJob;
|
||||
}
|
||||
|
||||
private async releaseDistributedLockSafely(
|
||||
lockOwnership: DistributedLockOwnership,
|
||||
): Promise<void> {
|
||||
await this.distributedLockService
|
||||
.releaseLockIfOwned(lockOwnership)
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release legacy import lock ${lockOwnership.lockKey}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async isCancelled(parentJobId: string): Promise<boolean> {
|
||||
const LegacyImportJobModel =
|
||||
await this.connectionService.getModel('legacyImportJob');
|
||||
|
||||
Reference in new issue
Block a user