fix: harden multi-instance scheduler and worker execution

This commit is contained in:
dahyeon committed 2026-06-27 13:19:43 -07:00
1 parent df7dcd3a9c
commit 60a006008a
23 files changed
+1442 -299

No files matched your search

@@ -4,14 +4,14 @@ import type {
RiotMatch as StoredRiotMatch,
} from '../global/database/schema';
import { ConnectionService } from '../global/database/connection.service';
import { RedisService } from '../global/redis/redis.service';
import { DistributedLockService } from '../global/redis/distributed-lock.service';
import type { RiotRegionalRoute } from './riot.types';
import { RiotMatchService } from './riot-match.service';
export const RIOT_RECENT_MATCH_LIMIT = 20;
export const RIOT_SOLO_QUEUE_ID = 420;
const MATCH_ENRICHMENT_LOCK_TTL_SECONDS = 60;
const MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
const MATCH_DETAIL_CONCURRENCY = 2;
const DEFAULT_REGION: RiotRegionalRoute = 'asia';
@@ -22,7 +22,7 @@ export class RiotRecentMatchesService {
constructor(
private readonly connectionService: ConnectionService,
private readonly matchService: RiotMatchService,
private readonly redisService: RedisService,
private readonly distributedLockService: DistributedLockService,
) {}
public async syncRecentMatches(params: {
@@ -34,83 +34,84 @@ export class RiotRecentMatchesService {
const region = params.region ?? DEFAULT_REGION;
const limit = params.limit ?? RIOT_RECENT_MATCH_LIMIT;
const queueId = params.queueId ?? RIOT_SOLO_QUEUE_ID;
const lockKey = `riot:enrich:matches:${params.puuid}`;
const lockValue = `${process.pid}:${Date.now()}`;
const acquired = await this.redisService.setStringIfAbsent(
lockKey,
lockValue,
MATCH_ENRICHMENT_LOCK_TTL_SECONDS,
);
if (!acquired) {
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
try {
const matchIds = await this.matchService.getMatchIdsByPuuid(params.puuid, {
region,
queue: queueId,
count: limit,
});
if (matchIds.length === 0) {
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
const RiotMatchModel = await this.connectionService.getModel('riotMatch');
const existingMatches = await RiotMatchModel.find({
matchId: { $in: matchIds },
})
.select({ matchId: 1 })
.lean()
.exec();
const existingMatchIds = new Set(existingMatches.map((match) => match.matchId));
const missingMatchIds = matchIds.filter((matchId) => !existingMatchIds.has(matchId));
await runWithConcurrency(missingMatchIds, MATCH_DETAIL_CONCURRENCY, async (matchId) => {
try {
const dto = await this.matchService.getMatch(matchId, { region });
if (dto.info.queueId !== queueId) {
this.logger.warn(
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
);
return;
}
const payload: Partial<StoredRiotMatch> = {
matchId: dto.metadata.matchId,
gameCreation: dto.info.gameCreation,
gameDuration: dto.info.gameDuration,
gameVersion: dto.info.gameVersion,
queueId: dto.info.queueId,
participantPuuids: dto.metadata.participants,
raw: dto as unknown as Record<string, unknown>,
fetchedAt: new Date(),
};
await RiotMatchModel.updateOne(
{ matchId: dto.metadata.matchId },
{ $setOnInsert: payload },
{ upsert: true },
).exec();
} catch (error: unknown) {
this.logger.warn(
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
error instanceof Error ? error.message : error
}`,
);
}
});
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
} finally {
await this.redisService.delete(lockKey).catch((error: unknown) => {
this.logger.warn(
`Failed to release recent matches lock for ${params.puuid}: ${
(error as Error)?.message ?? error
}`,
const result = await this.distributedLockService.runWithLock(
{
lockKey: `distributed-lock:riot-recent-matches:sync:puuid:${params.puuid}`,
lockExpirationDurationMilliseconds:
MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS,
},
async () => {
const matchIds = await this.matchService.getMatchIdsByPuuid(
params.puuid,
{
region,
queue: queueId,
count: limit,
},
);
});
}
if (matchIds.length === 0) {
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
const RiotMatchModel =
await this.connectionService.getModel('riotMatch');
const existingMatches = await RiotMatchModel.find({
matchId: { $in: matchIds },
})
.select({ matchId: 1 })
.lean()
.exec();
const existingMatchIds = new Set(
existingMatches.map((match) => match.matchId),
);
const missingMatchIds = matchIds.filter(
(matchId) => !existingMatchIds.has(matchId),
);
await runWithConcurrency(
missingMatchIds,
MATCH_DETAIL_CONCURRENCY,
async (matchId) => {
try {
const dto = await this.matchService.getMatch(matchId, { region });
if (dto.info.queueId !== queueId) {
this.logger.warn(
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
);
return;
}
const payload: Partial<StoredRiotMatch> = {
matchId: dto.metadata.matchId,
gameCreation: dto.info.gameCreation,
gameDuration: dto.info.gameDuration,
gameVersion: dto.info.gameVersion,
queueId: dto.info.queueId,
participantPuuids: dto.metadata.participants,
raw: dto as unknown as Record<string, unknown>,
fetchedAt: new Date(),
};
await RiotMatchModel.updateOne(
{ matchId: dto.metadata.matchId },
{ $setOnInsert: payload },
{ upsert: true },
).exec();
} catch (error: unknown) {
this.logger.warn(
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
error instanceof Error ? error.message : error
}`,
);
}
},
);
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
},
);
return result ?? this.loadFromDb({ puuid: params.puuid, limit, queueId });
}
public async ensureFreshForQueue(params: {