fix: harden multi-instance scheduler and worker execution
This commit is contained in:
1 parent
df7dcd3a9c
commit
60a006008a
23 files changed
+1442
-299
No files matched your search
@@ -4,14 +4,14 @@ import type {
|
||||
RiotMatch as StoredRiotMatch,
|
||||
} from '../global/database/schema';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import type { RiotRegionalRoute } from './riot.types';
|
||||
import { RiotMatchService } from './riot-match.service';
|
||||
|
||||
export const RIOT_RECENT_MATCH_LIMIT = 20;
|
||||
export const RIOT_SOLO_QUEUE_ID = 420;
|
||||
|
||||
const MATCH_ENRICHMENT_LOCK_TTL_SECONDS = 60;
|
||||
const MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS = 60_000;
|
||||
const MATCH_DETAIL_CONCURRENCY = 2;
|
||||
const DEFAULT_REGION: RiotRegionalRoute = 'asia';
|
||||
|
||||
@@ -22,7 +22,7 @@ export class RiotRecentMatchesService {
|
||||
constructor(
|
||||
private readonly connectionService: ConnectionService,
|
||||
private readonly matchService: RiotMatchService,
|
||||
private readonly redisService: RedisService,
|
||||
private readonly distributedLockService: DistributedLockService,
|
||||
) {}
|
||||
|
||||
public async syncRecentMatches(params: {
|
||||
@@ -34,83 +34,84 @@ export class RiotRecentMatchesService {
|
||||
const region = params.region ?? DEFAULT_REGION;
|
||||
const limit = params.limit ?? RIOT_RECENT_MATCH_LIMIT;
|
||||
const queueId = params.queueId ?? RIOT_SOLO_QUEUE_ID;
|
||||
const lockKey = `riot:enrich:matches:${params.puuid}`;
|
||||
const lockValue = `${process.pid}:${Date.now()}`;
|
||||
|
||||
const acquired = await this.redisService.setStringIfAbsent(
|
||||
lockKey,
|
||||
lockValue,
|
||||
MATCH_ENRICHMENT_LOCK_TTL_SECONDS,
|
||||
);
|
||||
if (!acquired) {
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
try {
|
||||
const matchIds = await this.matchService.getMatchIdsByPuuid(params.puuid, {
|
||||
region,
|
||||
queue: queueId,
|
||||
count: limit,
|
||||
});
|
||||
if (matchIds.length === 0) {
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
const RiotMatchModel = await this.connectionService.getModel('riotMatch');
|
||||
const existingMatches = await RiotMatchModel.find({
|
||||
matchId: { $in: matchIds },
|
||||
})
|
||||
.select({ matchId: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
const existingMatchIds = new Set(existingMatches.map((match) => match.matchId));
|
||||
const missingMatchIds = matchIds.filter((matchId) => !existingMatchIds.has(matchId));
|
||||
|
||||
await runWithConcurrency(missingMatchIds, MATCH_DETAIL_CONCURRENCY, async (matchId) => {
|
||||
try {
|
||||
const dto = await this.matchService.getMatch(matchId, { region });
|
||||
if (dto.info.queueId !== queueId) {
|
||||
this.logger.warn(
|
||||
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: Partial<StoredRiotMatch> = {
|
||||
matchId: dto.metadata.matchId,
|
||||
gameCreation: dto.info.gameCreation,
|
||||
gameDuration: dto.info.gameDuration,
|
||||
gameVersion: dto.info.gameVersion,
|
||||
queueId: dto.info.queueId,
|
||||
participantPuuids: dto.metadata.participants,
|
||||
raw: dto as unknown as Record<string, unknown>,
|
||||
fetchedAt: new Date(),
|
||||
};
|
||||
|
||||
await RiotMatchModel.updateOne(
|
||||
{ matchId: dto.metadata.matchId },
|
||||
{ $setOnInsert: payload },
|
||||
{ upsert: true },
|
||||
).exec();
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn(
|
||||
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
} finally {
|
||||
await this.redisService.delete(lockKey).catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Failed to release recent matches lock for ${params.puuid}: ${
|
||||
(error as Error)?.message ?? error
|
||||
}`,
|
||||
const result = await this.distributedLockService.runWithLock(
|
||||
{
|
||||
lockKey: `distributed-lock:riot-recent-matches:sync:puuid:${params.puuid}`,
|
||||
lockExpirationDurationMilliseconds:
|
||||
MATCH_ENRICHMENT_LOCK_EXPIRATION_DURATION_MILLISECONDS,
|
||||
},
|
||||
async () => {
|
||||
const matchIds = await this.matchService.getMatchIdsByPuuid(
|
||||
params.puuid,
|
||||
{
|
||||
region,
|
||||
queue: queueId,
|
||||
count: limit,
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
if (matchIds.length === 0) {
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
const RiotMatchModel =
|
||||
await this.connectionService.getModel('riotMatch');
|
||||
const existingMatches = await RiotMatchModel.find({
|
||||
matchId: { $in: matchIds },
|
||||
})
|
||||
.select({ matchId: 1 })
|
||||
.lean()
|
||||
.exec();
|
||||
const existingMatchIds = new Set(
|
||||
existingMatches.map((match) => match.matchId),
|
||||
);
|
||||
const missingMatchIds = matchIds.filter(
|
||||
(matchId) => !existingMatchIds.has(matchId),
|
||||
);
|
||||
|
||||
await runWithConcurrency(
|
||||
missingMatchIds,
|
||||
MATCH_DETAIL_CONCURRENCY,
|
||||
async (matchId) => {
|
||||
try {
|
||||
const dto = await this.matchService.getMatch(matchId, { region });
|
||||
if (dto.info.queueId !== queueId) {
|
||||
this.logger.warn(
|
||||
`Skipping unexpected queue ${dto.info.queueId} for ${matchId} while syncing ${params.puuid}.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: Partial<StoredRiotMatch> = {
|
||||
matchId: dto.metadata.matchId,
|
||||
gameCreation: dto.info.gameCreation,
|
||||
gameDuration: dto.info.gameDuration,
|
||||
gameVersion: dto.info.gameVersion,
|
||||
queueId: dto.info.queueId,
|
||||
participantPuuids: dto.metadata.participants,
|
||||
raw: dto as unknown as Record<string, unknown>,
|
||||
fetchedAt: new Date(),
|
||||
};
|
||||
|
||||
await RiotMatchModel.updateOne(
|
||||
{ matchId: dto.metadata.matchId },
|
||||
{ $setOnInsert: payload },
|
||||
{ upsert: true },
|
||||
).exec();
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn(
|
||||
`Skipping Riot match ${matchId} while syncing ${params.puuid}: ${
|
||||
error instanceof Error ? error.message : error
|
||||
}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
return this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
},
|
||||
);
|
||||
|
||||
return result ?? this.loadFromDb({ puuid: params.puuid, limit, queueId });
|
||||
}
|
||||
|
||||
public async ensureFreshForQueue(params: {
|
||||
|
||||
Reference in new issue
Block a user