chore: add api new relic observability

This commit is contained in:
dahyeon committed 2026-06-26 01:22:55 -07:00
1 parent d2fc126fed
commit 6dbc643c44
9 files changed
+1125 -21

No files matched your search

@@ -29,6 +29,7 @@ import {
AUCTION_DURABILITY_RETRY_QUEUE_KEY,
buildAuctionRedisKeys,
} from './redis/auction-redis.keys';
import { recordNewRelicCustomEvent } from '../global/observability/new-relic';
const AUCTION_DURABILITY_TICK_INTERVAL_MS = 1000;
const AUCTION_DURABILITY_BATCH_LIMIT = 20;
const AUCTION_DURABILITY_RETRY_DELAY_MS = 5000;
@@ -167,6 +168,21 @@ export class AuctionDurabilityScheduler
}
await this.applyDurabilityPayload(payload);
applied += 1;
// 영속 지연(lag): Redis 커밋 → Mongo 영속까지 걸린 시간 + 이번 배치 백로그.
// 사용자가 본 "되돌리기/이력이 2~3 뒤처짐"을 수치화한다(지연이 클수록 undo
// 패널이 stale 시퀀스를 보여 엉뚱한 스냅샷 복원 가능성↑).
recordNewRelicCustomEvent('AuctionServerDurability', {
matchId: typeof payload.matchId === 'string' ? payload.matchId : '',
auctionStateTransitionId: payload.auctionStateTransitionId,
auctionActionType: payload.auctionActionType,
revision: payload.revision,
auctionHistorySequence: payload.auctionHistorySequence,
persistLatencyMS:
typeof payload.committedAtRedisTimestampMS === 'number'
? currentRedisTimestampMS - payload.committedAtRedisTimestampMS
: -1,
batchBacklog: payloadKeys.length,
});
await this.clearDurabilityPayload(payloadKey);
} catch (error) {
failed += 1;
@@ -40,6 +40,7 @@ import {
import { AuctionRedisService } from './redis/auction-redis.service';
import { AuctionRedisStateReaderService } from './redis/auction-redis-state-reader.service';
import { shouldHydrateRedisAuctionState } from './redis/auction-redis.state';
import { recordNewRelicCustomEvent } from '../global/observability/new-relic';
const AUCTION_MAX_TEAM_SIZE = 5;
const AUCTION_MIN_BID_POINTS = 10;
@@ -909,6 +910,33 @@ export class AuctionService {
return this.resolveReplayedRedisActionRequest(matchId, actionRequest);
}
// 중복 멤버 버그 스모킹건: 낙찰 직전, 이 선수가 이미 어느 팀에 들어있는지.
// (markRedisSold는 검사 없이 append하므로 true면 그대로 중복이 만들어진다)
const soldUserId = String(currentPlayer.userId);
const existingTeamIndex = structureInput.auctionTeams.find((entry) =>
(entry.members ?? []).some(
(member) => String(member.userId) === soldUserId,
),
)?.teamIndex;
recordNewRelicCustomEvent('AuctionServerSold', {
matchId,
auctionActionRequestId: params.auctionActionRequestId ?? null,
auctionLifecycleId: this.stringFromState(state.auctionLifecycleId),
revision: this.numberFromState(state.revision),
currentAuctionPlayerOrderIndex:
structureInput.currentAuctionPlayerOrderIndex,
soldUserId,
winningTeamIndex,
currentBidPointAmount,
playerAlreadyInTeam: existingTeamIndex !== undefined,
existingTeamIndex:
existingTeamIndex !== undefined ? Number(existingTeamIndex) : -1,
totalAssignedMemberCount: structureInput.auctionTeams.reduce(
(sum, entry) => sum + (entry.members?.length ?? 0),
0,
),
});
const preferredPrimary = currentPlayer.preferredPositions?.[0]?.position;
team.members = [
...(team.members ?? []),
@@ -1721,6 +1749,27 @@ export class AuctionService {
actionRequest: params.actionRequest?.actionRequest,
},
);
// 모든 redis 전환(sold/undo/next/open-bidding/complete)의 백본 타임라인.
// auctionStateTransitionId = `${action}:${lifecycleId}:${auctionActionRequestId}`
// 라서 클라의 auctionActionRequestId와 조인된다.
recordNewRelicCustomEvent('AuctionServerTransition', {
matchId,
action: params.action,
auctionStateTransitionId,
auctionLifecycleId,
expectedRevision,
newRevision: 'rejected' in result ? -1 : expectedRevision + 1,
auctionHistorySequence,
currentAuctionPlayerOrderIndex:
params.structureInput.currentAuctionPlayerOrderIndex,
totalAssignedMemberCount: params.structureInput.auctionTeams.reduce(
(sum, entry) => sum + (entry.members?.length ?? 0),
0,
),
rejected: 'rejected' in result,
rejectionCode: 'rejected' in result ? result.code : null,
});
if ('rejected' in result) {
// 게이트웨이가 설명적 거부코드로 변환할 수 있도록 내부 code(예: CONFLICT)를
// 구조화해 던진다(message만 던지면 code 자리에 풀 문장이 들어가 매핑이 어렵다).
@@ -1835,6 +1884,33 @@ export class AuctionService {
0,
);
// undo가 "어느 시점으로" 복원하는지 + 복원된 팀 구성을 기록한다. 라이브 revision과
// 복원 스냅샷 revision의 간격(restoredRevisionLag)이 사용자가 본 "2~3 뒤처짐".
// 이 이벤트 직후의 AuctionServerSold가 playerAlreadyInTeam:true면 "undo가 선수를
// 안 지운 채 재낙찰 → 중복"이 한 타임라인으로 증명된다.
const liveRevisionBeforeUndo = this.numberFromState(state.revision);
const restoredRevision = this.numberFromState(params.snapshot.revision);
recordNewRelicCustomEvent('AuctionServerUndo', {
matchId,
auctionActionRequestId:
params.actionRequestParams?.auctionActionRequestId ?? null,
auctionLifecycleId: this.stringFromState(state.auctionLifecycleId),
targetHistorySequence: params.targetHistorySequence,
liveRevisionBeforeUndo,
restoredRevision,
restoredRevisionLag: liveRevisionBeforeUndo - restoredRevision,
restoredHistorySequence: this.numberFromState(
params.snapshot.currentAuctionHistorySequence,
),
restoredPhase: phase,
restoredOrderIndex: structureInput.currentAuctionPlayerOrderIndex,
restoredAssignedMemberCount: structureInput.auctionTeams.reduce(
(sum, entry) => sum + (entry.members?.length ?? 0),
0,
),
undoingCompletedAuction: params.undoingCompletedAuction,
});
return this.applyRedisStructureTransition({
action: 'undo',
actionRequest,
@@ -0,0 +1,37 @@
import newrelicAgent from 'newrelic';
export type NewRelicAttributeValue = string | number | boolean | null;
// newrelic 기본 export는 타입이 느슨해 unsafe-call로 잡힌다. 우리가 쓰는 부분만
// 좁게 타입을 고정한다(에이전트는 -r newrelic로 이미 preload됨).
interface NewRelicCustomEventApi {
recordCustomEvent(
eventType: string,
attributes: Record<string, NewRelicAttributeValue>,
): void;
}
const agent = newrelicAgent as unknown as NewRelicCustomEventApi;
/**
* 서버 커스텀 이벤트를 New Relic으로 보낸다(클라의 addPageAction 대응).
*
* - 에이전트는 `-r newrelic`로 이미 preload되어 있으므로 여기서 import만 하면 된다.
* - `recordCustomEvent`는 활성 트랜잭션이 없어도 동작하므로 durability 스케줄러
* tick이나 socket.io 핸들러처럼 HTTP 컨텍스트 밖에서도 그대로 호출할 수 있다.
* - 평탄한 스칼라만 받는다(중첩 객체/배열 불가). undefined 속성은 제거한다.
* - 에이전트 비활성/테스트 환경에서도 호출부가 깨지지 않도록 예외를 삼킨다.
*/
export function recordNewRelicCustomEvent(
eventType: string,
attributes: Record<string, NewRelicAttributeValue | undefined>,
): void {
try {
const flat: Record<string, NewRelicAttributeValue> = {};
for (const [key, value] of Object.entries(attributes)) {
if (value !== undefined) flat[key] = value;
}
agent.recordCustomEvent(eventType, flat);
} catch {
// 관측성 실패가 비즈니스 로직을 막아선 안 된다.
}
}
+90
View File
@@ -0,0 +1,90 @@
import { createRequire } from 'node:module';
const CONFIG_PATH = '../newrelic.cjs';
const requireConfig = createRequire(__filename);
interface NewRelicConfig {
app_name: string[];
license_key: string;
agent_enabled: boolean;
ai_monitoring: {
enabled: boolean;
};
custom_insights_events: {
max_samples_stored: number;
};
span_events: {
max_samples_stored: number;
};
}
interface NewRelicConfigModule {
config: NewRelicConfig;
}
function loadConfig(): NewRelicConfig {
jest.resetModules();
const module = requireConfig(CONFIG_PATH) as unknown as NewRelicConfigModule;
return module.config;
}
describe('New Relic config', () => {
let originalEnv: NodeJS.ProcessEnv;
beforeEach(() => {
originalEnv = { ...process.env };
delete process.env.NEW_RELIC_ENABLED;
delete process.env.NEW_RELIC_LICENSE_KEY;
delete process.env.NEW_RELIC_APP_NAME;
delete process.env.NEW_RELIC_AI_MONITORING_ENABLED;
delete process.env.NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED;
delete process.env.NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED;
jest.resetModules();
});
afterEach(() => {
process.env = originalEnv;
jest.resetModules();
});
it('uses safe defaults and stays disabled without a license key', () => {
const config = loadConfig();
expect(config.app_name).toEqual(['lolympus']);
expect(config.agent_enabled).toBe(false);
expect(config.ai_monitoring.enabled).toBe(true);
expect(config.custom_insights_events.max_samples_stored).toBe(100000);
expect(config.span_events.max_samples_stored).toBe(10000);
});
it('enables the agent when a license key exists and honors runtime overrides', () => {
process.env.NEW_RELIC_LICENSE_KEY = 'test-license';
process.env.NEW_RELIC_APP_NAME = 'lolympus-api-test';
process.env.NEW_RELIC_AI_MONITORING_ENABLED = 'false';
process.env.NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED = '5000';
process.env.NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED = '4000';
const config = loadConfig();
expect(config.app_name).toEqual(['lolympus-api-test']);
expect(config.license_key).toBe('test-license');
expect(config.agent_enabled).toBe(true);
expect(config.ai_monitoring.enabled).toBe(false);
expect(config.custom_insights_events.max_samples_stored).toBe(5000);
expect(config.span_events.max_samples_stored).toBe(4000);
});
it('normalizes shorthand sample sizes before New Relic applies env overrides', () => {
process.env.NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED = '100k';
process.env.NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED = '10k';
const config = loadConfig();
expect(config.custom_insights_events.max_samples_stored).toBe(100000);
expect(config.span_events.max_samples_stored).toBe(10000);
expect(
process.env.NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED,
).toBe('100000');
expect(process.env.NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED).toBe('10000');
});
});