feat: add API performance observability reporting
This commit is contained in:
1 parent
871a5fe78a
commit
88de2148ae
14 files changed
+2082
-1
No files matched your search
@@ -149,6 +149,12 @@ NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED=10000
|
|||||||
|
|
||||||
New Relic Node agent의 정수 환경변수는 `100k` 같은 축약형 대신 `100000`처럼 숫자 문자열을 사용합니다.
|
New Relic Node agent의 정수 환경변수는 `100k` 같은 축약형 대신 `100000`처럼 숫자 문자열을 사용합니다.
|
||||||
|
|
||||||
|
### API 성능 리포트 운영
|
||||||
|
|
||||||
|
API 성능 계측은 기본 비활성화이며 필요할 때 `API_PERF_OBSERVABILITY_ENABLED=true`로 켭니다. 전체 API route 목록은 `pnpm --filter api perf:routes`, 성능 리포트는 `pnpm --filter api perf:report`로 생성합니다. New Relic Transaction 실측 duration, New Relic custom event, local NDJSON 파일을 데이터 소스로 사용할 수 있습니다.
|
||||||
|
|
||||||
|
운영 절차와 해석 기준은 `docs/operations/api-performance-runbook.md`를 참고합니다.
|
||||||
|
|
||||||
카카오톡 기반 명령어 기능은 Dakao(커스텀 카카오톡 게이트웨이) 연동이 필요합니다. Dakao가 실행 중이 아니거나 NATS를 통해 API 서버와 연결되어 있지 않으면 일정 조회, 내전 참가, 참가 취소, 계정 등록 같은 카카오톡 명령어는 사용할 수 없습니다.
|
카카오톡 기반 명령어 기능은 Dakao(커스텀 카카오톡 게이트웨이) 연동이 필요합니다. Dakao가 실행 중이 아니거나 NATS를 통해 API 서버와 연결되어 있지 않으면 일정 조회, 내전 참가, 참가 취소, 계정 등록 같은 카카오톡 명령어는 사용할 수 없습니다.
|
||||||
|
|
||||||
## 커맨드
|
## 커맨드
|
||||||
@@ -158,6 +164,8 @@ New Relic Node agent의 정수 환경변수는 `100k` 같은 축약형 대신 `1
|
|||||||
| `pnpm start:web:dev` | 웹 앱 개발 서버 실행 |
|
| `pnpm start:web:dev` | 웹 앱 개발 서버 실행 |
|
||||||
| `pnpm start:api:dev` | API 서버 dev 환경 실행 |
|
| `pnpm start:api:dev` | API 서버 dev 환경 실행 |
|
||||||
| `pnpm --filter api test` | API 테스트 실행 |
|
| `pnpm --filter api test` | API 테스트 실행 |
|
||||||
|
| `pnpm --filter api perf:routes` | API 성능 리포트 대상 route 출력 |
|
||||||
|
| `pnpm --filter api perf:report` | API 성능 리포트 생성 |
|
||||||
| `pnpm --filter web build:staging` | 웹 staging 빌드 검증 |
|
| `pnpm --filter web build:staging` | 웹 staging 빌드 검증 |
|
||||||
| `pnpm deploy:web:staging` | Cloudflare Pages staging 배포 |
|
| `pnpm deploy:web:staging` | Cloudflare Pages staging 배포 |
|
||||||
| `pnpm deploy:web:prod` | Cloudflare Pages production 배포 |
|
| `pnpm deploy:web:prod` | Cloudflare Pages production 배포 |
|
||||||
|
|||||||
@@ -16,6 +16,8 @@
|
|||||||
"pm2:dev": "pm2 start npm --name lolympus-tournament-support --update-env -- run start:dev",
|
"pm2:dev": "pm2 start npm --name lolympus-tournament-support --update-env -- run start:dev",
|
||||||
"pm2:staging": "pm2 start npm --name lolympus-tournament-support --update-env -- run start:staging",
|
"pm2:staging": "pm2 start npm --name lolympus-tournament-support --update-env -- run start:staging",
|
||||||
"pm2:prod": "pm2 start npm --name lolympus-tournament-support --update-env -- run start:prod",
|
"pm2:prod": "pm2 start npm --name lolympus-tournament-support --update-env -- run start:prod",
|
||||||
|
"perf:report": "ts-node src/scripts/api-performance-report.ts",
|
||||||
|
"perf:routes": "ts-node src/scripts/api-performance-report.ts --routes-only",
|
||||||
"start:debug": "nest start --debug --watch",
|
"start:debug": "nest start --debug --watch",
|
||||||
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
||||||
"test": "jest",
|
"test": "jest",
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { ConfigModule } from '@nestjs/config';
|
import { ConfigModule } from '@nestjs/config';
|
||||||
|
import { APP_INTERCEPTOR } from '@nestjs/core';
|
||||||
import { AppController } from './app.controller';
|
import { AppController } from './app.controller';
|
||||||
import { AppService } from './app.service';
|
import { AppService } from './app.service';
|
||||||
|
import { ApiPerformanceInterceptor } from './modules/global/observability/api-performance.interceptor';
|
||||||
import { AuctionModule } from './modules/auction/auction.module';
|
import { AuctionModule } from './modules/auction/auction.module';
|
||||||
import { AuthModule } from './modules/auth/auth.module';
|
import { AuthModule } from './modules/auth/auth.module';
|
||||||
import { CommunityModule } from './modules/community/community.module';
|
import { CommunityModule } from './modules/community/community.module';
|
||||||
@@ -39,6 +41,12 @@ import { UserModule } from './modules/user/user.module';
|
|||||||
DakaoModule,
|
DakaoModule,
|
||||||
],
|
],
|
||||||
controllers: [AppController],
|
controllers: [AppController],
|
||||||
providers: [AppService],
|
providers: [
|
||||||
|
AppService,
|
||||||
|
{
|
||||||
|
provide: APP_INTERCEPTOR,
|
||||||
|
useClass: ApiPerformanceInterceptor,
|
||||||
|
},
|
||||||
|
],
|
||||||
})
|
})
|
||||||
export class AppModule {}
|
export class AppModule {}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { Injectable, Logger, Scope, OnModuleInit } from '@nestjs/common';
|
import { Injectable, Logger, Scope, OnModuleInit } from '@nestjs/common';
|
||||||
import mongoose from 'mongoose';
|
import mongoose from 'mongoose';
|
||||||
|
import { applyMongoosePerformancePlugin } from '../observability/mongoose-performance.plugin';
|
||||||
import { SecretService } from '../secret/secret.service';
|
import { SecretService } from '../secret/secret.service';
|
||||||
import { tournamentModelDefinitions, TournamentModelTypes } from './schema';
|
import { tournamentModelDefinitions, TournamentModelTypes } from './schema';
|
||||||
|
|
||||||
@@ -81,6 +82,7 @@ export class ConnectionService implements OnModuleInit {
|
|||||||
const connection = await this.getConnection();
|
const connection = await this.getConnection();
|
||||||
|
|
||||||
tournamentModelDefinitions.forEach((modelDefinition) => {
|
tournamentModelDefinitions.forEach((modelDefinition) => {
|
||||||
|
applyMongoosePerformancePlugin(modelDefinition.schema);
|
||||||
// 이미 생성된 경우 컴파일 스킵.
|
// 이미 생성된 경우 컴파일 스킵.
|
||||||
if (
|
if (
|
||||||
Object.prototype.hasOwnProperty.call(
|
Object.prototype.hasOwnProperty.call(
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import {
|
||||||
|
CallHandler,
|
||||||
|
ExecutionContext,
|
||||||
|
Injectable,
|
||||||
|
NestInterceptor,
|
||||||
|
} from '@nestjs/common';
|
||||||
|
import { Observable, catchError, finalize, throwError } from 'rxjs';
|
||||||
|
import { setNewRelicTransactionName } from './new-relic';
|
||||||
|
import {
|
||||||
|
buildApiRequestPerformanceEvent,
|
||||||
|
createApiPerformanceRequestContext,
|
||||||
|
isApiPerformanceObservabilityEnabled,
|
||||||
|
recordApiRequestPerformance,
|
||||||
|
runWithApiPerformanceContext,
|
||||||
|
} from './performance-observer';
|
||||||
|
|
||||||
|
interface RequestLike {
|
||||||
|
method?: string;
|
||||||
|
originalUrl?: string;
|
||||||
|
url?: string;
|
||||||
|
path?: string;
|
||||||
|
route?: {
|
||||||
|
path?: string | RegExp | Array<string | RegExp>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ResponseLike {
|
||||||
|
statusCode?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class ApiPerformanceInterceptor implements NestInterceptor {
|
||||||
|
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
|
||||||
|
if (!isApiPerformanceObservabilityEnabled()) {
|
||||||
|
return next.handle();
|
||||||
|
}
|
||||||
|
|
||||||
|
const http = context.switchToHttp();
|
||||||
|
const request = http.getRequest<RequestLike>();
|
||||||
|
const response = http.getResponse<ResponseLike>();
|
||||||
|
const method = request.method ?? 'UNKNOWN';
|
||||||
|
const route = resolveRouteTemplate(request);
|
||||||
|
const path = resolveRequestPath(request);
|
||||||
|
const requestContext = createApiPerformanceRequestContext({
|
||||||
|
method,
|
||||||
|
route,
|
||||||
|
path,
|
||||||
|
});
|
||||||
|
const startedAt = process.hrtime.bigint();
|
||||||
|
let thrownError: unknown;
|
||||||
|
|
||||||
|
setNewRelicTransactionName(`${method} ${route}`);
|
||||||
|
|
||||||
|
return runWithApiPerformanceContext(requestContext, () =>
|
||||||
|
next.handle().pipe(
|
||||||
|
catchError((error: unknown) => {
|
||||||
|
thrownError = error;
|
||||||
|
return throwError(() => error);
|
||||||
|
}),
|
||||||
|
finalize(() => {
|
||||||
|
const statusCode = response.statusCode ?? (thrownError ? 500 : 200);
|
||||||
|
const durationMs =
|
||||||
|
Number(process.hrtime.bigint() - startedAt) / 1_000_000;
|
||||||
|
recordApiRequestPerformance(
|
||||||
|
buildApiRequestPerformanceEvent({
|
||||||
|
context: requestContext,
|
||||||
|
statusCode,
|
||||||
|
durationMs,
|
||||||
|
isError: Boolean(thrownError) || statusCode >= 400,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveRouteTemplate(request: RequestLike): string {
|
||||||
|
const routePath = request.route?.path;
|
||||||
|
if (Array.isArray(routePath)) {
|
||||||
|
return normalizeRoute(routePath.map(String).join('|'));
|
||||||
|
}
|
||||||
|
if (routePath) return normalizeRoute(String(routePath));
|
||||||
|
return normalizeRoute(resolveRequestPath(request));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveRequestPath(request: RequestLike): string {
|
||||||
|
const rawPath = request.originalUrl ?? request.path ?? request.url ?? '/';
|
||||||
|
return rawPath.split('?')[0] || '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRoute(route: string): string {
|
||||||
|
const normalized = route.startsWith('/') ? route : `/${route}`;
|
||||||
|
return normalized.replace(/\/+/g, '/').replace(/\/$/, '') || '/';
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
import {
|
||||||
|
countHydratedDocuments,
|
||||||
|
countReturnedDocuments,
|
||||||
|
} from './mongoose-performance.plugin';
|
||||||
|
|
||||||
|
describe('Mongoose performance helpers', () => {
|
||||||
|
it('counts returned documents for common query result shapes', () => {
|
||||||
|
expect(countReturnedDocuments([{ id: 1 }, { id: 2 }])).toBe(2);
|
||||||
|
expect(countReturnedDocuments({ id: 1 })).toBe(1);
|
||||||
|
expect(countReturnedDocuments(null)).toBe(0);
|
||||||
|
expect(countReturnedDocuments(7)).toBe(7);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('counts hydrated documents separately from lean documents', () => {
|
||||||
|
const hydrated = { $__: {} };
|
||||||
|
const lean = { id: 1 };
|
||||||
|
|
||||||
|
expect(countHydratedDocuments([hydrated, lean, hydrated])).toBe(2);
|
||||||
|
expect(countHydratedDocuments(lean)).toBe(0);
|
||||||
|
expect(countHydratedDocuments(hydrated)).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { recordMongooseQueryPerformance } from './performance-observer';
|
||||||
|
|
||||||
|
const QUERY_PERFORMANCE_START = Symbol('queryPerformanceStart');
|
||||||
|
const QUERY_PERFORMANCE_RECORDED = Symbol('queryPerformanceRecorded');
|
||||||
|
const PERFORMANCE_PLUGIN_APPLIED = Symbol('performancePluginApplied');
|
||||||
|
|
||||||
|
const QUERY_OPERATIONS = [
|
||||||
|
'countDocuments',
|
||||||
|
'deleteMany',
|
||||||
|
'deleteOne',
|
||||||
|
'find',
|
||||||
|
'findOne',
|
||||||
|
'findOneAndDelete',
|
||||||
|
'findOneAndReplace',
|
||||||
|
'findOneAndUpdate',
|
||||||
|
'replaceOne',
|
||||||
|
'updateMany',
|
||||||
|
'updateOne',
|
||||||
|
] as const;
|
||||||
|
const QUERY_OPERATION_REGEX = new RegExp(`^(${QUERY_OPERATIONS.join('|')})$`);
|
||||||
|
|
||||||
|
type QueryWithPerformanceState = mongoose.Query<unknown, unknown> & {
|
||||||
|
[QUERY_PERFORMANCE_START]?: bigint;
|
||||||
|
[QUERY_PERFORMANCE_RECORDED]?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type AggregateWithPerformanceState = mongoose.Aggregate<unknown[]> & {
|
||||||
|
[QUERY_PERFORMANCE_START]?: bigint;
|
||||||
|
[QUERY_PERFORMANCE_RECORDED]?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type SchemaWithPerformanceMarker = mongoose.Schema & {
|
||||||
|
[PERFORMANCE_PLUGIN_APPLIED]?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function applyMongoosePerformancePlugin(schema: mongoose.Schema): void {
|
||||||
|
const markedSchema = schema as SchemaWithPerformanceMarker;
|
||||||
|
if (markedSchema[PERFORMANCE_PLUGIN_APPLIED]) return;
|
||||||
|
markedSchema[PERFORMANCE_PLUGIN_APPLIED] = true;
|
||||||
|
const hookableSchema = schema as any;
|
||||||
|
|
||||||
|
hookableSchema.pre(QUERY_OPERATION_REGEX, function () {
|
||||||
|
const query = this as QueryWithPerformanceState;
|
||||||
|
query[QUERY_PERFORMANCE_START] = process.hrtime.bigint();
|
||||||
|
query[QUERY_PERFORMANCE_RECORDED] = false;
|
||||||
|
});
|
||||||
|
|
||||||
|
hookableSchema.post(
|
||||||
|
QUERY_OPERATION_REGEX,
|
||||||
|
function (result: unknown) {
|
||||||
|
recordQueryPerformance(
|
||||||
|
this as unknown as QueryWithPerformanceState,
|
||||||
|
result,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
hookableSchema.post(
|
||||||
|
QUERY_OPERATION_REGEX,
|
||||||
|
function (error: unknown, _result: unknown, next: (error: unknown) => void) {
|
||||||
|
recordQueryPerformance(
|
||||||
|
this as unknown as QueryWithPerformanceState,
|
||||||
|
undefined,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
next(error);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
hookableSchema.pre('aggregate', function () {
|
||||||
|
const aggregate = this as AggregateWithPerformanceState;
|
||||||
|
aggregate[QUERY_PERFORMANCE_START] = process.hrtime.bigint();
|
||||||
|
aggregate[QUERY_PERFORMANCE_RECORDED] = false;
|
||||||
|
});
|
||||||
|
|
||||||
|
hookableSchema.post('aggregate', function (result: unknown) {
|
||||||
|
recordAggregatePerformance(this as AggregateWithPerformanceState, result);
|
||||||
|
});
|
||||||
|
|
||||||
|
hookableSchema.post(
|
||||||
|
'aggregate',
|
||||||
|
function (error: unknown, _result: unknown, next: (error: unknown) => void) {
|
||||||
|
recordAggregatePerformance(
|
||||||
|
this as AggregateWithPerformanceState,
|
||||||
|
undefined,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
next(error);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordQueryPerformance(
|
||||||
|
query: QueryWithPerformanceState,
|
||||||
|
result: unknown,
|
||||||
|
error?: unknown,
|
||||||
|
): void {
|
||||||
|
if (query[QUERY_PERFORMANCE_RECORDED]) return;
|
||||||
|
query[QUERY_PERFORMANCE_RECORDED] = true;
|
||||||
|
|
||||||
|
const startedAt = query[QUERY_PERFORMANCE_START];
|
||||||
|
if (!startedAt) return;
|
||||||
|
|
||||||
|
const hydratedDocsReturned = countHydratedDocuments(result);
|
||||||
|
const model = query.model as mongoose.Model<unknown> | undefined;
|
||||||
|
const options = query.getOptions();
|
||||||
|
|
||||||
|
recordMongooseQueryPerformance({
|
||||||
|
operation: (query as unknown as { op?: string }).op ?? 'query',
|
||||||
|
modelName: model?.modelName ?? 'unknown',
|
||||||
|
collectionName: model?.collection?.name,
|
||||||
|
durationMs: toDurationMs(startedAt),
|
||||||
|
docsReturned: countReturnedDocuments(result),
|
||||||
|
hydratedDocsReturned,
|
||||||
|
lean: Boolean(query.mongooseOptions().lean),
|
||||||
|
limit: numberOption(options.limit),
|
||||||
|
skip: numberOption(options.skip),
|
||||||
|
filterKeys: joinObjectKeys(query.getFilter()),
|
||||||
|
projectionKeys: joinObjectKeys(query.projection()),
|
||||||
|
sortKeys: joinObjectKeys(options.sort),
|
||||||
|
errorName: errorName(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordAggregatePerformance(
|
||||||
|
aggregate: AggregateWithPerformanceState,
|
||||||
|
result: unknown,
|
||||||
|
error?: unknown,
|
||||||
|
): void {
|
||||||
|
if (aggregate[QUERY_PERFORMANCE_RECORDED]) return;
|
||||||
|
aggregate[QUERY_PERFORMANCE_RECORDED] = true;
|
||||||
|
|
||||||
|
const startedAt = aggregate[QUERY_PERFORMANCE_START];
|
||||||
|
if (!startedAt) return;
|
||||||
|
|
||||||
|
const model = aggregate.model();
|
||||||
|
recordMongooseQueryPerformance({
|
||||||
|
operation: 'aggregate',
|
||||||
|
modelName: model?.modelName ?? 'unknown',
|
||||||
|
collectionName: model?.collection?.name,
|
||||||
|
durationMs: toDurationMs(startedAt),
|
||||||
|
docsReturned: countReturnedDocuments(result),
|
||||||
|
hydratedDocsReturned: countHydratedDocuments(result),
|
||||||
|
lean: true,
|
||||||
|
errorName: errorName(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function countReturnedDocuments(result: unknown): number {
|
||||||
|
if (Array.isArray(result)) return result.length;
|
||||||
|
if (result == null) return 0;
|
||||||
|
if (typeof result === 'number') return result;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function countHydratedDocuments(result: unknown): number {
|
||||||
|
if (Array.isArray(result)) {
|
||||||
|
return result.filter((item) => isHydratedDocument(item)).length;
|
||||||
|
}
|
||||||
|
return isHydratedDocument(result) ? 1 : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHydratedDocument(value: unknown): boolean {
|
||||||
|
return Boolean(
|
||||||
|
value &&
|
||||||
|
typeof value === 'object' &&
|
||||||
|
'$__' in (value as Record<string, unknown>),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function toDurationMs(startedAt: bigint): number {
|
||||||
|
return Number(process.hrtime.bigint() - startedAt) / 1_000_000;
|
||||||
|
}
|
||||||
|
|
||||||
|
function numberOption(value: unknown): number | undefined {
|
||||||
|
return typeof value === 'number' ? value : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function joinObjectKeys(value: unknown): string | undefined {
|
||||||
|
if (!value || typeof value !== 'object') return undefined;
|
||||||
|
const keys = Object.keys(value as Record<string, unknown>).sort();
|
||||||
|
return keys.length > 0 ? keys.join(',') : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorName(error: unknown): string | undefined {
|
||||||
|
return error instanceof Error ? error.name : undefined;
|
||||||
|
}
|
||||||
@@ -11,6 +11,10 @@ interface NewRelicCustomEventApi {
|
|||||||
eventType: string,
|
eventType: string,
|
||||||
attributes: Record<string, NewRelicAttributeValue>,
|
attributes: Record<string, NewRelicAttributeValue>,
|
||||||
): void;
|
): void;
|
||||||
|
addCustomAttributes(
|
||||||
|
attributes: Record<string, NewRelicAttributeValue>,
|
||||||
|
): void;
|
||||||
|
setTransactionName(name: string): void;
|
||||||
}
|
}
|
||||||
const agent = newrelicAgent as unknown as NewRelicCustomEventApi;
|
const agent = newrelicAgent as unknown as NewRelicCustomEventApi;
|
||||||
|
|
||||||
@@ -37,3 +41,25 @@ export function recordNewRelicCustomEvent(
|
|||||||
// 관측성 실패가 비즈니스 로직을 막아선 안 된다.
|
// 관측성 실패가 비즈니스 로직을 막아선 안 된다.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function addNewRelicCustomAttributes(
|
||||||
|
attributes: Record<string, NewRelicAttributeValue | undefined>,
|
||||||
|
): void {
|
||||||
|
try {
|
||||||
|
const flat: Record<string, NewRelicAttributeValue> = {};
|
||||||
|
for (const [key, value] of Object.entries(attributes)) {
|
||||||
|
if (value !== undefined) flat[key] = value;
|
||||||
|
}
|
||||||
|
agent.addCustomAttributes(flat);
|
||||||
|
} catch {
|
||||||
|
// 관측성 실패가 비즈니스 로직을 막아선 안 된다.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function setNewRelicTransactionName(name: string): void {
|
||||||
|
try {
|
||||||
|
agent.setTransactionName(name);
|
||||||
|
} catch {
|
||||||
|
// 관측성 실패가 비즈니스 로직을 막아선 안 된다.
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
import {
|
||||||
|
buildApiRequestPerformanceEvent,
|
||||||
|
createApiPerformanceRequestContext,
|
||||||
|
readBoolean,
|
||||||
|
recordMongooseQueryPerformance,
|
||||||
|
resolveApiPerformanceConfig,
|
||||||
|
runWithApiPerformanceContext,
|
||||||
|
} from './performance-observer';
|
||||||
|
|
||||||
|
describe('performance observer config', () => {
|
||||||
|
it('keeps observability disabled by default', () => {
|
||||||
|
expect(resolveApiPerformanceConfig({}).enabled).toBe(false);
|
||||||
|
expect(resolveApiPerformanceConfig({}).localEventsEnabled).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reads common boolean values', () => {
|
||||||
|
expect(readBoolean(undefined, true)).toBe(true);
|
||||||
|
expect(readBoolean('false', true)).toBe(false);
|
||||||
|
expect(readBoolean('enabled', false)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('API performance request context', () => {
|
||||||
|
const originalEnv = process.env;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
process.env = {
|
||||||
|
...originalEnv,
|
||||||
|
API_PERF_OBSERVABILITY_ENABLED: 'true',
|
||||||
|
API_PERF_LOCAL_EVENTS_ENABLED: 'false',
|
||||||
|
API_PERF_NEW_RELIC_QUERY_EVENTS_ENABLED: 'false',
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
process.env = originalEnv;
|
||||||
|
});
|
||||||
|
|
||||||
|
it('aggregates Mongoose document-return metrics into the request event', () => {
|
||||||
|
const context = createApiPerformanceRequestContext({
|
||||||
|
method: 'GET',
|
||||||
|
route: '/communities/:communityId/internal-stats/matches',
|
||||||
|
path: '/communities/abc/internal-stats/matches',
|
||||||
|
});
|
||||||
|
|
||||||
|
const event = runWithApiPerformanceContext(context, () => {
|
||||||
|
recordMongooseQueryPerformance({
|
||||||
|
operation: 'find',
|
||||||
|
modelName: 'matchStats',
|
||||||
|
collectionName: 'match_stats',
|
||||||
|
durationMs: 12,
|
||||||
|
docsReturned: 300,
|
||||||
|
hydratedDocsReturned: 300,
|
||||||
|
lean: false,
|
||||||
|
});
|
||||||
|
recordMongooseQueryPerformance({
|
||||||
|
operation: 'find',
|
||||||
|
modelName: 'matchStats',
|
||||||
|
collectionName: 'match_stats',
|
||||||
|
durationMs: 3,
|
||||||
|
docsReturned: 20,
|
||||||
|
hydratedDocsReturned: 0,
|
||||||
|
lean: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
return buildApiRequestPerformanceEvent({
|
||||||
|
context,
|
||||||
|
statusCode: 200,
|
||||||
|
durationMs: 50,
|
||||||
|
isError: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(event.queryCount).toBe(2);
|
||||||
|
expect(event.mongooseDurationMs).toBe(15);
|
||||||
|
expect(event.docsReturned).toBe(320);
|
||||||
|
expect(event.hydratedDocsReturned).toBe(300);
|
||||||
|
expect(event.leanQueryCount).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
import { appendFile } from 'node:fs/promises';
|
||||||
|
import {
|
||||||
|
addNewRelicCustomAttributes,
|
||||||
|
NewRelicAttributeValue,
|
||||||
|
recordNewRelicCustomEvent,
|
||||||
|
} from './new-relic';
|
||||||
|
|
||||||
|
export type PerformanceEventType = 'api-request' | 'mongoose-query';
|
||||||
|
|
||||||
|
export interface ApiPerformanceConfig {
|
||||||
|
enabled: boolean;
|
||||||
|
localEventsEnabled: boolean;
|
||||||
|
localEventFile: string;
|
||||||
|
newRelicRequestEventsEnabled: boolean;
|
||||||
|
newRelicQueryEventsEnabled: boolean;
|
||||||
|
appName: string;
|
||||||
|
nodeEnv: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MongooseQueryPerformance {
|
||||||
|
operation: string;
|
||||||
|
modelName: string;
|
||||||
|
collectionName?: string;
|
||||||
|
durationMs: number;
|
||||||
|
docsReturned: number;
|
||||||
|
hydratedDocsReturned: number;
|
||||||
|
lean: boolean;
|
||||||
|
limit?: number;
|
||||||
|
skip?: number;
|
||||||
|
filterKeys?: string;
|
||||||
|
projectionKeys?: string;
|
||||||
|
sortKeys?: string;
|
||||||
|
errorName?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ApiRequestPerformanceEvent {
|
||||||
|
type: 'api-request';
|
||||||
|
timestamp: string;
|
||||||
|
requestId: string;
|
||||||
|
method: string;
|
||||||
|
route: string;
|
||||||
|
path: string;
|
||||||
|
statusCode: number;
|
||||||
|
durationMs: number;
|
||||||
|
isError: boolean;
|
||||||
|
queryCount: number;
|
||||||
|
mongooseDurationMs: number;
|
||||||
|
docsReturned: number;
|
||||||
|
hydratedDocsReturned: number;
|
||||||
|
leanQueryCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MongooseQueryPerformanceEvent
|
||||||
|
extends MongooseQueryPerformance {
|
||||||
|
type: 'mongoose-query';
|
||||||
|
timestamp: string;
|
||||||
|
requestId?: string;
|
||||||
|
method?: string;
|
||||||
|
route?: string;
|
||||||
|
path?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ApiPerformanceEvent =
|
||||||
|
| ApiRequestPerformanceEvent
|
||||||
|
| MongooseQueryPerformanceEvent;
|
||||||
|
|
||||||
|
export interface ApiPerformanceContext {
|
||||||
|
requestId: string;
|
||||||
|
method: string;
|
||||||
|
route: string;
|
||||||
|
path: string;
|
||||||
|
queries: MongooseQueryPerformance[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const storage = new AsyncLocalStorage<ApiPerformanceContext>();
|
||||||
|
|
||||||
|
export function readBoolean(
|
||||||
|
value: string | undefined,
|
||||||
|
defaultValue: boolean,
|
||||||
|
): boolean {
|
||||||
|
if (value == null || value.trim() === '') return defaultValue;
|
||||||
|
return !['0', 'false', 'f', 'no', 'n', 'disabled', 'off'].includes(
|
||||||
|
value.trim().toLowerCase(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveApiPerformanceConfig(
|
||||||
|
env: NodeJS.ProcessEnv = process.env,
|
||||||
|
): ApiPerformanceConfig {
|
||||||
|
const enabled = readBoolean(env.API_PERF_OBSERVABILITY_ENABLED, false);
|
||||||
|
return {
|
||||||
|
enabled,
|
||||||
|
localEventsEnabled: readBoolean(
|
||||||
|
env.API_PERF_LOCAL_EVENTS_ENABLED,
|
||||||
|
enabled,
|
||||||
|
),
|
||||||
|
localEventFile:
|
||||||
|
env.API_PERF_LOCAL_EVENT_FILE || '/tmp/lolympus-api-performance.ndjson',
|
||||||
|
newRelicRequestEventsEnabled: readBoolean(
|
||||||
|
env.API_PERF_NEW_RELIC_REQUEST_EVENTS_ENABLED,
|
||||||
|
enabled,
|
||||||
|
),
|
||||||
|
newRelicQueryEventsEnabled: readBoolean(
|
||||||
|
env.API_PERF_NEW_RELIC_QUERY_EVENTS_ENABLED,
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
appName: env.NEW_RELIC_APP_NAME || 'lolympus',
|
||||||
|
nodeEnv: env.NODE_ENV || 'dev',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isApiPerformanceObservabilityEnabled(): boolean {
|
||||||
|
return resolveApiPerformanceConfig().enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createApiPerformanceRequestContext(params: {
|
||||||
|
method: string;
|
||||||
|
route: string;
|
||||||
|
path: string;
|
||||||
|
}): ApiPerformanceContext {
|
||||||
|
return {
|
||||||
|
requestId: randomUUID(),
|
||||||
|
method: params.method,
|
||||||
|
route: params.route,
|
||||||
|
path: params.path,
|
||||||
|
queries: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runWithApiPerformanceContext<T>(
|
||||||
|
context: ApiPerformanceContext,
|
||||||
|
callback: () => T,
|
||||||
|
): T {
|
||||||
|
return storage.run(context, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCurrentApiPerformanceContext():
|
||||||
|
| ApiPerformanceContext
|
||||||
|
| undefined {
|
||||||
|
return storage.getStore();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordMongooseQueryPerformance(
|
||||||
|
query: MongooseQueryPerformance,
|
||||||
|
): void {
|
||||||
|
if (!isApiPerformanceObservabilityEnabled()) return;
|
||||||
|
|
||||||
|
const config = resolveApiPerformanceConfig();
|
||||||
|
const context = storage.getStore();
|
||||||
|
context?.queries.push(query);
|
||||||
|
|
||||||
|
const event: MongooseQueryPerformanceEvent = {
|
||||||
|
...query,
|
||||||
|
type: 'mongoose-query',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
requestId: context?.requestId,
|
||||||
|
method: context?.method,
|
||||||
|
route: context?.route,
|
||||||
|
path: context?.path,
|
||||||
|
};
|
||||||
|
if (config.localEventsEnabled) appendLocalPerformanceEvent(event, config);
|
||||||
|
if (config.newRelicQueryEventsEnabled) {
|
||||||
|
recordNewRelicCustomEvent('MongooseQueryPerformance', {
|
||||||
|
appName: config.appName,
|
||||||
|
nodeEnv: config.nodeEnv,
|
||||||
|
requestId: event.requestId,
|
||||||
|
method: event.method,
|
||||||
|
route: event.route,
|
||||||
|
operation: event.operation,
|
||||||
|
modelName: event.modelName,
|
||||||
|
collectionName: event.collectionName,
|
||||||
|
durationMs: event.durationMs,
|
||||||
|
docsReturned: event.docsReturned,
|
||||||
|
hydratedDocsReturned: event.hydratedDocsReturned,
|
||||||
|
lean: event.lean,
|
||||||
|
limit: event.limit,
|
||||||
|
skip: event.skip,
|
||||||
|
errorName: event.errorName,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildApiRequestPerformanceEvent(params: {
|
||||||
|
context: ApiPerformanceContext;
|
||||||
|
statusCode: number;
|
||||||
|
durationMs: number;
|
||||||
|
isError: boolean;
|
||||||
|
}): ApiRequestPerformanceEvent {
|
||||||
|
const queries = params.context.queries;
|
||||||
|
return {
|
||||||
|
type: 'api-request',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
requestId: params.context.requestId,
|
||||||
|
method: params.context.method,
|
||||||
|
route: params.context.route,
|
||||||
|
path: params.context.path,
|
||||||
|
statusCode: params.statusCode,
|
||||||
|
durationMs: params.durationMs,
|
||||||
|
isError: params.isError,
|
||||||
|
queryCount: queries.length,
|
||||||
|
mongooseDurationMs: sum(queries.map((query) => query.durationMs)),
|
||||||
|
docsReturned: sum(queries.map((query) => query.docsReturned)),
|
||||||
|
hydratedDocsReturned: sum(
|
||||||
|
queries.map((query) => query.hydratedDocsReturned),
|
||||||
|
),
|
||||||
|
leanQueryCount: queries.filter((query) => query.lean).length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordApiRequestPerformance(
|
||||||
|
event: ApiRequestPerformanceEvent,
|
||||||
|
): void {
|
||||||
|
if (!isApiPerformanceObservabilityEnabled()) return;
|
||||||
|
|
||||||
|
const config = resolveApiPerformanceConfig();
|
||||||
|
if (config.localEventsEnabled) appendLocalPerformanceEvent(event, config);
|
||||||
|
if (config.newRelicRequestEventsEnabled) {
|
||||||
|
recordNewRelicCustomEvent('ApiRequestPerformance', {
|
||||||
|
appName: config.appName,
|
||||||
|
nodeEnv: config.nodeEnv,
|
||||||
|
requestId: event.requestId,
|
||||||
|
method: event.method,
|
||||||
|
route: event.route,
|
||||||
|
statusCode: event.statusCode,
|
||||||
|
durationMs: event.durationMs,
|
||||||
|
isError: event.isError,
|
||||||
|
queryCount: event.queryCount,
|
||||||
|
mongooseDurationMs: event.mongooseDurationMs,
|
||||||
|
docsReturned: event.docsReturned,
|
||||||
|
hydratedDocsReturned: event.hydratedDocsReturned,
|
||||||
|
leanQueryCount: event.leanQueryCount,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
addNewRelicCustomAttributes({
|
||||||
|
appName: config.appName,
|
||||||
|
nodeEnv: config.nodeEnv,
|
||||||
|
'api.request_id': event.requestId,
|
||||||
|
'api.method': event.method,
|
||||||
|
'api.route': event.route,
|
||||||
|
'api.status_code': event.statusCode,
|
||||||
|
'api.duration_ms': event.durationMs,
|
||||||
|
'api.is_error': event.isError,
|
||||||
|
'mongoose.query_count': event.queryCount,
|
||||||
|
'mongoose.duration_ms': event.mongooseDurationMs,
|
||||||
|
'mongoose.docs_returned': event.docsReturned,
|
||||||
|
'mongoose.hydrated_docs_returned': event.hydratedDocsReturned,
|
||||||
|
'mongoose.lean_query_count': event.leanQueryCount,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function appendLocalPerformanceEvent(
|
||||||
|
event: ApiPerformanceEvent,
|
||||||
|
config: ApiPerformanceConfig = resolveApiPerformanceConfig(),
|
||||||
|
): void {
|
||||||
|
void appendFile(config.localEventFile, `${JSON.stringify(event)}\n`).catch(
|
||||||
|
() => {
|
||||||
|
// 관측성 실패가 API 응답을 막아선 안 된다.
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function toNewRelicAttributes(
|
||||||
|
event: ApiRequestPerformanceEvent,
|
||||||
|
): Record<string, NewRelicAttributeValue> {
|
||||||
|
return {
|
||||||
|
requestId: event.requestId,
|
||||||
|
method: event.method,
|
||||||
|
route: event.route,
|
||||||
|
statusCode: event.statusCode,
|
||||||
|
durationMs: event.durationMs,
|
||||||
|
isError: event.isError,
|
||||||
|
queryCount: event.queryCount,
|
||||||
|
mongooseDurationMs: event.mongooseDurationMs,
|
||||||
|
docsReturned: event.docsReturned,
|
||||||
|
hydratedDocsReturned: event.hydratedDocsReturned,
|
||||||
|
leanQueryCount: event.leanQueryCount,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sum(values: number[]): number {
|
||||||
|
return values.reduce((acc, value) => acc + value, 0);
|
||||||
|
}
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import {
|
||||||
|
buildEndpointReportFromLocalEvents,
|
||||||
|
buildNewRelicCustomEventNrql,
|
||||||
|
buildNewRelicTransactionNrql,
|
||||||
|
discoverApiRoutes,
|
||||||
|
formatMarkdownReport,
|
||||||
|
mergeRowsWithManifest,
|
||||||
|
} from './api-performance-report';
|
||||||
|
import type { ApiPerformanceEvent } from '../modules/global/observability/performance-observer';
|
||||||
|
|
||||||
|
describe('api performance route discovery', () => {
|
||||||
|
let rootDir: string;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
rootDir = await mkdtemp(path.join(os.tmpdir(), 'api-performance-routes-'));
|
||||||
|
await mkdir(path.join(rootDir, 'modules', 'match'), { recursive: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await rm(rootDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('discovers controller routes including multiline decorators', async () => {
|
||||||
|
await writeFile(
|
||||||
|
path.join(rootDir, 'modules', 'match', 'match.controller.ts'),
|
||||||
|
`
|
||||||
|
import { Controller, Get, Post } from '@nestjs/common';
|
||||||
|
|
||||||
|
@Controller('communities/:communityId/matches')
|
||||||
|
export class MatchController {
|
||||||
|
@Get()
|
||||||
|
list() {}
|
||||||
|
|
||||||
|
@Post(
|
||||||
|
':matchId/participants'
|
||||||
|
)
|
||||||
|
join() {}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await writeFile(
|
||||||
|
path.join(rootDir, 'app.controller.ts'),
|
||||||
|
`
|
||||||
|
import { Controller, Get } from '@nestjs/common';
|
||||||
|
|
||||||
|
@Controller()
|
||||||
|
export class AppController {
|
||||||
|
@Get()
|
||||||
|
root() {}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(discoverApiRoutes(rootDir)).resolves.toEqual([
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
route: '/',
|
||||||
|
controllerFile: 'app.controller.ts',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
route: '/communities/:communityId/matches',
|
||||||
|
controllerFile: 'modules/match/match.controller.ts',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
route: '/communities/:communityId/matches/:matchId/participants',
|
||||||
|
controllerFile: 'modules/match/match.controller.ts',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('api performance report generation', () => {
|
||||||
|
it('reports every manifest route, including routes with no samples', () => {
|
||||||
|
const routes = [
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
route: '/communities/:communityId/matches',
|
||||||
|
controllerFile: 'match.controller.ts',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
route: '/communities/:communityId/matches',
|
||||||
|
controllerFile: 'match.controller.ts',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
route: '/communities/:communityId/internal-stats/matches',
|
||||||
|
controllerFile: 'community-stats.controller.ts',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const events: ApiPerformanceEvent[] = [
|
||||||
|
{
|
||||||
|
type: 'api-request',
|
||||||
|
timestamp: '2026-06-28T00:00:00.000Z',
|
||||||
|
requestId: 'request-1',
|
||||||
|
method: 'GET',
|
||||||
|
route: '/communities/:communityId/matches',
|
||||||
|
path: '/communities/abc/matches',
|
||||||
|
statusCode: 200,
|
||||||
|
durationMs: 100,
|
||||||
|
isError: false,
|
||||||
|
queryCount: 2,
|
||||||
|
mongooseDurationMs: 40,
|
||||||
|
docsReturned: 50,
|
||||||
|
hydratedDocsReturned: 50,
|
||||||
|
leanQueryCount: 0,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const rows = buildEndpointReportFromLocalEvents({
|
||||||
|
routes,
|
||||||
|
events,
|
||||||
|
p95BudgetMs: 300,
|
||||||
|
docsReturnedP95Budget: 500,
|
||||||
|
hydratedDocsReturnedP95Budget: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(rows).toHaveLength(3);
|
||||||
|
expect(
|
||||||
|
rows.find(
|
||||||
|
(row) => row.route === '/communities/:communityId/matches',
|
||||||
|
)?.status,
|
||||||
|
).toBe('document-heavy');
|
||||||
|
expect(rows.filter((row) => row.status === 'no-samples')).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps observed-only New Relic rows while adding no-sample manifest routes', () => {
|
||||||
|
const rows = mergeRowsWithManifest({
|
||||||
|
routes: [
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
route: '/known',
|
||||||
|
controllerFile: 'known.controller.ts',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
measuredRows: [
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
route: '/observed-only',
|
||||||
|
source: 'newrelic-transaction',
|
||||||
|
samples: 10,
|
||||||
|
avgMs: 40,
|
||||||
|
p50Ms: 35,
|
||||||
|
p95Ms: 80,
|
||||||
|
p99Ms: 120,
|
||||||
|
maxMs: 130,
|
||||||
|
errorRatePercentage: 0,
|
||||||
|
queryCountP95: null,
|
||||||
|
mongooseDurationP95Ms: null,
|
||||||
|
docsReturnedP95: null,
|
||||||
|
hydratedDocsReturnedP95: null,
|
||||||
|
status: 'ok',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
fallbackSource: 'newrelic-transaction',
|
||||||
|
p95BudgetMs: 300,
|
||||||
|
docsReturnedP95Budget: 500,
|
||||||
|
hydratedDocsReturnedP95Budget: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(rows.map((row) => `${row.method} ${row.route}`)).toEqual([
|
||||||
|
'GET /known',
|
||||||
|
'GET /observed-only',
|
||||||
|
]);
|
||||||
|
expect(rows[0].status).toBe('no-samples');
|
||||||
|
expect(rows[1].samples).toBe(10);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('formats all rows instead of truncating to a top list', () => {
|
||||||
|
const rows = Array.from({ length: 12 }, (_, index) => ({
|
||||||
|
method: 'GET',
|
||||||
|
route: `/route-${index}`,
|
||||||
|
source: 'local' as const,
|
||||||
|
samples: 1,
|
||||||
|
avgMs: 10,
|
||||||
|
p50Ms: 10,
|
||||||
|
p95Ms: 10,
|
||||||
|
p99Ms: 10,
|
||||||
|
maxMs: 10,
|
||||||
|
errorRatePercentage: 0,
|
||||||
|
queryCountP95: 1,
|
||||||
|
mongooseDurationP95Ms: 1,
|
||||||
|
docsReturnedP95: 1,
|
||||||
|
hydratedDocsReturnedP95: 0,
|
||||||
|
status: 'ok' as const,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const report = formatMarkdownReport({
|
||||||
|
rows,
|
||||||
|
source: 'local',
|
||||||
|
since: 'fixture',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(report).toContain('/route-0');
|
||||||
|
expect(report).toContain('/route-11');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('builds New Relic queries for custom events and transaction duration data', () => {
|
||||||
|
expect(buildNewRelicCustomEventNrql('lolympus-api', '1 day ago')).toContain(
|
||||||
|
'FROM ApiRequestPerformance',
|
||||||
|
);
|
||||||
|
expect(buildNewRelicCustomEventNrql('lolympus-api', '1 day ago')).toContain(
|
||||||
|
'FACET method, route',
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
buildNewRelicTransactionNrql('lolympus-api', '1 day ago'),
|
||||||
|
).toContain('FROM Transaction');
|
||||||
|
expect(
|
||||||
|
buildNewRelicTransactionNrql('lolympus-api', '1 day ago'),
|
||||||
|
).toContain('FACET name');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,808 @@
|
|||||||
|
import { existsSync } from 'node:fs';
|
||||||
|
import { readdir, readFile, writeFile } from 'node:fs/promises';
|
||||||
|
import path from 'node:path';
|
||||||
|
import process from 'node:process';
|
||||||
|
import type {
|
||||||
|
ApiPerformanceEvent,
|
||||||
|
ApiRequestPerformanceEvent,
|
||||||
|
} from '../modules/global/observability/performance-observer';
|
||||||
|
|
||||||
|
export type ReportSource =
|
||||||
|
| 'local'
|
||||||
|
| 'newrelic-custom'
|
||||||
|
| 'newrelic-transaction';
|
||||||
|
|
||||||
|
export interface ApiRoute {
|
||||||
|
method: string;
|
||||||
|
route: string;
|
||||||
|
controllerFile: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EndpointReportRow {
|
||||||
|
method: string;
|
||||||
|
route: string;
|
||||||
|
controllerFile?: string;
|
||||||
|
source: ReportSource | 'none';
|
||||||
|
samples: number;
|
||||||
|
avgMs: number | null;
|
||||||
|
p50Ms: number | null;
|
||||||
|
p95Ms: number | null;
|
||||||
|
p99Ms: number | null;
|
||||||
|
maxMs: number | null;
|
||||||
|
errorRatePercentage: number | null;
|
||||||
|
queryCountP95: number | null;
|
||||||
|
mongooseDurationP95Ms: number | null;
|
||||||
|
docsReturnedP95: number | null;
|
||||||
|
hydratedDocsReturnedP95: number | null;
|
||||||
|
status: 'ok' | 'slow' | 'document-heavy' | 'no-samples';
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ReportOptions {
|
||||||
|
source: ReportSource;
|
||||||
|
rootDir: string;
|
||||||
|
inputFile: string;
|
||||||
|
outputFile?: string;
|
||||||
|
format: 'markdown' | 'json';
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
since: string;
|
||||||
|
newRelicAccountId?: string;
|
||||||
|
newRelicApiKey?: string;
|
||||||
|
newRelicAppName: string;
|
||||||
|
routesOnly: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface NewRelicNrqlResponse {
|
||||||
|
data?: {
|
||||||
|
actor?: {
|
||||||
|
account?: {
|
||||||
|
nrql?: {
|
||||||
|
results?: Record<string, unknown>[];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
errors?: Array<{ message?: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const HTTP_DECORATOR_TO_METHOD: Record<string, string> = {
|
||||||
|
All: 'ALL',
|
||||||
|
Delete: 'DELETE',
|
||||||
|
Get: 'GET',
|
||||||
|
Head: 'HEAD',
|
||||||
|
Options: 'OPTIONS',
|
||||||
|
Patch: 'PATCH',
|
||||||
|
Post: 'POST',
|
||||||
|
Put: 'PUT',
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function discoverApiRoutes(rootDir: string): Promise<ApiRoute[]> {
|
||||||
|
const files = await listControllerFiles(rootDir);
|
||||||
|
const routes = new Map<string, ApiRoute>();
|
||||||
|
|
||||||
|
for (const file of files) {
|
||||||
|
const source = await readFile(file, 'utf8');
|
||||||
|
const basePath = extractDecoratorStringArgument(source, 'Controller');
|
||||||
|
if (basePath === undefined) continue;
|
||||||
|
|
||||||
|
const methodDecoratorRegex =
|
||||||
|
/@(Get|Post|Patch|Put|Delete|Head|Options|All)\s*\(([\s\S]*?)\)/g;
|
||||||
|
for (const match of source.matchAll(methodDecoratorRegex)) {
|
||||||
|
const method = HTTP_DECORATOR_TO_METHOD[match[1]];
|
||||||
|
const route = normalizeRoute(
|
||||||
|
joinRouteParts(basePath, extractFirstStringLiteral(match[2]) ?? ''),
|
||||||
|
);
|
||||||
|
const key = routeKey(method, route);
|
||||||
|
routes.set(key, {
|
||||||
|
method,
|
||||||
|
route,
|
||||||
|
controllerFile: path.relative(rootDir, file),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...routes.values()].sort(
|
||||||
|
(left, right) =>
|
||||||
|
left.route.localeCompare(right.route) ||
|
||||||
|
left.method.localeCompare(right.method),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function readLocalPerformanceEvents(
|
||||||
|
inputFile: string,
|
||||||
|
): Promise<ApiPerformanceEvent[]> {
|
||||||
|
if (!existsSync(inputFile)) return [];
|
||||||
|
|
||||||
|
const content = await readFile(inputFile, 'utf8');
|
||||||
|
return content
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.filter(Boolean)
|
||||||
|
.flatMap((line) => {
|
||||||
|
try {
|
||||||
|
return [JSON.parse(line) as ApiPerformanceEvent];
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildEndpointReportFromLocalEvents(params: {
|
||||||
|
routes: ApiRoute[];
|
||||||
|
events: ApiPerformanceEvent[];
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
}): EndpointReportRow[] {
|
||||||
|
const requestsByRoute = new Map<string, ApiRequestPerformanceEvent[]>();
|
||||||
|
for (const event of params.events) {
|
||||||
|
if (event.type !== 'api-request') continue;
|
||||||
|
const key = routeKey(event.method, event.route);
|
||||||
|
requestsByRoute.set(key, [...(requestsByRoute.get(key) ?? []), event]);
|
||||||
|
}
|
||||||
|
|
||||||
|
const rowsByKey = new Map<string, EndpointReportRow>();
|
||||||
|
for (const route of params.routes) {
|
||||||
|
const events = requestsByRoute.get(routeKey(route.method, route.route)) ?? [];
|
||||||
|
rowsByKey.set(
|
||||||
|
routeKey(route.method, route.route),
|
||||||
|
summarizeRequestEvents(route, events, {
|
||||||
|
source: 'local',
|
||||||
|
p95BudgetMs: params.p95BudgetMs,
|
||||||
|
docsReturnedP95Budget: params.docsReturnedP95Budget,
|
||||||
|
hydratedDocsReturnedP95Budget: params.hydratedDocsReturnedP95Budget,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [key, events] of requestsByRoute.entries()) {
|
||||||
|
if (rowsByKey.has(key)) continue;
|
||||||
|
const first = events[0];
|
||||||
|
rowsByKey.set(
|
||||||
|
key,
|
||||||
|
summarizeRequestEvents(
|
||||||
|
{
|
||||||
|
method: first.method,
|
||||||
|
route: first.route,
|
||||||
|
controllerFile: 'observed-only',
|
||||||
|
},
|
||||||
|
events,
|
||||||
|
{
|
||||||
|
source: 'local',
|
||||||
|
p95BudgetMs: params.p95BudgetMs,
|
||||||
|
docsReturnedP95Budget: params.docsReturnedP95Budget,
|
||||||
|
hydratedDocsReturnedP95Budget: params.hydratedDocsReturnedP95Budget,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...rowsByKey.values()].sort(compareReportRows);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchNewRelicEndpointRows(params: {
|
||||||
|
source: Exclude<ReportSource, 'local'>;
|
||||||
|
accountId: string;
|
||||||
|
apiKey: string;
|
||||||
|
appName: string;
|
||||||
|
since: string;
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
}): Promise<EndpointReportRow[]> {
|
||||||
|
const nrql =
|
||||||
|
params.source === 'newrelic-custom'
|
||||||
|
? buildNewRelicCustomEventNrql(params.appName, params.since)
|
||||||
|
: buildNewRelicTransactionNrql(params.appName, params.since);
|
||||||
|
const response = await fetch('https://api.newrelic.com/graphql', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Api-Key': params.apiKey,
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
query: `{
|
||||||
|
actor {
|
||||||
|
account(id: ${Number(params.accountId)}) {
|
||||||
|
nrql(query: ${JSON.stringify(nrql)}) {
|
||||||
|
results
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(
|
||||||
|
`New Relic query failed: ${response.status} ${response.statusText}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const json = (await response.json()) as NewRelicNrqlResponse;
|
||||||
|
if (json.errors?.length) {
|
||||||
|
throw new Error(
|
||||||
|
`New Relic query failed: ${json.errors
|
||||||
|
.map((error) => error.message ?? 'unknown error')
|
||||||
|
.join('; ')}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = json.data?.actor?.account?.nrql?.results ?? [];
|
||||||
|
return results
|
||||||
|
.map((result) =>
|
||||||
|
params.source === 'newrelic-custom'
|
||||||
|
? newRelicCustomResultToRow(result, params)
|
||||||
|
: newRelicTransactionResultToRow(result, params),
|
||||||
|
)
|
||||||
|
.filter((row): row is EndpointReportRow => row !== null)
|
||||||
|
.sort(compareReportRows);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mergeRowsWithManifest(params: {
|
||||||
|
routes: ApiRoute[];
|
||||||
|
measuredRows: EndpointReportRow[];
|
||||||
|
fallbackSource: ReportSource;
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
}): EndpointReportRow[] {
|
||||||
|
const rowsByKey = new Map(
|
||||||
|
params.measuredRows.map((row) => [routeKey(row.method, row.route), row]),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const route of params.routes) {
|
||||||
|
const key = routeKey(route.method, route.route);
|
||||||
|
if (rowsByKey.has(key)) {
|
||||||
|
rowsByKey.set(key, {
|
||||||
|
...rowsByKey.get(key)!,
|
||||||
|
controllerFile: route.controllerFile,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
rowsByKey.set(key, {
|
||||||
|
method: route.method,
|
||||||
|
route: route.route,
|
||||||
|
controllerFile: route.controllerFile,
|
||||||
|
source: 'none',
|
||||||
|
samples: 0,
|
||||||
|
avgMs: null,
|
||||||
|
p50Ms: null,
|
||||||
|
p95Ms: null,
|
||||||
|
p99Ms: null,
|
||||||
|
maxMs: null,
|
||||||
|
errorRatePercentage: null,
|
||||||
|
queryCountP95: null,
|
||||||
|
mongooseDurationP95Ms: null,
|
||||||
|
docsReturnedP95: null,
|
||||||
|
hydratedDocsReturnedP95: null,
|
||||||
|
status: 'no-samples',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...rowsByKey.values()].sort(compareReportRows);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatMarkdownReport(params: {
|
||||||
|
rows: EndpointReportRow[];
|
||||||
|
source: ReportSource;
|
||||||
|
since: string;
|
||||||
|
notes?: string[];
|
||||||
|
}): string {
|
||||||
|
const sampled = params.rows.filter((row) => row.samples > 0);
|
||||||
|
const noSamples = params.rows.length - sampled.length;
|
||||||
|
const slow = sampled.filter((row) => row.status === 'slow').length;
|
||||||
|
const documentHeavy = sampled.filter(
|
||||||
|
(row) => row.status === 'document-heavy',
|
||||||
|
).length;
|
||||||
|
|
||||||
|
const lines = [
|
||||||
|
'# API Performance Report',
|
||||||
|
'',
|
||||||
|
`- Source: ${params.source}`,
|
||||||
|
`- Window: ${params.since}`,
|
||||||
|
`- Routes: ${params.rows.length}`,
|
||||||
|
`- Sampled routes: ${sampled.length}`,
|
||||||
|
`- No-sample routes: ${noSamples}`,
|
||||||
|
`- Slow routes: ${slow}`,
|
||||||
|
`- Document-heavy routes: ${documentHeavy}`,
|
||||||
|
];
|
||||||
|
|
||||||
|
if (params.notes?.length) {
|
||||||
|
lines.push('', '## Notes', '');
|
||||||
|
for (const note of params.notes) lines.push(`- ${note}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
'',
|
||||||
|
'| Method | Route | Samples | p95 ms | p99 ms | Avg ms | Error % | Query p95 | Docs p95 | Hydrated p95 | Status |',
|
||||||
|
'|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|',
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const row of params.rows) {
|
||||||
|
lines.push(
|
||||||
|
[
|
||||||
|
`| ${row.method}`,
|
||||||
|
markdownCell(row.route),
|
||||||
|
String(row.samples),
|
||||||
|
formatNumber(row.p95Ms),
|
||||||
|
formatNumber(row.p99Ms),
|
||||||
|
formatNumber(row.avgMs),
|
||||||
|
formatNumber(row.errorRatePercentage),
|
||||||
|
formatNumber(row.queryCountP95),
|
||||||
|
formatNumber(row.docsReturnedP95),
|
||||||
|
formatNumber(row.hydratedDocsReturnedP95),
|
||||||
|
`${row.status} |`,
|
||||||
|
].join(' | '),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${lines.join('\n')}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildNewRelicCustomEventNrql(
|
||||||
|
appName: string,
|
||||||
|
since: string,
|
||||||
|
): string {
|
||||||
|
return [
|
||||||
|
'FROM ApiRequestPerformance',
|
||||||
|
'SELECT count(*) AS samples,',
|
||||||
|
'average(durationMs) AS avgDurationMs,',
|
||||||
|
'percentile(durationMs, 50, 95, 99) AS durationPercentilesMs,',
|
||||||
|
'max(durationMs) AS maxDurationMs,',
|
||||||
|
'percentage(count(*), WHERE isError IS true) AS errorRatePercentage,',
|
||||||
|
'percentile(queryCount, 95) AS queryCountPercentiles,',
|
||||||
|
'percentile(docsReturned, 95) AS docsReturnedPercentiles,',
|
||||||
|
'percentile(hydratedDocsReturned, 95) AS hydratedDocsReturnedPercentiles,',
|
||||||
|
'percentile(mongooseDurationMs, 95) AS mongooseDurationPercentiles',
|
||||||
|
`WHERE appName = '${escapeNrqlString(appName)}'`,
|
||||||
|
'FACET method, route',
|
||||||
|
`SINCE ${since}`,
|
||||||
|
'LIMIT MAX',
|
||||||
|
].join(' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildNewRelicTransactionNrql(
|
||||||
|
appName: string,
|
||||||
|
since: string,
|
||||||
|
): string {
|
||||||
|
return [
|
||||||
|
'FROM Transaction',
|
||||||
|
'SELECT count(*) AS samples,',
|
||||||
|
'average(duration) AS avgDurationSec,',
|
||||||
|
'percentile(duration, 50, 95, 99) AS durationPercentilesSec,',
|
||||||
|
'max(duration) AS maxDurationSec,',
|
||||||
|
'percentage(count(*), WHERE error IS true) AS errorRatePercentage',
|
||||||
|
`WHERE appName = '${escapeNrqlString(appName)}'`,
|
||||||
|
'FACET name',
|
||||||
|
`SINCE ${since}`,
|
||||||
|
'LIMIT MAX',
|
||||||
|
].join(' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarizeRequestEvents(
|
||||||
|
route: ApiRoute,
|
||||||
|
events: ApiRequestPerformanceEvent[],
|
||||||
|
options: {
|
||||||
|
source: ReportSource;
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
},
|
||||||
|
): EndpointReportRow {
|
||||||
|
const row: EndpointReportRow = {
|
||||||
|
method: route.method,
|
||||||
|
route: route.route,
|
||||||
|
controllerFile: route.controllerFile,
|
||||||
|
source: events.length > 0 ? options.source : 'none',
|
||||||
|
samples: events.length,
|
||||||
|
avgMs: average(events.map((event) => event.durationMs)),
|
||||||
|
p50Ms: percentile(
|
||||||
|
events.map((event) => event.durationMs),
|
||||||
|
50,
|
||||||
|
),
|
||||||
|
p95Ms: percentile(
|
||||||
|
events.map((event) => event.durationMs),
|
||||||
|
95,
|
||||||
|
),
|
||||||
|
p99Ms: percentile(
|
||||||
|
events.map((event) => event.durationMs),
|
||||||
|
99,
|
||||||
|
),
|
||||||
|
maxMs: max(events.map((event) => event.durationMs)),
|
||||||
|
errorRatePercentage:
|
||||||
|
events.length > 0
|
||||||
|
? (events.filter((event) => event.isError).length / events.length) * 100
|
||||||
|
: null,
|
||||||
|
queryCountP95: percentile(
|
||||||
|
events.map((event) => event.queryCount),
|
||||||
|
95,
|
||||||
|
),
|
||||||
|
mongooseDurationP95Ms: percentile(
|
||||||
|
events.map((event) => event.mongooseDurationMs),
|
||||||
|
95,
|
||||||
|
),
|
||||||
|
docsReturnedP95: percentile(
|
||||||
|
events.map((event) => event.docsReturned),
|
||||||
|
95,
|
||||||
|
),
|
||||||
|
hydratedDocsReturnedP95: percentile(
|
||||||
|
events.map((event) => event.hydratedDocsReturned),
|
||||||
|
95,
|
||||||
|
),
|
||||||
|
status: 'ok',
|
||||||
|
};
|
||||||
|
row.status = classifyRow(row, options);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
function newRelicCustomResultToRow(
|
||||||
|
result: Record<string, unknown>,
|
||||||
|
options: {
|
||||||
|
source: ReportSource;
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
},
|
||||||
|
): EndpointReportRow | null {
|
||||||
|
const facet = result.facet;
|
||||||
|
const method = Array.isArray(facet) ? String(facet[0] ?? '') : undefined;
|
||||||
|
const route = Array.isArray(facet) ? String(facet[1] ?? '') : undefined;
|
||||||
|
if (!method || !route) return null;
|
||||||
|
|
||||||
|
const row: EndpointReportRow = {
|
||||||
|
method,
|
||||||
|
route: normalizeRoute(route),
|
||||||
|
source: 'newrelic-custom',
|
||||||
|
samples: numberField(result.samples),
|
||||||
|
avgMs: numberOrNull(result.avgDurationMs),
|
||||||
|
p50Ms: percentileField(result.durationPercentilesMs, 50),
|
||||||
|
p95Ms: percentileField(result.durationPercentilesMs, 95),
|
||||||
|
p99Ms: percentileField(result.durationPercentilesMs, 99),
|
||||||
|
maxMs: numberOrNull(result.maxDurationMs),
|
||||||
|
errorRatePercentage: numberOrNull(result.errorRatePercentage),
|
||||||
|
queryCountP95: percentileField(result.queryCountPercentiles, 95),
|
||||||
|
mongooseDurationP95Ms: percentileField(
|
||||||
|
result.mongooseDurationPercentiles,
|
||||||
|
95,
|
||||||
|
),
|
||||||
|
docsReturnedP95: percentileField(result.docsReturnedPercentiles, 95),
|
||||||
|
hydratedDocsReturnedP95: percentileField(
|
||||||
|
result.hydratedDocsReturnedPercentiles,
|
||||||
|
95,
|
||||||
|
),
|
||||||
|
status: 'ok',
|
||||||
|
};
|
||||||
|
row.status = classifyRow(row, options);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
function newRelicTransactionResultToRow(
|
||||||
|
result: Record<string, unknown>,
|
||||||
|
options: {
|
||||||
|
source: ReportSource;
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
},
|
||||||
|
): EndpointReportRow | null {
|
||||||
|
const parsed = parseTransactionName(String(result.facet ?? ''));
|
||||||
|
if (!parsed) return null;
|
||||||
|
|
||||||
|
const row: EndpointReportRow = {
|
||||||
|
method: parsed.method,
|
||||||
|
route: parsed.route,
|
||||||
|
source: 'newrelic-transaction',
|
||||||
|
samples: numberField(result.samples),
|
||||||
|
avgMs: secondsToMs(result.avgDurationSec),
|
||||||
|
p50Ms: secondsToMs(percentileField(result.durationPercentilesSec, 50)),
|
||||||
|
p95Ms: secondsToMs(percentileField(result.durationPercentilesSec, 95)),
|
||||||
|
p99Ms: secondsToMs(percentileField(result.durationPercentilesSec, 99)),
|
||||||
|
maxMs: secondsToMs(result.maxDurationSec),
|
||||||
|
errorRatePercentage: numberOrNull(result.errorRatePercentage),
|
||||||
|
queryCountP95: null,
|
||||||
|
mongooseDurationP95Ms: null,
|
||||||
|
docsReturnedP95: null,
|
||||||
|
hydratedDocsReturnedP95: null,
|
||||||
|
status: 'ok',
|
||||||
|
};
|
||||||
|
row.status = classifyRow(row, options);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseTransactionName(
|
||||||
|
name: string,
|
||||||
|
): { method: string; route: string } | null {
|
||||||
|
const match = name.match(
|
||||||
|
/\b(GET|POST|PATCH|PUT|DELETE|HEAD|OPTIONS|ALL)[\s/]+(\/.*)$/i,
|
||||||
|
);
|
||||||
|
if (!match) return null;
|
||||||
|
return {
|
||||||
|
method: match[1].toUpperCase(),
|
||||||
|
route: normalizeRoute(match[2]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function classifyRow(
|
||||||
|
row: EndpointReportRow,
|
||||||
|
options: {
|
||||||
|
p95BudgetMs: number;
|
||||||
|
docsReturnedP95Budget: number;
|
||||||
|
hydratedDocsReturnedP95Budget: number;
|
||||||
|
},
|
||||||
|
): EndpointReportRow['status'] {
|
||||||
|
if (row.samples === 0) return 'no-samples';
|
||||||
|
if (row.p95Ms !== null && row.p95Ms > options.p95BudgetMs) return 'slow';
|
||||||
|
if (
|
||||||
|
(row.docsReturnedP95 !== null &&
|
||||||
|
row.docsReturnedP95 > options.docsReturnedP95Budget) ||
|
||||||
|
(row.hydratedDocsReturnedP95 !== null &&
|
||||||
|
row.hydratedDocsReturnedP95 >
|
||||||
|
options.hydratedDocsReturnedP95Budget)
|
||||||
|
) {
|
||||||
|
return 'document-heavy';
|
||||||
|
}
|
||||||
|
return 'ok';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listControllerFiles(rootDir: string): Promise<string[]> {
|
||||||
|
const entries = await readdir(rootDir, { withFileTypes: true });
|
||||||
|
const files = await Promise.all(
|
||||||
|
entries.map(async (entry) => {
|
||||||
|
const fullPath = path.join(rootDir, entry.name);
|
||||||
|
if (entry.isDirectory()) return listControllerFiles(fullPath);
|
||||||
|
return entry.isFile() && entry.name.endsWith('.controller.ts')
|
||||||
|
? [fullPath]
|
||||||
|
: [];
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return files.flat();
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractDecoratorStringArgument(
|
||||||
|
source: string,
|
||||||
|
decoratorName: string,
|
||||||
|
): string | undefined {
|
||||||
|
const regex = new RegExp(`@${decoratorName}\\s*\\(([\\s\\S]*?)\\)`);
|
||||||
|
const match = source.match(regex);
|
||||||
|
if (!match) return undefined;
|
||||||
|
return extractFirstStringLiteral(match[1]) ?? '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractFirstStringLiteral(input: string): string | undefined {
|
||||||
|
const match = input.match(/['"`]([^'"`]*)['"`]/);
|
||||||
|
return match?.[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
function joinRouteParts(basePath: string, routePath: string): string {
|
||||||
|
return [basePath, routePath]
|
||||||
|
.filter((part) => part !== '')
|
||||||
|
.join('/')
|
||||||
|
.replace(/\/+/g, '/');
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRoute(route: string): string {
|
||||||
|
const normalized = route.startsWith('/') ? route : `/${route}`;
|
||||||
|
return normalized.replace(/\/+/g, '/').replace(/\/$/, '') || '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
function routeKey(method: string, route: string): string {
|
||||||
|
return `${method.toUpperCase()} ${normalizeRoute(route)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function compareReportRows(left: EndpointReportRow, right: EndpointReportRow) {
|
||||||
|
return (
|
||||||
|
left.route.localeCompare(right.route) ||
|
||||||
|
left.method.localeCompare(right.method)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function percentile(values: number[], percentileValue: number): number | null {
|
||||||
|
if (values.length === 0) return null;
|
||||||
|
const sorted = [...values].sort((left, right) => left - right);
|
||||||
|
const index = Math.ceil((percentileValue / 100) * sorted.length) - 1;
|
||||||
|
return sorted[Math.max(0, Math.min(sorted.length - 1, index))];
|
||||||
|
}
|
||||||
|
|
||||||
|
function average(values: number[]): number | null {
|
||||||
|
if (values.length === 0) return null;
|
||||||
|
return values.reduce((acc, value) => acc + value, 0) / values.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
function max(values: number[]): number | null {
|
||||||
|
if (values.length === 0) return null;
|
||||||
|
return Math.max(...values);
|
||||||
|
}
|
||||||
|
|
||||||
|
function percentileField(value: unknown, percentileValue: number): number | null {
|
||||||
|
if (!value || typeof value !== 'object') return null;
|
||||||
|
const map = value as Record<string, unknown>;
|
||||||
|
return numberOrNull(map[String(percentileValue)] ?? map[percentileValue]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function numberField(value: unknown): number {
|
||||||
|
return numberOrNull(value) ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function numberOrNull(value: unknown): number | null {
|
||||||
|
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function secondsToMs(value: unknown): number | null {
|
||||||
|
const seconds = numberOrNull(value);
|
||||||
|
return seconds === null ? null : seconds * 1000;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatNumber(value: number | null): string {
|
||||||
|
return value === null ? '-' : value.toFixed(value >= 100 ? 0 : 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
function markdownCell(value: string): string {
|
||||||
|
return value.replace(/\|/g, '\\|');
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeNrqlString(value: string): string {
|
||||||
|
return value.replace(/'/g, "\\'");
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseArgs(argv: string[]): ReportOptions {
|
||||||
|
const args = new Map<string, string | boolean>();
|
||||||
|
for (const arg of argv) {
|
||||||
|
if (!arg.startsWith('--')) continue;
|
||||||
|
const [key, rawValue] = arg.slice(2).split('=', 2);
|
||||||
|
args.set(key, rawValue ?? true);
|
||||||
|
}
|
||||||
|
|
||||||
|
const source = stringArg(args, 'source', defaultReportSource()) as ReportSource;
|
||||||
|
return {
|
||||||
|
source,
|
||||||
|
rootDir: path.resolve(
|
||||||
|
stringArg(args, 'root-dir', path.join(process.cwd(), 'src')),
|
||||||
|
),
|
||||||
|
inputFile: path.resolve(
|
||||||
|
stringArg(
|
||||||
|
args,
|
||||||
|
'input',
|
||||||
|
process.env.API_PERF_LOCAL_EVENT_FILE ??
|
||||||
|
'/tmp/lolympus-api-performance.ndjson',
|
||||||
|
),
|
||||||
|
),
|
||||||
|
outputFile: optionalStringArg(args, 'output'),
|
||||||
|
format: stringArg(args, 'format', 'markdown') as 'markdown' | 'json',
|
||||||
|
p95BudgetMs: numberArg(args, 'p95-budget-ms', 300),
|
||||||
|
docsReturnedP95Budget: numberArg(args, 'docs-returned-p95-budget', 500),
|
||||||
|
hydratedDocsReturnedP95Budget: numberArg(
|
||||||
|
args,
|
||||||
|
'hydrated-docs-returned-p95-budget',
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
since: stringArg(args, 'since', '1 day ago'),
|
||||||
|
newRelicAccountId:
|
||||||
|
optionalStringArg(args, 'new-relic-account-id') ??
|
||||||
|
process.env.NEW_RELIC_ACCOUNT_ID,
|
||||||
|
newRelicApiKey:
|
||||||
|
optionalStringArg(args, 'new-relic-api-key') ??
|
||||||
|
process.env.NEW_RELIC_USER_API_KEY ??
|
||||||
|
process.env.NEW_RELIC_API_KEY,
|
||||||
|
newRelicAppName:
|
||||||
|
optionalStringArg(args, 'new-relic-app-name') ??
|
||||||
|
process.env.NEW_RELIC_APP_NAME ??
|
||||||
|
'lolympus',
|
||||||
|
routesOnly: args.has('routes-only'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultReportSource(): ReportSource {
|
||||||
|
return process.env.NEW_RELIC_ACCOUNT_ID &&
|
||||||
|
(process.env.NEW_RELIC_USER_API_KEY || process.env.NEW_RELIC_API_KEY)
|
||||||
|
? 'newrelic-custom'
|
||||||
|
: 'local';
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringArg(
|
||||||
|
args: Map<string, string | boolean>,
|
||||||
|
name: string,
|
||||||
|
defaultValue: string,
|
||||||
|
): string {
|
||||||
|
const value = args.get(name);
|
||||||
|
return typeof value === 'string' && value !== '' ? value : defaultValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
function optionalStringArg(
|
||||||
|
args: Map<string, string | boolean>,
|
||||||
|
name: string,
|
||||||
|
): string | undefined {
|
||||||
|
const value = args.get(name);
|
||||||
|
return typeof value === 'string' && value !== '' ? value : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function numberArg(
|
||||||
|
args: Map<string, string | boolean>,
|
||||||
|
name: string,
|
||||||
|
defaultValue: number,
|
||||||
|
): number {
|
||||||
|
const value = Number(stringArg(args, name, String(defaultValue)));
|
||||||
|
return Number.isFinite(value) ? value : defaultValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function buildReport(options: ReportOptions): Promise<string> {
|
||||||
|
const routes = await discoverApiRoutes(options.rootDir);
|
||||||
|
if (options.routesOnly) {
|
||||||
|
return `${JSON.stringify(routes, null, 2)}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const measuredRows =
|
||||||
|
options.source === 'local'
|
||||||
|
? buildEndpointReportFromLocalEvents({
|
||||||
|
routes,
|
||||||
|
events: await readLocalPerformanceEvents(options.inputFile),
|
||||||
|
p95BudgetMs: options.p95BudgetMs,
|
||||||
|
docsReturnedP95Budget: options.docsReturnedP95Budget,
|
||||||
|
hydratedDocsReturnedP95Budget:
|
||||||
|
options.hydratedDocsReturnedP95Budget,
|
||||||
|
})
|
||||||
|
: await fetchNewRelicEndpointRows({
|
||||||
|
source: options.source,
|
||||||
|
accountId: requireOption(
|
||||||
|
options.newRelicAccountId,
|
||||||
|
'NEW_RELIC_ACCOUNT_ID',
|
||||||
|
),
|
||||||
|
apiKey: requireOption(options.newRelicApiKey, 'NEW_RELIC_API_KEY'),
|
||||||
|
appName: options.newRelicAppName,
|
||||||
|
since: options.since,
|
||||||
|
p95BudgetMs: options.p95BudgetMs,
|
||||||
|
docsReturnedP95Budget: options.docsReturnedP95Budget,
|
||||||
|
hydratedDocsReturnedP95Budget:
|
||||||
|
options.hydratedDocsReturnedP95Budget,
|
||||||
|
});
|
||||||
|
const notes =
|
||||||
|
options.source === 'newrelic-custom' && measuredRows.length === 0
|
||||||
|
? [
|
||||||
|
'No ApiRequestPerformance custom events were found for this window and appName.',
|
||||||
|
'Custom events are created only after the API runs with API_PERF_OBSERVABILITY_ENABLED=true and API_PERF_NEW_RELIC_REQUEST_EVENTS_ENABLED=true.',
|
||||||
|
'Use --source=newrelic-transaction for historical latency before custom instrumentation was enabled.',
|
||||||
|
]
|
||||||
|
: [];
|
||||||
|
const rows = mergeRowsWithManifest({
|
||||||
|
routes,
|
||||||
|
measuredRows,
|
||||||
|
fallbackSource: options.source,
|
||||||
|
p95BudgetMs: options.p95BudgetMs,
|
||||||
|
docsReturnedP95Budget: options.docsReturnedP95Budget,
|
||||||
|
hydratedDocsReturnedP95Budget: options.hydratedDocsReturnedP95Budget,
|
||||||
|
});
|
||||||
|
|
||||||
|
return options.format === 'json'
|
||||||
|
? `${JSON.stringify(rows, null, 2)}\n`
|
||||||
|
: formatMarkdownReport({
|
||||||
|
rows,
|
||||||
|
source: options.source,
|
||||||
|
since: options.since,
|
||||||
|
notes,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireOption(value: string | undefined, name: string): string {
|
||||||
|
if (value) return value;
|
||||||
|
throw new Error(`${name} is required for New Relic report sources.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const options = parseArgs(process.argv.slice(2));
|
||||||
|
const report = await buildReport(options);
|
||||||
|
if (options.outputFile) {
|
||||||
|
await writeFile(path.resolve(options.outputFile), report);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
process.stdout.write(report);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
void main().catch((error: unknown) => {
|
||||||
|
process.stderr.write(
|
||||||
|
`${error instanceof Error ? error.message : String(error)}\n`,
|
||||||
|
);
|
||||||
|
process.exitCode = 1;
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
# API Performance Observability Runbook
|
||||||
|
|
||||||
|
이 문서는 API p95 latency, Mongoose query count, returned document count,
|
||||||
|
hydrated document count를 기준으로 전체 API 성능 리포트를 생성하는 운영 절차를
|
||||||
|
정리한다.
|
||||||
|
|
||||||
|
## 목적
|
||||||
|
|
||||||
|
- 특정 top 10 API만 보는 것이 아니라, controller에 정의된 모든 API를 대상으로
|
||||||
|
리포트를 만든다.
|
||||||
|
- 운영 New Relic 실측 Transaction duration 데이터를 우선 활용한다.
|
||||||
|
- `ApiRequestPerformance` custom event가 쌓인 환경에서는 Mongoose document 반환량까지
|
||||||
|
함께 분석한다.
|
||||||
|
- 로컬/staging 리허설에서는 NDJSON 파일로 동일한 리포트를 만든다.
|
||||||
|
|
||||||
|
## 데이터 소스
|
||||||
|
|
||||||
|
| Source | 명령 옵션 | 용도 | 포함 지표 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Local NDJSON | `--source=local` | staging/local 재현 테스트 | latency, query count, Mongoose duration, docs returned, hydrated docs |
|
||||||
|
| New Relic custom event | `--source=newrelic-custom` | 운영/staging 상세 실측 분석 | latency, query count, Mongoose duration, docs returned, hydrated docs |
|
||||||
|
| New Relic Transaction | `--source=newrelic-transaction` | 기존 New Relic latency 확인 | latency, error rate |
|
||||||
|
|
||||||
|
## 환경변수
|
||||||
|
|
||||||
|
### API 서버 계측
|
||||||
|
|
||||||
|
| 환경변수 | 기본값 | 설명 |
|
||||||
|
|---|---:|---|
|
||||||
|
| `API_PERF_OBSERVABILITY_ENABLED` | `false` | API/Mongoose 성능 계측 전체 활성화 |
|
||||||
|
| `API_PERF_LOCAL_EVENTS_ENABLED` | `API_PERF_OBSERVABILITY_ENABLED` 값 | 로컬 NDJSON 파일 기록 여부 |
|
||||||
|
| `API_PERF_LOCAL_EVENT_FILE` | `/tmp/lolympus-api-performance.ndjson` | 로컬 이벤트 파일 경로 |
|
||||||
|
| `API_PERF_NEW_RELIC_REQUEST_EVENTS_ENABLED` | `API_PERF_OBSERVABILITY_ENABLED` 값 | `ApiRequestPerformance` custom event 기록 여부 |
|
||||||
|
| `API_PERF_NEW_RELIC_QUERY_EVENTS_ENABLED` | `false` | query별 `MongooseQueryPerformance` custom event 기록 여부 |
|
||||||
|
|
||||||
|
`API_PERF_NEW_RELIC_QUERY_EVENTS_ENABLED`는 이벤트 수가 많을 수 있으므로 기본값을
|
||||||
|
`false`로 유지한다. request 단위 지표만으로도 route별 문서 반환량은 확인할 수 있다.
|
||||||
|
|
||||||
|
### New Relic 리포트 조회
|
||||||
|
|
||||||
|
| 환경변수 | 설명 |
|
||||||
|
|---|---|
|
||||||
|
| `NEW_RELIC_ACCOUNT_ID` | New Relic account id |
|
||||||
|
| `NEW_RELIC_USER_API_KEY` 또는 `NEW_RELIC_API_KEY` | NerdGraph 조회용 API key |
|
||||||
|
| `NEW_RELIC_APP_NAME` | API app name. 기본값은 `lolympus` |
|
||||||
|
|
||||||
|
## 기본 운영 흐름
|
||||||
|
|
||||||
|
### 1. 전체 API 목록 확인
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm --filter api perf:routes
|
||||||
|
```
|
||||||
|
|
||||||
|
이 명령은 `*.controller.ts` 파일을 읽어 전체 route manifest를 출력한다. 리포트는
|
||||||
|
이 manifest를 기준으로 모든 API를 출력한다. 트래픽이 없었던 API는 `no-samples`로
|
||||||
|
표시된다.
|
||||||
|
|
||||||
|
### 2. New Relic Transaction 기반 전체 latency 리포트
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NEW_RELIC_ACCOUNT_ID=... \
|
||||||
|
NEW_RELIC_USER_API_KEY=... \
|
||||||
|
NEW_RELIC_APP_NAME=lolympus \
|
||||||
|
pnpm --filter api perf:report -- --source=newrelic-transaction --since="1 day ago"
|
||||||
|
```
|
||||||
|
|
||||||
|
이미 New Relic에 쌓인 Transaction duration을 사용한다. 가장 먼저 실행해 전체 API의
|
||||||
|
p95/p99 latency와 error rate를 확인한다.
|
||||||
|
|
||||||
|
제한사항:
|
||||||
|
|
||||||
|
- Mongoose docs returned, hydrated docs returned는 나오지 않는다.
|
||||||
|
- 이번 계측 배포 이후에는 transaction name이 `METHOD /route-template` 형태로 잡혀
|
||||||
|
route별 집계 정확도가 올라간다.
|
||||||
|
|
||||||
|
### 3. New Relic custom event 기반 상세 리포트
|
||||||
|
|
||||||
|
API 서버가 아래 환경변수로 실행된 뒤 트래픽이 쌓여야 한다.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
API_PERF_OBSERVABILITY_ENABLED=true
|
||||||
|
API_PERF_NEW_RELIC_REQUEST_EVENTS_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
중요: custom event는 소급 생성되지 않는다. 계측 코드가 배포되고 위 환경변수로
|
||||||
|
서버가 재시작된 이후의 요청부터 `ApiRequestPerformance`가 쌓인다. 기존 New Relic
|
||||||
|
Transaction 데이터가 있어도 custom event 리포트는 0건일 수 있다.
|
||||||
|
|
||||||
|
리포트 생성:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NEW_RELIC_ACCOUNT_ID=... \
|
||||||
|
NEW_RELIC_USER_API_KEY=... \
|
||||||
|
NEW_RELIC_APP_NAME=lolympus \
|
||||||
|
pnpm --filter api perf:report -- --source=newrelic-custom --since="1 day ago"
|
||||||
|
```
|
||||||
|
|
||||||
|
이 리포트는 route별로 다음 지표를 포함한다.
|
||||||
|
|
||||||
|
- `samples`
|
||||||
|
- `avgMs`, `p50Ms`, `p95Ms`, `p99Ms`, `maxMs`
|
||||||
|
- `errorRatePercentage`
|
||||||
|
- `queryCountP95`
|
||||||
|
- `mongooseDurationP95Ms`
|
||||||
|
- `docsReturnedP95`
|
||||||
|
- `hydratedDocsReturnedP95`
|
||||||
|
|
||||||
|
### 4. Local NDJSON 기반 재현 테스트
|
||||||
|
|
||||||
|
staging/local API 서버를 계측 모드로 실행한다.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
API_PERF_OBSERVABILITY_ENABLED=true \
|
||||||
|
API_PERF_LOCAL_EVENTS_ENABLED=true \
|
||||||
|
API_PERF_LOCAL_EVENT_FILE=/tmp/lolympus-api-performance.ndjson \
|
||||||
|
pnpm --filter api start:staging
|
||||||
|
```
|
||||||
|
|
||||||
|
브라우저, curl, Postman, k6 등으로 테스트 대상 API를 호출한다. 호출하지 않은 API는
|
||||||
|
리포트에서 `no-samples`로 남는다.
|
||||||
|
|
||||||
|
리포트 생성:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm --filter api perf:report -- \
|
||||||
|
--source=local \
|
||||||
|
--input=/tmp/lolympus-api-performance.ndjson
|
||||||
|
```
|
||||||
|
|
||||||
|
JSON 출력:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm --filter api perf:report -- \
|
||||||
|
--source=local \
|
||||||
|
--input=/tmp/lolympus-api-performance.ndjson \
|
||||||
|
--format=json
|
||||||
|
```
|
||||||
|
|
||||||
|
파일 저장:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm --filter api perf:report -- \
|
||||||
|
--source=local \
|
||||||
|
--input=/tmp/lolympus-api-performance.ndjson \
|
||||||
|
--output=/tmp/api-performance-report.md
|
||||||
|
```
|
||||||
|
|
||||||
|
## 예산값 조정
|
||||||
|
|
||||||
|
기본 기준:
|
||||||
|
|
||||||
|
- p95 latency budget: `300ms`
|
||||||
|
- docs returned p95 budget: `500`
|
||||||
|
- hydrated docs returned p95 budget: `0`
|
||||||
|
|
||||||
|
명령에서 덮어쓸 수 있다.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm --filter api perf:report -- \
|
||||||
|
--source=newrelic-custom \
|
||||||
|
--since="1 day ago" \
|
||||||
|
--p95-budget-ms=500 \
|
||||||
|
--docs-returned-p95-budget=300 \
|
||||||
|
--hydrated-docs-returned-p95-budget=0
|
||||||
|
```
|
||||||
|
|
||||||
|
## 리포트 해석
|
||||||
|
|
||||||
|
| Status | 의미 | 다음 작업 |
|
||||||
|
|---|---|---|
|
||||||
|
| `ok` | 현재 budget 안에 있음 | 유지 관찰 |
|
||||||
|
| `slow` | p95 latency가 budget 초과 | New Relic trace, Mongoose duration, 외부 호출 확인 |
|
||||||
|
| `document-heavy` | docs returned 또는 hydrated docs returned가 budget 초과 | `.lean()`, `.select()`, pagination, aggregation/materialized view 검토 |
|
||||||
|
| `no-samples` | route manifest에는 있으나 측정 기간에 호출 샘플 없음 | 테스트 커버리지/운영 트래픽 여부 확인 |
|
||||||
|
|
||||||
|
`document-heavy`가 뜨는 API는 latency가 아직 괜찮아도 우선 조사 대상이다. 데이터가
|
||||||
|
늘어나면 p95 latency가 악화될 가능성이 높다.
|
||||||
|
|
||||||
|
## 권장 분석 순서
|
||||||
|
|
||||||
|
1. `newrelic-transaction`으로 전체 route의 p95/p99 latency를 확인한다.
|
||||||
|
2. `newrelic-custom`으로 `slow` 또는 `document-heavy` route를 확인한다.
|
||||||
|
3. `docsReturnedP95`가 크면 projection, pagination, aggregate result 축소를 본다.
|
||||||
|
4. `hydratedDocsReturnedP95`가 0보다 크면 읽기 전용 경로에 `.lean()` 적용을 검토한다.
|
||||||
|
5. `queryCountP95`가 크면 N+1 query 또는 반복 조회를 찾는다.
|
||||||
|
6. `mongooseDurationP95Ms`가 낮은데 `p95Ms`만 높으면 외부 API, CPU work, serialization,
|
||||||
|
payload size를 본다.
|
||||||
|
|
||||||
|
## 전체 API 테스트 체크리스트
|
||||||
|
|
||||||
|
- [ ] `pnpm --filter api perf:routes`로 route 목록을 확인했다.
|
||||||
|
- [ ] 주요 화면/운영 기능을 실제로 한 번 이상 호출했다.
|
||||||
|
- [ ] 리포트의 `no-samples` route를 확인했다.
|
||||||
|
- [ ] intentionally unused/admin-only API와 테스트 누락 API를 구분했다.
|
||||||
|
- [ ] `slow` route의 New Relic trace를 확인했다.
|
||||||
|
- [ ] `document-heavy` route의 Mongoose 반환량과 hydration 여부를 확인했다.
|
||||||
|
- [ ] 개선 전 리포트를 저장했다.
|
||||||
|
- [ ] 개선 후 같은 window/traffic 조건으로 리포트를 다시 생성했다.
|
||||||
|
|
||||||
|
## 운영 주의사항
|
||||||
|
|
||||||
|
- 계측은 기본 비활성화다. 운영에서 켤 때는 배포 환경변수로 명시한다.
|
||||||
|
- request body, auth header, cookie, raw Mongo filter/result는 기록하지 않는다.
|
||||||
|
- query별 custom event는 cardinality와 event volume이 커질 수 있어 필요할 때만 켠다.
|
||||||
|
- local NDJSON 파일은 장시간 부하 테스트 시 커질 수 있다. 테스트 전 파일을 지우거나
|
||||||
|
새 경로를 사용한다.
|
||||||
|
- New Relic custom event 기반 리포트는 계측 활성화 이후 쌓인 데이터만 포함한다.
|
||||||
|
- Transaction 기반 리포트는 기존 실측 latency를 볼 수 있지만 Mongoose 문서 반환량은
|
||||||
|
포함하지 않는다.
|
||||||
|
|
||||||
|
## 문제 해결
|
||||||
|
|
||||||
|
### 리포트가 전부 `no-samples`로 나온다
|
||||||
|
|
||||||
|
- local source라면 `--input` 경로가 `API_PERF_LOCAL_EVENT_FILE`과 같은지 확인한다.
|
||||||
|
- API 서버가 `API_PERF_OBSERVABILITY_ENABLED=true`로 실행됐는지 확인한다.
|
||||||
|
- API를 실제로 호출했는지 확인한다.
|
||||||
|
- New Relic source라면 `--since` window 안에 데이터가 있는지 확인한다.
|
||||||
|
- `newrelic-custom` source라면 계측 배포/환경변수 활성화 이후의 요청인지 확인한다.
|
||||||
|
|
||||||
|
New Relic에서 custom event 존재 여부를 먼저 확인한다.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
FROM ApiRequestPerformance SELECT count(*) SINCE 1 day ago
|
||||||
|
```
|
||||||
|
|
||||||
|
app name 필터까지 확인한다.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
FROM ApiRequestPerformance
|
||||||
|
SELECT count(*)
|
||||||
|
WHERE appName = 'lolympus'
|
||||||
|
SINCE 1 day ago
|
||||||
|
```
|
||||||
|
|
||||||
|
위 값이 0이면 `pnpm --filter api perf:report -- --source=newrelic-custom`도 전부
|
||||||
|
`no-samples`가 맞다. 이 경우 historical latency는 `newrelic-transaction` source로
|
||||||
|
보고, 상세 Mongoose 지표는 계측 활성화 후 다시 수집한다.
|
||||||
|
|
||||||
|
### New Relic 조회가 실패한다
|
||||||
|
|
||||||
|
- `NEW_RELIC_ACCOUNT_ID`가 숫자인지 확인한다.
|
||||||
|
- `NEW_RELIC_USER_API_KEY` 또는 `NEW_RELIC_API_KEY`가 NerdGraph 조회 권한을 갖는지 확인한다.
|
||||||
|
- `NEW_RELIC_APP_NAME`이 New Relic app name과 일치하는지 확인한다.
|
||||||
|
|
||||||
|
### Mongoose 지표가 비어 있다
|
||||||
|
|
||||||
|
- `newrelic-transaction` source는 Mongoose 지표를 제공하지 않는다.
|
||||||
|
- `newrelic-custom` 또는 `local` source를 사용한다.
|
||||||
|
- 계측 활성화 이후 발생한 request인지 확인한다.
|
||||||
|
|
||||||
|
## 관련 파일
|
||||||
|
|
||||||
|
- `apps/api/src/modules/global/observability/api-performance.interceptor.ts`
|
||||||
|
- `apps/api/src/modules/global/observability/performance-observer.ts`
|
||||||
|
- `apps/api/src/modules/global/observability/mongoose-performance.plugin.ts`
|
||||||
|
- `apps/api/src/scripts/api-performance-report.ts`
|
||||||
|
- `docs/specs/api-performance-observability.md`
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
# Spec: API Performance Observability And Full Route Report
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Add Phase 1 and Phase 2 support for API performance work:
|
||||||
|
|
||||||
|
- Phase 1 records request latency and Mongoose document-return metrics.
|
||||||
|
- Phase 2 generates a report for every API route, not only the slowest routes.
|
||||||
|
- New Relic Transaction or custom event data is the preferred production source.
|
||||||
|
- Local NDJSON files provide a staging/local fallback with query-level detail.
|
||||||
|
|
||||||
|
Success means an engineer can run representative traffic against the API and produce
|
||||||
|
a route-complete report that includes routes with no samples.
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
|
||||||
|
- NestJS API in `apps/api`
|
||||||
|
- Mongoose models created through `ConnectionService`
|
||||||
|
- New Relic Node agent, already configured by `apps/api/newrelic.cjs`
|
||||||
|
- Jest for unit tests
|
||||||
|
- `ts-node` for API maintenance scripts
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
- Build: `pnpm --filter api build`
|
||||||
|
- Test: `pnpm --filter api test`
|
||||||
|
- Generate local/New Relic report: `pnpm --filter api perf:report`
|
||||||
|
- Generate route manifest only: `pnpm --filter api perf:routes`
|
||||||
|
|
||||||
|
## Project Structure
|
||||||
|
|
||||||
|
- `apps/api/src/modules/global/observability/` contains request/query instrumentation.
|
||||||
|
- `apps/api/src/scripts/api-performance-report.ts` generates full API reports.
|
||||||
|
- `docs/specs/api-performance-observability.md` defines this behavior.
|
||||||
|
|
||||||
|
## Code Style
|
||||||
|
|
||||||
|
Instrumentation must be opt-in and defensive:
|
||||||
|
|
||||||
|
```ts
|
||||||
|
if (!isPerformanceObservabilityEnabled()) return;
|
||||||
|
recordApiPerformanceEvent({
|
||||||
|
type: 'api-request',
|
||||||
|
method,
|
||||||
|
route,
|
||||||
|
durationMs,
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not record raw request bodies, authorization headers, cookies, raw Mongo filters,
|
||||||
|
or raw query results.
|
||||||
|
|
||||||
|
## Testing Strategy
|
||||||
|
|
||||||
|
- Unit test pure metric aggregation and route discovery logic.
|
||||||
|
- Unit test request context aggregation without a database.
|
||||||
|
- Build the API to catch Nest and TypeScript integration issues.
|
||||||
|
- Production/staging validation uses New Relic Transaction data and optional local
|
||||||
|
NDJSON from representative traffic.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
- Always: keep instrumentation disabled unless explicitly enabled.
|
||||||
|
- Always: include all discovered controller routes in the report.
|
||||||
|
- Always: report `no-samples` for routes that did not receive traffic.
|
||||||
|
- Ask first: adding external load-test dependencies or CI performance gates.
|
||||||
|
- Never: log secrets, request bodies, auth headers, cookies, or raw Mongo filters.
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
- API request events include duration, status, route, query count, Mongoose duration,
|
||||||
|
returned document count, and hydrated document count.
|
||||||
|
- Mongoose query events include model, collection, operation, duration, returned
|
||||||
|
documents, hydrated documents, and lean status.
|
||||||
|
- New Relic receives route-level custom events when instrumentation is enabled.
|
||||||
|
- New Relic Transaction traces can be grouped by route through custom transaction names
|
||||||
|
and attributes.
|
||||||
|
- The report generator can use local NDJSON, New Relic custom events, or New Relic
|
||||||
|
Transaction duration data.
|
||||||
|
- The report lists every API route from controller source files.
|
||||||
Reference in new issue
Block a user