import { existsSync } from 'node:fs'; import { readdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import process from 'node:process'; import type { ApiPerformanceEvent, ApiRequestPerformanceEvent, } from '../modules/global/observability/performance-observer'; export type ReportSource = | 'local' | 'newrelic-custom' | 'newrelic-transaction'; export interface ApiRoute { method: string; route: string; controllerFile: string; } export interface EndpointReportRow { method: string; route: string; controllerFile?: string; source: ReportSource | 'none'; samples: number; avgMs: number | null; p50Ms: number | null; p95Ms: number | null; p99Ms: number | null; maxMs: number | null; errorRatePercentage: number | null; queryCountP95: number | null; mongooseDurationP95Ms: number | null; docsReturnedP95: number | null; hydratedDocsReturnedP95: number | null; status: 'ok' | 'slow' | 'document-heavy' | 'no-samples'; } interface ReportOptions { source: ReportSource; rootDir: string; inputFile: string; outputFile?: string; format: 'markdown' | 'json'; p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; since: string; newRelicAccountId?: string; newRelicApiKey?: string; newRelicAppName: string; routesOnly: boolean; } interface NewRelicNrqlResponse { data?: { actor?: { account?: { nrql?: { results?: Record[]; }; }; }; }; errors?: Array<{ message?: string }>; } const HTTP_DECORATOR_TO_METHOD: Record = { All: 'ALL', Delete: 'DELETE', Get: 'GET', Head: 'HEAD', Options: 'OPTIONS', Patch: 'PATCH', Post: 'POST', Put: 'PUT', }; export async function discoverApiRoutes(rootDir: string): Promise { const files = await listControllerFiles(rootDir); const routes = new Map(); for (const file of files) { const source = await readFile(file, 'utf8'); const basePath = extractDecoratorStringArgument(source, 'Controller'); if (basePath === undefined) continue; const methodDecoratorRegex = /@(Get|Post|Patch|Put|Delete|Head|Options|All)\s*\(([\s\S]*?)\)/g; for (const match of source.matchAll(methodDecoratorRegex)) { const method = HTTP_DECORATOR_TO_METHOD[match[1]]; const route = normalizeRoute( joinRouteParts(basePath, extractFirstStringLiteral(match[2]) ?? ''), ); const key = routeKey(method, route); routes.set(key, { method, route, controllerFile: path.relative(rootDir, file), }); } } return [...routes.values()].sort( (left, right) => left.route.localeCompare(right.route) || left.method.localeCompare(right.method), ); } export async function readLocalPerformanceEvents( inputFile: string, ): Promise { if (!existsSync(inputFile)) return []; const content = await readFile(inputFile, 'utf8'); return content .split(/\r?\n/) .filter(Boolean) .flatMap((line) => { try { return [JSON.parse(line) as ApiPerformanceEvent]; } catch { return []; } }); } export function buildEndpointReportFromLocalEvents(params: { routes: ApiRoute[]; events: ApiPerformanceEvent[]; p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; }): EndpointReportRow[] { const requestsByRoute = new Map(); for (const event of params.events) { if (event.type !== 'api-request') continue; const key = routeKey(event.method, event.route); requestsByRoute.set(key, [...(requestsByRoute.get(key) ?? []), event]); } const rowsByKey = new Map(); for (const route of params.routes) { const events = requestsByRoute.get(routeKey(route.method, route.route)) ?? []; rowsByKey.set( routeKey(route.method, route.route), summarizeRequestEvents(route, events, { source: 'local', p95BudgetMs: params.p95BudgetMs, docsReturnedP95Budget: params.docsReturnedP95Budget, hydratedDocsReturnedP95Budget: params.hydratedDocsReturnedP95Budget, }), ); } for (const [key, events] of requestsByRoute.entries()) { if (rowsByKey.has(key)) continue; const first = events[0]; rowsByKey.set( key, summarizeRequestEvents( { method: first.method, route: first.route, controllerFile: 'observed-only', }, events, { source: 'local', p95BudgetMs: params.p95BudgetMs, docsReturnedP95Budget: params.docsReturnedP95Budget, hydratedDocsReturnedP95Budget: params.hydratedDocsReturnedP95Budget, }, ), ); } return [...rowsByKey.values()].sort(compareReportRows); } export async function fetchNewRelicEndpointRows(params: { source: Exclude; accountId: string; apiKey: string; appName: string; since: string; p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; }): Promise { const nrql = params.source === 'newrelic-custom' ? buildNewRelicCustomEventNrql(params.appName, params.since) : buildNewRelicTransactionNrql(params.appName, params.since); const response = await fetch('https://api.newrelic.com/graphql', { method: 'POST', headers: { 'Api-Key': params.apiKey, 'Content-Type': 'application/json', }, body: JSON.stringify({ query: `{ actor { account(id: ${Number(params.accountId)}) { nrql(query: ${JSON.stringify(nrql)}) { results } } } }`, }), }); if (!response.ok) { throw new Error( `New Relic query failed: ${response.status} ${response.statusText}`, ); } const json = (await response.json()) as NewRelicNrqlResponse; if (json.errors?.length) { throw new Error( `New Relic query failed: ${json.errors .map((error) => error.message ?? 'unknown error') .join('; ')}`, ); } const results = json.data?.actor?.account?.nrql?.results ?? []; return results .map((result) => params.source === 'newrelic-custom' ? newRelicCustomResultToRow(result, params) : newRelicTransactionResultToRow(result, params), ) .filter((row): row is EndpointReportRow => row !== null) .sort(compareReportRows); } export function mergeRowsWithManifest(params: { routes: ApiRoute[]; measuredRows: EndpointReportRow[]; fallbackSource: ReportSource; p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; }): EndpointReportRow[] { const rowsByKey = new Map( params.measuredRows.map((row) => [routeKey(row.method, row.route), row]), ); for (const route of params.routes) { const key = routeKey(route.method, route.route); if (rowsByKey.has(key)) { rowsByKey.set(key, { ...rowsByKey.get(key)!, controllerFile: route.controllerFile, }); continue; } rowsByKey.set(key, { method: route.method, route: route.route, controllerFile: route.controllerFile, source: 'none', samples: 0, avgMs: null, p50Ms: null, p95Ms: null, p99Ms: null, maxMs: null, errorRatePercentage: null, queryCountP95: null, mongooseDurationP95Ms: null, docsReturnedP95: null, hydratedDocsReturnedP95: null, status: 'no-samples', }); } return [...rowsByKey.values()].sort(compareReportRows); } export function formatMarkdownReport(params: { rows: EndpointReportRow[]; source: ReportSource; since: string; notes?: string[]; }): string { const sampled = params.rows.filter((row) => row.samples > 0); const noSamples = params.rows.length - sampled.length; const slow = sampled.filter((row) => row.status === 'slow').length; const documentHeavy = sampled.filter( (row) => row.status === 'document-heavy', ).length; const lines = [ '# API Performance Report', '', `- Source: ${params.source}`, `- Window: ${params.since}`, `- Routes: ${params.rows.length}`, `- Sampled routes: ${sampled.length}`, `- No-sample routes: ${noSamples}`, `- Slow routes: ${slow}`, `- Document-heavy routes: ${documentHeavy}`, ]; if (params.notes?.length) { lines.push('', '## Notes', ''); for (const note of params.notes) lines.push(`- ${note}`); } lines.push( '', '| Method | Route | Samples | p95 ms | p99 ms | Avg ms | Error % | Query p95 | Docs p95 | Hydrated p95 | Status |', '|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|', ); for (const row of params.rows) { lines.push( [ `| ${row.method}`, markdownCell(row.route), String(row.samples), formatNumber(row.p95Ms), formatNumber(row.p99Ms), formatNumber(row.avgMs), formatNumber(row.errorRatePercentage), formatNumber(row.queryCountP95), formatNumber(row.docsReturnedP95), formatNumber(row.hydratedDocsReturnedP95), `${row.status} |`, ].join(' | '), ); } return `${lines.join('\n')}\n`; } export function buildNewRelicCustomEventNrql( appName: string, since: string, ): string { return [ 'FROM ApiRequestPerformance', 'SELECT count(*) AS samples,', 'average(durationMs) AS avgDurationMs,', 'percentile(durationMs, 50, 95, 99) AS durationPercentilesMs,', 'max(durationMs) AS maxDurationMs,', 'percentage(count(*), WHERE isError IS true) AS errorRatePercentage,', 'percentile(queryCount, 95) AS queryCountPercentiles,', 'percentile(docsReturned, 95) AS docsReturnedPercentiles,', 'percentile(hydratedDocsReturned, 95) AS hydratedDocsReturnedPercentiles,', 'percentile(mongooseDurationMs, 95) AS mongooseDurationPercentiles', `WHERE appName = '${escapeNrqlString(appName)}'`, 'FACET method, route', `SINCE ${since}`, 'LIMIT MAX', ].join(' '); } export function buildNewRelicTransactionNrql( appName: string, since: string, ): string { return [ 'FROM Transaction', 'SELECT count(*) AS samples,', 'average(duration) AS avgDurationSec,', 'percentile(duration, 50, 95, 99) AS durationPercentilesSec,', 'max(duration) AS maxDurationSec,', 'percentage(count(*), WHERE error IS true) AS errorRatePercentage', `WHERE appName = '${escapeNrqlString(appName)}'`, 'FACET name', `SINCE ${since}`, 'LIMIT MAX', ].join(' '); } function summarizeRequestEvents( route: ApiRoute, events: ApiRequestPerformanceEvent[], options: { source: ReportSource; p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; }, ): EndpointReportRow { const row: EndpointReportRow = { method: route.method, route: route.route, controllerFile: route.controllerFile, source: events.length > 0 ? options.source : 'none', samples: events.length, avgMs: average(events.map((event) => event.durationMs)), p50Ms: percentile( events.map((event) => event.durationMs), 50, ), p95Ms: percentile( events.map((event) => event.durationMs), 95, ), p99Ms: percentile( events.map((event) => event.durationMs), 99, ), maxMs: max(events.map((event) => event.durationMs)), errorRatePercentage: events.length > 0 ? (events.filter((event) => event.isError).length / events.length) * 100 : null, queryCountP95: percentile( events.map((event) => event.queryCount), 95, ), mongooseDurationP95Ms: percentile( events.map((event) => event.mongooseDurationMs), 95, ), docsReturnedP95: percentile( events.map((event) => event.docsReturned), 95, ), hydratedDocsReturnedP95: percentile( events.map((event) => event.hydratedDocsReturned), 95, ), status: 'ok', }; row.status = classifyRow(row, options); return row; } function newRelicCustomResultToRow( result: Record, options: { source: ReportSource; p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; }, ): EndpointReportRow | null { const facet = result.facet; const method = Array.isArray(facet) ? String(facet[0] ?? '') : undefined; const route = Array.isArray(facet) ? String(facet[1] ?? '') : undefined; if (!method || !route) return null; const row: EndpointReportRow = { method, route: normalizeRoute(route), source: 'newrelic-custom', samples: numberField(result.samples), avgMs: numberOrNull(result.avgDurationMs), p50Ms: percentileField(result.durationPercentilesMs, 50), p95Ms: percentileField(result.durationPercentilesMs, 95), p99Ms: percentileField(result.durationPercentilesMs, 99), maxMs: numberOrNull(result.maxDurationMs), errorRatePercentage: numberOrNull(result.errorRatePercentage), queryCountP95: percentileField(result.queryCountPercentiles, 95), mongooseDurationP95Ms: percentileField( result.mongooseDurationPercentiles, 95, ), docsReturnedP95: percentileField(result.docsReturnedPercentiles, 95), hydratedDocsReturnedP95: percentileField( result.hydratedDocsReturnedPercentiles, 95, ), status: 'ok', }; row.status = classifyRow(row, options); return row; } function newRelicTransactionResultToRow( result: Record, options: { source: ReportSource; p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; }, ): EndpointReportRow | null { const parsed = parseTransactionName(String(result.facet ?? '')); if (!parsed) return null; const row: EndpointReportRow = { method: parsed.method, route: parsed.route, source: 'newrelic-transaction', samples: numberField(result.samples), avgMs: secondsToMs(result.avgDurationSec), p50Ms: secondsToMs(percentileField(result.durationPercentilesSec, 50)), p95Ms: secondsToMs(percentileField(result.durationPercentilesSec, 95)), p99Ms: secondsToMs(percentileField(result.durationPercentilesSec, 99)), maxMs: secondsToMs(result.maxDurationSec), errorRatePercentage: numberOrNull(result.errorRatePercentage), queryCountP95: null, mongooseDurationP95Ms: null, docsReturnedP95: null, hydratedDocsReturnedP95: null, status: 'ok', }; row.status = classifyRow(row, options); return row; } function parseTransactionName( name: string, ): { method: string; route: string } | null { const match = name.match( /\b(GET|POST|PATCH|PUT|DELETE|HEAD|OPTIONS|ALL)[\s/]+(\/.*)$/i, ); if (!match) return null; return { method: match[1].toUpperCase(), route: normalizeRoute(match[2]), }; } function classifyRow( row: EndpointReportRow, options: { p95BudgetMs: number; docsReturnedP95Budget: number; hydratedDocsReturnedP95Budget: number; }, ): EndpointReportRow['status'] { if (row.samples === 0) return 'no-samples'; if (row.p95Ms !== null && row.p95Ms > options.p95BudgetMs) return 'slow'; if ( (row.docsReturnedP95 !== null && row.docsReturnedP95 > options.docsReturnedP95Budget) || (row.hydratedDocsReturnedP95 !== null && row.hydratedDocsReturnedP95 > options.hydratedDocsReturnedP95Budget) ) { return 'document-heavy'; } return 'ok'; } async function listControllerFiles(rootDir: string): Promise { const entries = await readdir(rootDir, { withFileTypes: true }); const files = await Promise.all( entries.map(async (entry) => { const fullPath = path.join(rootDir, entry.name); if (entry.isDirectory()) return listControllerFiles(fullPath); return entry.isFile() && entry.name.endsWith('.controller.ts') ? [fullPath] : []; }), ); return files.flat(); } function extractDecoratorStringArgument( source: string, decoratorName: string, ): string | undefined { const regex = new RegExp(`@${decoratorName}\\s*\\(([\\s\\S]*?)\\)`); const match = source.match(regex); if (!match) return undefined; return extractFirstStringLiteral(match[1]) ?? ''; } function extractFirstStringLiteral(input: string): string | undefined { const match = input.match(/['"`]([^'"`]*)['"`]/); return match?.[1]; } function joinRouteParts(basePath: string, routePath: string): string { return [basePath, routePath] .filter((part) => part !== '') .join('/') .replace(/\/+/g, '/'); } function normalizeRoute(route: string): string { const normalized = route.startsWith('/') ? route : `/${route}`; return normalized.replace(/\/+/g, '/').replace(/\/$/, '') || '/'; } function routeKey(method: string, route: string): string { return `${method.toUpperCase()} ${normalizeRoute(route)}`; } function compareReportRows(left: EndpointReportRow, right: EndpointReportRow) { return ( left.route.localeCompare(right.route) || left.method.localeCompare(right.method) ); } function percentile(values: number[], percentileValue: number): number | null { if (values.length === 0) return null; const sorted = [...values].sort((left, right) => left - right); const index = Math.ceil((percentileValue / 100) * sorted.length) - 1; return sorted[Math.max(0, Math.min(sorted.length - 1, index))]; } function average(values: number[]): number | null { if (values.length === 0) return null; return values.reduce((acc, value) => acc + value, 0) / values.length; } function max(values: number[]): number | null { if (values.length === 0) return null; return Math.max(...values); } function percentileField(value: unknown, percentileValue: number): number | null { if (!value || typeof value !== 'object') return null; const map = value as Record; return numberOrNull(map[String(percentileValue)] ?? map[percentileValue]); } function numberField(value: unknown): number { return numberOrNull(value) ?? 0; } function numberOrNull(value: unknown): number | null { return typeof value === 'number' && Number.isFinite(value) ? value : null; } function secondsToMs(value: unknown): number | null { const seconds = numberOrNull(value); return seconds === null ? null : seconds * 1000; } function formatNumber(value: number | null): string { return value === null ? '-' : value.toFixed(value >= 100 ? 0 : 1); } function markdownCell(value: string): string { return value.replace(/\|/g, '\\|'); } function escapeNrqlString(value: string): string { return value.replace(/'/g, "\\'"); } function parseArgs(argv: string[]): ReportOptions { const args = new Map(); for (const arg of argv) { if (!arg.startsWith('--')) continue; const [key, rawValue] = arg.slice(2).split('=', 2); args.set(key, rawValue ?? true); } const source = stringArg(args, 'source', defaultReportSource()) as ReportSource; return { source, rootDir: path.resolve( stringArg(args, 'root-dir', path.join(process.cwd(), 'src')), ), inputFile: path.resolve( stringArg( args, 'input', process.env.API_PERF_LOCAL_EVENT_FILE ?? '/tmp/lolympus-api-performance.ndjson', ), ), outputFile: optionalStringArg(args, 'output'), format: stringArg(args, 'format', 'markdown') as 'markdown' | 'json', p95BudgetMs: numberArg(args, 'p95-budget-ms', 300), docsReturnedP95Budget: numberArg(args, 'docs-returned-p95-budget', 500), hydratedDocsReturnedP95Budget: numberArg( args, 'hydrated-docs-returned-p95-budget', 0, ), since: stringArg(args, 'since', '1 day ago'), newRelicAccountId: optionalStringArg(args, 'new-relic-account-id') ?? process.env.NEW_RELIC_ACCOUNT_ID, newRelicApiKey: optionalStringArg(args, 'new-relic-api-key') ?? process.env.NEW_RELIC_USER_API_KEY ?? process.env.NEW_RELIC_API_KEY, newRelicAppName: optionalStringArg(args, 'new-relic-app-name') ?? process.env.NEW_RELIC_APP_NAME ?? 'lolympus', routesOnly: args.has('routes-only'), }; } function defaultReportSource(): ReportSource { return process.env.NEW_RELIC_ACCOUNT_ID && (process.env.NEW_RELIC_USER_API_KEY || process.env.NEW_RELIC_API_KEY) ? 'newrelic-custom' : 'local'; } function stringArg( args: Map, name: string, defaultValue: string, ): string { const value = args.get(name); return typeof value === 'string' && value !== '' ? value : defaultValue; } function optionalStringArg( args: Map, name: string, ): string | undefined { const value = args.get(name); return typeof value === 'string' && value !== '' ? value : undefined; } function numberArg( args: Map, name: string, defaultValue: number, ): number { const value = Number(stringArg(args, name, String(defaultValue))); return Number.isFinite(value) ? value : defaultValue; } async function buildReport(options: ReportOptions): Promise { const routes = await discoverApiRoutes(options.rootDir); if (options.routesOnly) { return `${JSON.stringify(routes, null, 2)}\n`; } const measuredRows = options.source === 'local' ? buildEndpointReportFromLocalEvents({ routes, events: await readLocalPerformanceEvents(options.inputFile), p95BudgetMs: options.p95BudgetMs, docsReturnedP95Budget: options.docsReturnedP95Budget, hydratedDocsReturnedP95Budget: options.hydratedDocsReturnedP95Budget, }) : await fetchNewRelicEndpointRows({ source: options.source, accountId: requireOption( options.newRelicAccountId, 'NEW_RELIC_ACCOUNT_ID', ), apiKey: requireOption(options.newRelicApiKey, 'NEW_RELIC_API_KEY'), appName: options.newRelicAppName, since: options.since, p95BudgetMs: options.p95BudgetMs, docsReturnedP95Budget: options.docsReturnedP95Budget, hydratedDocsReturnedP95Budget: options.hydratedDocsReturnedP95Budget, }); const notes = options.source === 'newrelic-custom' && measuredRows.length === 0 ? [ 'No ApiRequestPerformance custom events were found for this window and appName.', 'Custom events are created only after the API runs with API_PERF_OBSERVABILITY_ENABLED=true and API_PERF_NEW_RELIC_REQUEST_EVENTS_ENABLED=true.', 'Use --source=newrelic-transaction for historical latency before custom instrumentation was enabled.', ] : []; const rows = mergeRowsWithManifest({ routes, measuredRows, fallbackSource: options.source, p95BudgetMs: options.p95BudgetMs, docsReturnedP95Budget: options.docsReturnedP95Budget, hydratedDocsReturnedP95Budget: options.hydratedDocsReturnedP95Budget, }); return options.format === 'json' ? `${JSON.stringify(rows, null, 2)}\n` : formatMarkdownReport({ rows, source: options.source, since: options.since, notes, }); } function requireOption(value: string | undefined, name: string): string { if (value) return value; throw new Error(`${name} is required for New Relic report sources.`); } async function main() { const options = parseArgs(process.argv.slice(2)); const report = await buildReport(options); if (options.outputFile) { await writeFile(path.resolve(options.outputFile), report); return; } process.stdout.write(report); } if (require.main === module) { void main().catch((error: unknown) => { process.stderr.write( `${error instanceof Error ? error.message : String(error)}\n`, ); process.exitCode = 1; }); }