Add Phoenix OTLP tracing and LLM span attributes

This commit is contained in:
dahyeon committed 2026-05-18 21:09:08 -07:00
1 parent f15459bb99
commit 1d0be56842
11 files changed
+244 -17

No files matched your search

+29 -2
View File
@@ -461,6 +461,8 @@ npm test
| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 |
| `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path |
| `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 |
| `PHOENIX_OTLP_ENDPOINT` | unset | Phoenix OTLP trace endpoint |
| `PHOENIX_LOG_PROMPTS` | false | LLM prompt/completion content trace 기록 |
| `METRICS_PORT` | 9464 | Prometheus `/metrics` HTTP endpoint port |
| `SLACK_ALERT_WEBHOOK_URL` | unset | alert Slack incoming webhook URL |
| `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | Slack webhook POST timeout |
@@ -517,7 +519,32 @@ ALERT_THRESHOLD_DRIFT_PCT=15
예를 들어 onboarding latency alert와 moderation latency alert는 서로 throttle하지 않는다.
skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한다.
### 16.2 Prometheus scrape
### 16.2 Phoenix trace
LLM 호출, skill 실행, tool 실행 span은 OpenTelemetry trace로 남긴다.
`PHOENIX_OTLP_ENDPOINT`가 없으면 기존 설정을 따른다. 개발 환경 기본값은 trace exporter 비활성화이고, `OTEL_EXPORTER=stdout`이면 콘솔 exporter를 쓴다.
Phoenix를 로컬에서 볼 때:
```bash
docker run --rm -p 6006:6006 arizephoenix/phoenix:latest
```
Agent/worker env:
```env
PHOENIX_OTLP_ENDPOINT=http://localhost:6006/v1/traces
OTEL_SAMPLING_RATIO=1
PHOENIX_LOG_PROMPTS=false
```
`PHOENIX_OTLP_ENDPOINT`가 설정되면 `OTEL_EXPORTER=none`이어도 Phoenix OTLP exporter를 우선 사용한다.
Phoenix UI에서는 `service.name=lolympus-agent`, `gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.cost_usd`, `llm.token_count.prompt`, `llm.token_count.completion` 속성으로 LLM span을 필터링한다.
Prompt와 completion 본문은 기본적으로 기록하지 않는다.
디버깅 때문에 필요할 때만 `PHOENIX_LOG_PROMPTS=true`를 켠다. 이 경우 email, 전화번호, token/API key 형태의 값은 span 기록 전에 masking된다.
### 16.3 Prometheus scrape
Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다.
@@ -569,7 +596,7 @@ sum by (skill, stage) (rate(agent_validator_fail_total[5m]))
| eval drift | 15% 이하 |
| archive stalled | 3회 연속 실패 0건 |
### 16.3 Slack webhook 설정
### 16.4 Slack webhook 설정
Slack incoming webhook URL을 env로 설정한다.
+1
View File
@@ -12,6 +12,7 @@
"@infisical/sdk": "^5.0.0",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
"@opentelemetry/sdk-node": "^0.207.0",
"@opentelemetry/sdk-trace-base": "^2.1.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
+1
View File
@@ -63,6 +63,7 @@
"@infisical/sdk": "^5.0.0",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
"@opentelemetry/sdk-node": "^0.207.0",
"@opentelemetry/sdk-trace-base": "^2.1.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
+15
View File
@@ -5,6 +5,8 @@ import {
getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs,
getPhoenixLogPrompts,
getPhoenixOtlpEndpoint,
getSlackAlertWebhookUrl,
getSlackWebhookTimeoutMs,
getSupervisorLlmResponseEnabled,
@@ -19,6 +21,8 @@ const ENV_KEYS = [
"ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING",
"ALERT_THRESHOLD_P95_LATENCY_MS",
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
"PHOENIX_LOG_PROMPTS",
"PHOENIX_OTLP_ENDPOINT",
"SLACK_ALERT_WEBHOOK_URL",
"SLACK_WEBHOOK_TIMEOUT_MS",
"AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED",
@@ -62,6 +66,17 @@ describe("config", () => {
expect(getSlackWebhookTimeoutMs()).toBe(1234);
});
it("reads Phoenix tracing configuration", () => {
expect(getPhoenixOtlpEndpoint()).toBeUndefined();
expect(getPhoenixLogPrompts()).toBe(false);
vi.stubEnv("PHOENIX_OTLP_ENDPOINT", " http://localhost:6006/v1/traces ");
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
expect(getPhoenixOtlpEndpoint()).toBe("http://localhost:6006/v1/traces");
expect(getPhoenixLogPrompts()).toBe(true);
});
it("enables supervisor LLM user messages by default with env override", () => {
expect(getSupervisorLlmResponseEnabled()).toBe(true);
+8
View File
@@ -121,6 +121,14 @@ export function getOtelExporter(): "stdout" | "none" {
return getNodeEnv() === "dev" ? "none" : "stdout";
}
export function getPhoenixOtlpEndpoint(): string | undefined {
return process.env["PHOENIX_OTLP_ENDPOINT"]?.trim() || undefined;
}
export function getPhoenixLogPrompts(): boolean {
return getBoolean("PHOENIX_LOG_PROMPTS", false);
}
export function getRequestSubject(): string {
return (
process.env["AGENT_REQUEST_SUBJECT"]?.trim() || "lolympus-agent.request"
+83 -1
View File
@@ -1,9 +1,36 @@
import { afterEach, describe, expect, it, vi } from "vitest";
const llmSpanMock = vi.hoisted(() => ({
initialAttributes: [] as Array<Record<string, unknown> | undefined>,
setAttributes: vi.fn((attributes: Record<string, unknown>) => attributes),
}));
vi.mock("@obs/spans", () => ({
maskTraceContent: (content: string) =>
content
.replace(
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
"[redacted-email]",
)
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, "[redacted-phone]"),
withLlmSpan: vi.fn(
(
attributes: Record<string, unknown> | undefined,
fn: (span: { setAttributes: typeof llmSpanMock.setAttributes }) => unknown,
) => {
llmSpanMock.initialAttributes.push(attributes);
return Promise.resolve(fn({ setAttributes: llmSpanMock.setAttributes }));
},
),
}));
import { LlmClient } from "./llm-client.js";
import type { ChatCompletionResponse } from "./types.js";
function okResponse(content = "ok"): Response {
function okResponse(
content = "ok",
overrides: Partial<ChatCompletionResponse> = {},
): Response {
const payload: ChatCompletionResponse = {
choices: [
{
@@ -15,6 +42,7 @@ function okResponse(content = "ok"): Response {
},
],
model: "test-model",
...overrides,
};
return new Response(JSON.stringify(payload), {
@@ -35,7 +63,10 @@ function stringifyFetchUrl(input: string | URL | Request): string {
describe("LlmClient endpoint normalization", () => {
afterEach(() => {
llmSpanMock.initialAttributes.length = 0;
llmSpanMock.setAttributes.mockClear();
vi.unstubAllGlobals();
vi.unstubAllEnvs();
});
it("normalizes OpenRouter API base URLs to chat completions", async () => {
@@ -86,4 +117,55 @@ describe("LlmClient endpoint normalization", () => {
expect(result?.content).toBe("ok");
expect(urls).toEqual(["https://openrouter.ai/api/v1/chat/completions"]);
});
it("records OpenInference token, cost, and masked prompt attributes", async () => {
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
vi.stubGlobal(
"fetch",
vi.fn(() =>
Promise.resolve(
okResponse("reply to 010-9999-8888", {
id: "chatcmpl-test",
usage: {
completion_tokens: 50,
prompt_tokens: 100,
total_tokens: 150,
},
}),
),
),
);
const client = new LlmClient({
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
providerApiKeys: { openrouter: "test-key" },
});
const result = await client.chatCompletion(
[{ content: "hi a@example.com 010-1234-5678", role: "user" }],
{
maxTokens: 128,
modelOverride: "gpt-5.3-codex-spark",
temperature: 0.25,
},
);
expect(result?.content).toBe("reply to 010-9999-8888");
expect(llmSpanMock.initialAttributes.at(-1)).toMatchObject({
"gen_ai.request.max_tokens": 128,
"gen_ai.request.model": "gpt-5.3-codex-spark",
"gen_ai.request.temperature": 0.25,
"llm.input_messages.0.message.content":
"hi [redacted-email] [redacted-phone]",
"llm.model_name": "gpt-5.3-codex-spark",
});
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
expect.objectContaining({
"gen_ai.response.id": "chatcmpl-test",
"gen_ai.usage.cost_usd": 0.000045,
"llm.output_messages.0.message.content": "reply to [redacted-phone]",
"llm.token_count.completion": 50,
"llm.token_count.prompt": 100,
}),
);
});
});
+51 -11
View File
@@ -8,11 +8,12 @@ import type {
ProviderName,
} from "./types.js";
import { getContext, getRequestLogger } from "@obs/context";
import { withLlmSpan } from "@obs/spans";
import { maskTraceContent, withLlmSpan } from "@obs/spans";
import {
getLlmHealthTimeoutMs,
getLlmRequestTimeoutMs,
getLlmTemperature,
getPhoenixLogPrompts,
} from "../config.js";
import { LlmCircuitBreaker } from "./circuit.js";
import { getProviderAdapter } from "./providers/index.js";
@@ -84,6 +85,33 @@ function getRequestSignal(
return controller.signal;
}
function buildPromptTraceAttributes(
messages: readonly ChatMessage[],
): Record<string, string> {
return Object.fromEntries(
messages.flatMap((message, index) => [
[`llm.input_messages.${index}.message.role`, message.role],
[
`llm.input_messages.${index}.message.content`,
maskTraceContent(message.content),
],
]),
);
}
function buildCompletionTraceAttributes(
result: ChatCompletionResult | null,
): Record<string, string> {
if (!result) {
return {};
}
return {
"llm.output_messages.0.message.content": maskTraceContent(result.content),
"llm.output_messages.0.message.role": result.message.role,
};
}
export class LlmClient {
private readonly circuit: LlmCircuitBreaker;
private readonly endpoints: LlmEndpointConfig[];
@@ -182,11 +210,18 @@ export class LlmClient {
temperature: options?.temperature ?? getLlmTemperature(),
tools: options?.tools,
});
const shouldLogPrompts = getPhoenixLogPrompts();
const llmResult = await withLlmSpan(
{
...(shouldLogPrompts
? buildPromptTraceAttributes(request.messages)
: {}),
"gen_ai.request.max_tokens": request.max_tokens,
"gen_ai.request.model": request.model,
"gen_ai.request.temperature": request.temperature,
"gen_ai.system": provider.name,
"llm.model_name": request.model,
},
async (span) => {
const response = await fetch(endpoint.completionUrl, {
@@ -213,24 +248,29 @@ export class LlmClient {
const payload = (await response.json()) as ChatCompletionResponse;
const result = provider.toResult(payload, endpoint.completionUrl);
metrics.tokenCostUsd.observe(
estimateChatCost(
request.model,
payload.usage?.prompt_tokens,
payload.usage?.completion_tokens,
),
{
model: request.model,
skill: getContext()?.skill,
},
const costUsd = estimateChatCost(
request.model,
payload.usage?.prompt_tokens,
payload.usage?.completion_tokens,
);
metrics.tokenCostUsd.observe(costUsd, {
model: request.model,
skill: getContext()?.skill,
});
span.setAttributes({
...(shouldLogPrompts
? buildCompletionTraceAttributes(result)
: {}),
"gen_ai.response.finish_reasons": payload.choices
?.map((choice) => choice.finish_reason ?? "unknown")
.filter((reason): reason is string => Boolean(reason)),
"gen_ai.response.id": payload.id,
"gen_ai.usage.cost_usd": costUsd,
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
"llm.token_count.completion": payload.usage?.completion_tokens,
"llm.token_count.prompt": payload.usage?.prompt_tokens,
});
if (!result) {
+1
View File
@@ -30,6 +30,7 @@ export interface ChatCompletionRequest {
}
export interface ChatCompletionResponse {
id?: string | undefined;
model?: string;
choices?: Array<{
finish_reason?: string | null;
+20
View File
@@ -0,0 +1,20 @@
import { describe, expect, it } from "vitest";
import { maskTraceContent } from "./spans.js";
describe("maskTraceContent", () => {
it("redacts common PII and secrets before prompt tracing", () => {
const masked = maskTraceContent(
"email a@example.com phone 010-1234-5678 token=super-secret sk-testsecret123",
);
expect(masked).toContain("[redacted-email]");
expect(masked).toContain("[redacted-phone]");
expect(masked).toContain("token=[redacted-secret]");
expect(masked).toContain("[redacted-token]");
expect(masked).not.toContain("a@example.com");
expect(masked).not.toContain("010-1234-5678");
expect(masked).not.toContain("super-secret");
expect(masked).not.toContain("sk-testsecret123");
});
});
+22
View File
@@ -7,6 +7,8 @@ import {
import { getServiceName } from '../config.js';
const MAX_TRACE_CONTENT_CHARS = 2_000;
function compactAttributes(attributes?: Attributes): Attributes {
if (!attributes) {
return {};
@@ -17,6 +19,26 @@ function compactAttributes(attributes?: Attributes): Attributes {
);
}
export function maskTraceContent(content: string): string {
const masked = content
.replace(
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
'[redacted-email]',
)
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, '[redacted-phone]')
.replace(/\b(?:Bearer\s+|sk-|pk-)[A-Za-z0-9._-]{8,}\b/gu, '[redacted-token]')
.replace(
/\b(password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+/giu,
'$1=[redacted-secret]',
);
if (masked.length <= MAX_TRACE_CONTENT_CHARS) {
return masked;
}
return `${masked.slice(0, MAX_TRACE_CONTENT_CHARS)}...[truncated]`;
}
export async function withSpan<T>(
name: string,
attributes: Attributes | undefined,
+13 -3
View File
@@ -1,12 +1,18 @@
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import {
ConsoleSpanExporter,
ParentBasedSampler,
TraceIdRatioBasedSampler,
} from '@opentelemetry/sdk-trace-base';
import { getOtelExporter, getOtelSamplingRatio, getServiceName } from '../config.js';
import {
getOtelExporter,
getOtelSamplingRatio,
getPhoenixOtlpEndpoint,
getServiceName,
} from '../config.js';
import { getLogger } from './logger.js';
let sdk: NodeSDK | null = null;
@@ -18,7 +24,8 @@ export function initTracing(): void {
}
const exporterType = getOtelExporter();
if (exporterType === 'none') {
const phoenixEndpoint = getPhoenixOtlpEndpoint();
if (exporterType === 'none' && !phoenixEndpoint) {
const logger = getLogger();
logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init');
started = true;
@@ -28,7 +35,9 @@ export function initTracing(): void {
process.env['OTEL_SERVICE_NAME'] ??= getServiceName();
const logger = getLogger();
const exporter = new ConsoleSpanExporter();
const exporter = phoenixEndpoint
? new OTLPTraceExporter({ url: phoenixEndpoint })
: new ConsoleSpanExporter();
sdk = new NodeSDK({
instrumentations: [getNodeAutoInstrumentations()],
@@ -44,6 +53,7 @@ export function initTracing(): void {
logger.debug(
{
exporter: getOtelExporter(),
phoenixOtlpEndpoint: phoenixEndpoint,
samplingRatio: getOtelSamplingRatio(),
},
'[Tracing] Initialized',