Add Phoenix OTLP tracing and LLM span attributes

This commit is contained in:
dahyeon committed 2026-05-18 21:09:08 -07:00
1 parent f15459bb99
commit 1d0be56842
11 files changed
+244 -17

No files matched your search

+15
View File
@@ -5,6 +5,8 @@ import {
getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs,
getPhoenixLogPrompts,
getPhoenixOtlpEndpoint,
getSlackAlertWebhookUrl,
getSlackWebhookTimeoutMs,
getSupervisorLlmResponseEnabled,
@@ -19,6 +21,8 @@ const ENV_KEYS = [
"ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING",
"ALERT_THRESHOLD_P95_LATENCY_MS",
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
"PHOENIX_LOG_PROMPTS",
"PHOENIX_OTLP_ENDPOINT",
"SLACK_ALERT_WEBHOOK_URL",
"SLACK_WEBHOOK_TIMEOUT_MS",
"AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED",
@@ -62,6 +66,17 @@ describe("config", () => {
expect(getSlackWebhookTimeoutMs()).toBe(1234);
});
it("reads Phoenix tracing configuration", () => {
expect(getPhoenixOtlpEndpoint()).toBeUndefined();
expect(getPhoenixLogPrompts()).toBe(false);
vi.stubEnv("PHOENIX_OTLP_ENDPOINT", " http://localhost:6006/v1/traces ");
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
expect(getPhoenixOtlpEndpoint()).toBe("http://localhost:6006/v1/traces");
expect(getPhoenixLogPrompts()).toBe(true);
});
it("enables supervisor LLM user messages by default with env override", () => {
expect(getSupervisorLlmResponseEnabled()).toBe(true);
+8
View File
@@ -121,6 +121,14 @@ export function getOtelExporter(): "stdout" | "none" {
return getNodeEnv() === "dev" ? "none" : "stdout";
}
export function getPhoenixOtlpEndpoint(): string | undefined {
return process.env["PHOENIX_OTLP_ENDPOINT"]?.trim() || undefined;
}
export function getPhoenixLogPrompts(): boolean {
return getBoolean("PHOENIX_LOG_PROMPTS", false);
}
export function getRequestSubject(): string {
return (
process.env["AGENT_REQUEST_SUBJECT"]?.trim() || "lolympus-agent.request"
+83 -1
View File
@@ -1,9 +1,36 @@
import { afterEach, describe, expect, it, vi } from "vitest";
const llmSpanMock = vi.hoisted(() => ({
initialAttributes: [] as Array<Record<string, unknown> | undefined>,
setAttributes: vi.fn((attributes: Record<string, unknown>) => attributes),
}));
vi.mock("@obs/spans", () => ({
maskTraceContent: (content: string) =>
content
.replace(
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
"[redacted-email]",
)
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, "[redacted-phone]"),
withLlmSpan: vi.fn(
(
attributes: Record<string, unknown> | undefined,
fn: (span: { setAttributes: typeof llmSpanMock.setAttributes }) => unknown,
) => {
llmSpanMock.initialAttributes.push(attributes);
return Promise.resolve(fn({ setAttributes: llmSpanMock.setAttributes }));
},
),
}));
import { LlmClient } from "./llm-client.js";
import type { ChatCompletionResponse } from "./types.js";
function okResponse(content = "ok"): Response {
function okResponse(
content = "ok",
overrides: Partial<ChatCompletionResponse> = {},
): Response {
const payload: ChatCompletionResponse = {
choices: [
{
@@ -15,6 +42,7 @@ function okResponse(content = "ok"): Response {
},
],
model: "test-model",
...overrides,
};
return new Response(JSON.stringify(payload), {
@@ -35,7 +63,10 @@ function stringifyFetchUrl(input: string | URL | Request): string {
describe("LlmClient endpoint normalization", () => {
afterEach(() => {
llmSpanMock.initialAttributes.length = 0;
llmSpanMock.setAttributes.mockClear();
vi.unstubAllGlobals();
vi.unstubAllEnvs();
});
it("normalizes OpenRouter API base URLs to chat completions", async () => {
@@ -86,4 +117,55 @@ describe("LlmClient endpoint normalization", () => {
expect(result?.content).toBe("ok");
expect(urls).toEqual(["https://openrouter.ai/api/v1/chat/completions"]);
});
it("records OpenInference token, cost, and masked prompt attributes", async () => {
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
vi.stubGlobal(
"fetch",
vi.fn(() =>
Promise.resolve(
okResponse("reply to 010-9999-8888", {
id: "chatcmpl-test",
usage: {
completion_tokens: 50,
prompt_tokens: 100,
total_tokens: 150,
},
}),
),
),
);
const client = new LlmClient({
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
providerApiKeys: { openrouter: "test-key" },
});
const result = await client.chatCompletion(
[{ content: "hi a@example.com 010-1234-5678", role: "user" }],
{
maxTokens: 128,
modelOverride: "gpt-5.3-codex-spark",
temperature: 0.25,
},
);
expect(result?.content).toBe("reply to 010-9999-8888");
expect(llmSpanMock.initialAttributes.at(-1)).toMatchObject({
"gen_ai.request.max_tokens": 128,
"gen_ai.request.model": "gpt-5.3-codex-spark",
"gen_ai.request.temperature": 0.25,
"llm.input_messages.0.message.content":
"hi [redacted-email] [redacted-phone]",
"llm.model_name": "gpt-5.3-codex-spark",
});
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
expect.objectContaining({
"gen_ai.response.id": "chatcmpl-test",
"gen_ai.usage.cost_usd": 0.000045,
"llm.output_messages.0.message.content": "reply to [redacted-phone]",
"llm.token_count.completion": 50,
"llm.token_count.prompt": 100,
}),
);
});
});
+51 -11
View File
@@ -8,11 +8,12 @@ import type {
ProviderName,
} from "./types.js";
import { getContext, getRequestLogger } from "@obs/context";
import { withLlmSpan } from "@obs/spans";
import { maskTraceContent, withLlmSpan } from "@obs/spans";
import {
getLlmHealthTimeoutMs,
getLlmRequestTimeoutMs,
getLlmTemperature,
getPhoenixLogPrompts,
} from "../config.js";
import { LlmCircuitBreaker } from "./circuit.js";
import { getProviderAdapter } from "./providers/index.js";
@@ -84,6 +85,33 @@ function getRequestSignal(
return controller.signal;
}
function buildPromptTraceAttributes(
messages: readonly ChatMessage[],
): Record<string, string> {
return Object.fromEntries(
messages.flatMap((message, index) => [
[`llm.input_messages.${index}.message.role`, message.role],
[
`llm.input_messages.${index}.message.content`,
maskTraceContent(message.content),
],
]),
);
}
function buildCompletionTraceAttributes(
result: ChatCompletionResult | null,
): Record<string, string> {
if (!result) {
return {};
}
return {
"llm.output_messages.0.message.content": maskTraceContent(result.content),
"llm.output_messages.0.message.role": result.message.role,
};
}
export class LlmClient {
private readonly circuit: LlmCircuitBreaker;
private readonly endpoints: LlmEndpointConfig[];
@@ -182,11 +210,18 @@ export class LlmClient {
temperature: options?.temperature ?? getLlmTemperature(),
tools: options?.tools,
});
const shouldLogPrompts = getPhoenixLogPrompts();
const llmResult = await withLlmSpan(
{
...(shouldLogPrompts
? buildPromptTraceAttributes(request.messages)
: {}),
"gen_ai.request.max_tokens": request.max_tokens,
"gen_ai.request.model": request.model,
"gen_ai.request.temperature": request.temperature,
"gen_ai.system": provider.name,
"llm.model_name": request.model,
},
async (span) => {
const response = await fetch(endpoint.completionUrl, {
@@ -213,24 +248,29 @@ export class LlmClient {
const payload = (await response.json()) as ChatCompletionResponse;
const result = provider.toResult(payload, endpoint.completionUrl);
metrics.tokenCostUsd.observe(
estimateChatCost(
request.model,
payload.usage?.prompt_tokens,
payload.usage?.completion_tokens,
),
{
model: request.model,
skill: getContext()?.skill,
},
const costUsd = estimateChatCost(
request.model,
payload.usage?.prompt_tokens,
payload.usage?.completion_tokens,
);
metrics.tokenCostUsd.observe(costUsd, {
model: request.model,
skill: getContext()?.skill,
});
span.setAttributes({
...(shouldLogPrompts
? buildCompletionTraceAttributes(result)
: {}),
"gen_ai.response.finish_reasons": payload.choices
?.map((choice) => choice.finish_reason ?? "unknown")
.filter((reason): reason is string => Boolean(reason)),
"gen_ai.response.id": payload.id,
"gen_ai.usage.cost_usd": costUsd,
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
"llm.token_count.completion": payload.usage?.completion_tokens,
"llm.token_count.prompt": payload.usage?.prompt_tokens,
});
if (!result) {
+1
View File
@@ -30,6 +30,7 @@ export interface ChatCompletionRequest {
}
export interface ChatCompletionResponse {
id?: string | undefined;
model?: string;
choices?: Array<{
finish_reason?: string | null;
+20
View File
@@ -0,0 +1,20 @@
import { describe, expect, it } from "vitest";
import { maskTraceContent } from "./spans.js";
describe("maskTraceContent", () => {
it("redacts common PII and secrets before prompt tracing", () => {
const masked = maskTraceContent(
"email a@example.com phone 010-1234-5678 token=super-secret sk-testsecret123",
);
expect(masked).toContain("[redacted-email]");
expect(masked).toContain("[redacted-phone]");
expect(masked).toContain("token=[redacted-secret]");
expect(masked).toContain("[redacted-token]");
expect(masked).not.toContain("a@example.com");
expect(masked).not.toContain("010-1234-5678");
expect(masked).not.toContain("super-secret");
expect(masked).not.toContain("sk-testsecret123");
});
});
+22
View File
@@ -7,6 +7,8 @@ import {
import { getServiceName } from '../config.js';
const MAX_TRACE_CONTENT_CHARS = 2_000;
function compactAttributes(attributes?: Attributes): Attributes {
if (!attributes) {
return {};
@@ -17,6 +19,26 @@ function compactAttributes(attributes?: Attributes): Attributes {
);
}
export function maskTraceContent(content: string): string {
const masked = content
.replace(
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
'[redacted-email]',
)
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, '[redacted-phone]')
.replace(/\b(?:Bearer\s+|sk-|pk-)[A-Za-z0-9._-]{8,}\b/gu, '[redacted-token]')
.replace(
/\b(password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+/giu,
'$1=[redacted-secret]',
);
if (masked.length <= MAX_TRACE_CONTENT_CHARS) {
return masked;
}
return `${masked.slice(0, MAX_TRACE_CONTENT_CHARS)}...[truncated]`;
}
export async function withSpan<T>(
name: string,
attributes: Attributes | undefined,
+13 -3
View File
@@ -1,12 +1,18 @@
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import {
ConsoleSpanExporter,
ParentBasedSampler,
TraceIdRatioBasedSampler,
} from '@opentelemetry/sdk-trace-base';
import { getOtelExporter, getOtelSamplingRatio, getServiceName } from '../config.js';
import {
getOtelExporter,
getOtelSamplingRatio,
getPhoenixOtlpEndpoint,
getServiceName,
} from '../config.js';
import { getLogger } from './logger.js';
let sdk: NodeSDK | null = null;
@@ -18,7 +24,8 @@ export function initTracing(): void {
}
const exporterType = getOtelExporter();
if (exporterType === 'none') {
const phoenixEndpoint = getPhoenixOtlpEndpoint();
if (exporterType === 'none' && !phoenixEndpoint) {
const logger = getLogger();
logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init');
started = true;
@@ -28,7 +35,9 @@ export function initTracing(): void {
process.env['OTEL_SERVICE_NAME'] ??= getServiceName();
const logger = getLogger();
const exporter = new ConsoleSpanExporter();
const exporter = phoenixEndpoint
? new OTLPTraceExporter({ url: phoenixEndpoint })
: new ConsoleSpanExporter();
sdk = new NodeSDK({
instrumentations: [getNodeAutoInstrumentations()],
@@ -44,6 +53,7 @@ export function initTracing(): void {
logger.debug(
{
exporter: getOtelExporter(),
phoenixOtlpEndpoint: phoenixEndpoint,
samplingRatio: getOtelSamplingRatio(),
},
'[Tracing] Initialized',