Add Phoenix OTLP tracing and LLM span attributes
This commit is contained in:
1 parent
f15459bb99
commit
1d0be56842
11 files changed
+244
-17
No files matched your search
@@ -461,6 +461,8 @@ npm test
|
||||
| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 |
|
||||
| `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path |
|
||||
| `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 |
|
||||
| `PHOENIX_OTLP_ENDPOINT` | unset | Phoenix OTLP trace endpoint |
|
||||
| `PHOENIX_LOG_PROMPTS` | false | LLM prompt/completion content trace 기록 |
|
||||
| `METRICS_PORT` | 9464 | Prometheus `/metrics` HTTP endpoint port |
|
||||
| `SLACK_ALERT_WEBHOOK_URL` | unset | alert Slack incoming webhook URL |
|
||||
| `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | Slack webhook POST timeout |
|
||||
@@ -517,7 +519,32 @@ ALERT_THRESHOLD_DRIFT_PCT=15
|
||||
예를 들어 onboarding latency alert와 moderation latency alert는 서로 throttle하지 않는다.
|
||||
skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한다.
|
||||
|
||||
### 16.2 Prometheus scrape
|
||||
### 16.2 Phoenix trace
|
||||
|
||||
LLM 호출, skill 실행, tool 실행 span은 OpenTelemetry trace로 남긴다.
|
||||
`PHOENIX_OTLP_ENDPOINT`가 없으면 기존 설정을 따른다. 개발 환경 기본값은 trace exporter 비활성화이고, `OTEL_EXPORTER=stdout`이면 콘솔 exporter를 쓴다.
|
||||
|
||||
Phoenix를 로컬에서 볼 때:
|
||||
|
||||
```bash
|
||||
docker run --rm -p 6006:6006 arizephoenix/phoenix:latest
|
||||
```
|
||||
|
||||
Agent/worker env:
|
||||
|
||||
```env
|
||||
PHOENIX_OTLP_ENDPOINT=http://localhost:6006/v1/traces
|
||||
OTEL_SAMPLING_RATIO=1
|
||||
PHOENIX_LOG_PROMPTS=false
|
||||
```
|
||||
|
||||
`PHOENIX_OTLP_ENDPOINT`가 설정되면 `OTEL_EXPORTER=none`이어도 Phoenix OTLP exporter를 우선 사용한다.
|
||||
Phoenix UI에서는 `service.name=lolympus-agent`, `gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.cost_usd`, `llm.token_count.prompt`, `llm.token_count.completion` 속성으로 LLM span을 필터링한다.
|
||||
|
||||
Prompt와 completion 본문은 기본적으로 기록하지 않는다.
|
||||
디버깅 때문에 필요할 때만 `PHOENIX_LOG_PROMPTS=true`를 켠다. 이 경우 email, 전화번호, token/API key 형태의 값은 span 기록 전에 masking된다.
|
||||
|
||||
### 16.3 Prometheus scrape
|
||||
|
||||
Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다.
|
||||
|
||||
@@ -569,7 +596,7 @@ sum by (skill, stage) (rate(agent_validator_fail_total[5m]))
|
||||
| eval drift | 15% 이하 |
|
||||
| archive stalled | 3회 연속 실패 0건 |
|
||||
|
||||
### 16.3 Slack webhook 설정
|
||||
### 16.4 Slack webhook 설정
|
||||
|
||||
Slack incoming webhook URL을 env로 설정한다.
|
||||
|
||||
|
||||
Generated
+1
@@ -12,6 +12,7 @@
|
||||
"@infisical/sdk": "^5.0.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
|
||||
"@opentelemetry/sdk-node": "^0.207.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.1.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.37.0",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"@infisical/sdk": "^5.0.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
|
||||
"@opentelemetry/sdk-node": "^0.207.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.1.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.37.0",
|
||||
|
||||
@@ -5,6 +5,8 @@ import {
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getPhoenixLogPrompts,
|
||||
getPhoenixOtlpEndpoint,
|
||||
getSlackAlertWebhookUrl,
|
||||
getSlackWebhookTimeoutMs,
|
||||
getSupervisorLlmResponseEnabled,
|
||||
@@ -19,6 +21,8 @@ const ENV_KEYS = [
|
||||
"ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING",
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
||||
"PHOENIX_LOG_PROMPTS",
|
||||
"PHOENIX_OTLP_ENDPOINT",
|
||||
"SLACK_ALERT_WEBHOOK_URL",
|
||||
"SLACK_WEBHOOK_TIMEOUT_MS",
|
||||
"AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED",
|
||||
@@ -62,6 +66,17 @@ describe("config", () => {
|
||||
expect(getSlackWebhookTimeoutMs()).toBe(1234);
|
||||
});
|
||||
|
||||
it("reads Phoenix tracing configuration", () => {
|
||||
expect(getPhoenixOtlpEndpoint()).toBeUndefined();
|
||||
expect(getPhoenixLogPrompts()).toBe(false);
|
||||
|
||||
vi.stubEnv("PHOENIX_OTLP_ENDPOINT", " http://localhost:6006/v1/traces ");
|
||||
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
|
||||
|
||||
expect(getPhoenixOtlpEndpoint()).toBe("http://localhost:6006/v1/traces");
|
||||
expect(getPhoenixLogPrompts()).toBe(true);
|
||||
});
|
||||
|
||||
it("enables supervisor LLM user messages by default with env override", () => {
|
||||
expect(getSupervisorLlmResponseEnabled()).toBe(true);
|
||||
|
||||
|
||||
@@ -121,6 +121,14 @@ export function getOtelExporter(): "stdout" | "none" {
|
||||
return getNodeEnv() === "dev" ? "none" : "stdout";
|
||||
}
|
||||
|
||||
export function getPhoenixOtlpEndpoint(): string | undefined {
|
||||
return process.env["PHOENIX_OTLP_ENDPOINT"]?.trim() || undefined;
|
||||
}
|
||||
|
||||
export function getPhoenixLogPrompts(): boolean {
|
||||
return getBoolean("PHOENIX_LOG_PROMPTS", false);
|
||||
}
|
||||
|
||||
export function getRequestSubject(): string {
|
||||
return (
|
||||
process.env["AGENT_REQUEST_SUBJECT"]?.trim() || "lolympus-agent.request"
|
||||
|
||||
@@ -1,9 +1,36 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const llmSpanMock = vi.hoisted(() => ({
|
||||
initialAttributes: [] as Array<Record<string, unknown> | undefined>,
|
||||
setAttributes: vi.fn((attributes: Record<string, unknown>) => attributes),
|
||||
}));
|
||||
|
||||
vi.mock("@obs/spans", () => ({
|
||||
maskTraceContent: (content: string) =>
|
||||
content
|
||||
.replace(
|
||||
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
|
||||
"[redacted-email]",
|
||||
)
|
||||
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, "[redacted-phone]"),
|
||||
withLlmSpan: vi.fn(
|
||||
(
|
||||
attributes: Record<string, unknown> | undefined,
|
||||
fn: (span: { setAttributes: typeof llmSpanMock.setAttributes }) => unknown,
|
||||
) => {
|
||||
llmSpanMock.initialAttributes.push(attributes);
|
||||
return Promise.resolve(fn({ setAttributes: llmSpanMock.setAttributes }));
|
||||
},
|
||||
),
|
||||
}));
|
||||
|
||||
import { LlmClient } from "./llm-client.js";
|
||||
import type { ChatCompletionResponse } from "./types.js";
|
||||
|
||||
function okResponse(content = "ok"): Response {
|
||||
function okResponse(
|
||||
content = "ok",
|
||||
overrides: Partial<ChatCompletionResponse> = {},
|
||||
): Response {
|
||||
const payload: ChatCompletionResponse = {
|
||||
choices: [
|
||||
{
|
||||
@@ -15,6 +42,7 @@ function okResponse(content = "ok"): Response {
|
||||
},
|
||||
],
|
||||
model: "test-model",
|
||||
...overrides,
|
||||
};
|
||||
|
||||
return new Response(JSON.stringify(payload), {
|
||||
@@ -35,7 +63,10 @@ function stringifyFetchUrl(input: string | URL | Request): string {
|
||||
|
||||
describe("LlmClient endpoint normalization", () => {
|
||||
afterEach(() => {
|
||||
llmSpanMock.initialAttributes.length = 0;
|
||||
llmSpanMock.setAttributes.mockClear();
|
||||
vi.unstubAllGlobals();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("normalizes OpenRouter API base URLs to chat completions", async () => {
|
||||
@@ -86,4 +117,55 @@ describe("LlmClient endpoint normalization", () => {
|
||||
expect(result?.content).toBe("ok");
|
||||
expect(urls).toEqual(["https://openrouter.ai/api/v1/chat/completions"]);
|
||||
});
|
||||
|
||||
it("records OpenInference token, cost, and masked prompt attributes", async () => {
|
||||
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(() =>
|
||||
Promise.resolve(
|
||||
okResponse("reply to 010-9999-8888", {
|
||||
id: "chatcmpl-test",
|
||||
usage: {
|
||||
completion_tokens: 50,
|
||||
prompt_tokens: 100,
|
||||
total_tokens: 150,
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
const client = new LlmClient({
|
||||
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
|
||||
providerApiKeys: { openrouter: "test-key" },
|
||||
});
|
||||
|
||||
const result = await client.chatCompletion(
|
||||
[{ content: "hi a@example.com 010-1234-5678", role: "user" }],
|
||||
{
|
||||
maxTokens: 128,
|
||||
modelOverride: "gpt-5.3-codex-spark",
|
||||
temperature: 0.25,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result?.content).toBe("reply to 010-9999-8888");
|
||||
expect(llmSpanMock.initialAttributes.at(-1)).toMatchObject({
|
||||
"gen_ai.request.max_tokens": 128,
|
||||
"gen_ai.request.model": "gpt-5.3-codex-spark",
|
||||
"gen_ai.request.temperature": 0.25,
|
||||
"llm.input_messages.0.message.content":
|
||||
"hi [redacted-email] [redacted-phone]",
|
||||
"llm.model_name": "gpt-5.3-codex-spark",
|
||||
});
|
||||
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
"gen_ai.response.id": "chatcmpl-test",
|
||||
"gen_ai.usage.cost_usd": 0.000045,
|
||||
"llm.output_messages.0.message.content": "reply to [redacted-phone]",
|
||||
"llm.token_count.completion": 50,
|
||||
"llm.token_count.prompt": 100,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
+51
-11
@@ -8,11 +8,12 @@ import type {
|
||||
ProviderName,
|
||||
} from "./types.js";
|
||||
import { getContext, getRequestLogger } from "@obs/context";
|
||||
import { withLlmSpan } from "@obs/spans";
|
||||
import { maskTraceContent, withLlmSpan } from "@obs/spans";
|
||||
import {
|
||||
getLlmHealthTimeoutMs,
|
||||
getLlmRequestTimeoutMs,
|
||||
getLlmTemperature,
|
||||
getPhoenixLogPrompts,
|
||||
} from "../config.js";
|
||||
import { LlmCircuitBreaker } from "./circuit.js";
|
||||
import { getProviderAdapter } from "./providers/index.js";
|
||||
@@ -84,6 +85,33 @@ function getRequestSignal(
|
||||
return controller.signal;
|
||||
}
|
||||
|
||||
function buildPromptTraceAttributes(
|
||||
messages: readonly ChatMessage[],
|
||||
): Record<string, string> {
|
||||
return Object.fromEntries(
|
||||
messages.flatMap((message, index) => [
|
||||
[`llm.input_messages.${index}.message.role`, message.role],
|
||||
[
|
||||
`llm.input_messages.${index}.message.content`,
|
||||
maskTraceContent(message.content),
|
||||
],
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
function buildCompletionTraceAttributes(
|
||||
result: ChatCompletionResult | null,
|
||||
): Record<string, string> {
|
||||
if (!result) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return {
|
||||
"llm.output_messages.0.message.content": maskTraceContent(result.content),
|
||||
"llm.output_messages.0.message.role": result.message.role,
|
||||
};
|
||||
}
|
||||
|
||||
export class LlmClient {
|
||||
private readonly circuit: LlmCircuitBreaker;
|
||||
private readonly endpoints: LlmEndpointConfig[];
|
||||
@@ -182,11 +210,18 @@ export class LlmClient {
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
tools: options?.tools,
|
||||
});
|
||||
const shouldLogPrompts = getPhoenixLogPrompts();
|
||||
|
||||
const llmResult = await withLlmSpan(
|
||||
{
|
||||
...(shouldLogPrompts
|
||||
? buildPromptTraceAttributes(request.messages)
|
||||
: {}),
|
||||
"gen_ai.request.max_tokens": request.max_tokens,
|
||||
"gen_ai.request.model": request.model,
|
||||
"gen_ai.request.temperature": request.temperature,
|
||||
"gen_ai.system": provider.name,
|
||||
"llm.model_name": request.model,
|
||||
},
|
||||
async (span) => {
|
||||
const response = await fetch(endpoint.completionUrl, {
|
||||
@@ -213,24 +248,29 @@ export class LlmClient {
|
||||
|
||||
const payload = (await response.json()) as ChatCompletionResponse;
|
||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||
metrics.tokenCostUsd.observe(
|
||||
estimateChatCost(
|
||||
request.model,
|
||||
payload.usage?.prompt_tokens,
|
||||
payload.usage?.completion_tokens,
|
||||
),
|
||||
{
|
||||
model: request.model,
|
||||
skill: getContext()?.skill,
|
||||
},
|
||||
const costUsd = estimateChatCost(
|
||||
request.model,
|
||||
payload.usage?.prompt_tokens,
|
||||
payload.usage?.completion_tokens,
|
||||
);
|
||||
metrics.tokenCostUsd.observe(costUsd, {
|
||||
model: request.model,
|
||||
skill: getContext()?.skill,
|
||||
});
|
||||
|
||||
span.setAttributes({
|
||||
...(shouldLogPrompts
|
||||
? buildCompletionTraceAttributes(result)
|
||||
: {}),
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
?.map((choice) => choice.finish_reason ?? "unknown")
|
||||
.filter((reason): reason is string => Boolean(reason)),
|
||||
"gen_ai.response.id": payload.id,
|
||||
"gen_ai.usage.cost_usd": costUsd,
|
||||
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
|
||||
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
||||
"llm.token_count.completion": payload.usage?.completion_tokens,
|
||||
"llm.token_count.prompt": payload.usage?.prompt_tokens,
|
||||
});
|
||||
|
||||
if (!result) {
|
||||
|
||||
@@ -30,6 +30,7 @@ export interface ChatCompletionRequest {
|
||||
}
|
||||
|
||||
export interface ChatCompletionResponse {
|
||||
id?: string | undefined;
|
||||
model?: string;
|
||||
choices?: Array<{
|
||||
finish_reason?: string | null;
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { maskTraceContent } from "./spans.js";
|
||||
|
||||
describe("maskTraceContent", () => {
|
||||
it("redacts common PII and secrets before prompt tracing", () => {
|
||||
const masked = maskTraceContent(
|
||||
"email a@example.com phone 010-1234-5678 token=super-secret sk-testsecret123",
|
||||
);
|
||||
|
||||
expect(masked).toContain("[redacted-email]");
|
||||
expect(masked).toContain("[redacted-phone]");
|
||||
expect(masked).toContain("token=[redacted-secret]");
|
||||
expect(masked).toContain("[redacted-token]");
|
||||
expect(masked).not.toContain("a@example.com");
|
||||
expect(masked).not.toContain("010-1234-5678");
|
||||
expect(masked).not.toContain("super-secret");
|
||||
expect(masked).not.toContain("sk-testsecret123");
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,8 @@ import {
|
||||
|
||||
import { getServiceName } from '../config.js';
|
||||
|
||||
const MAX_TRACE_CONTENT_CHARS = 2_000;
|
||||
|
||||
function compactAttributes(attributes?: Attributes): Attributes {
|
||||
if (!attributes) {
|
||||
return {};
|
||||
@@ -17,6 +19,26 @@ function compactAttributes(attributes?: Attributes): Attributes {
|
||||
);
|
||||
}
|
||||
|
||||
export function maskTraceContent(content: string): string {
|
||||
const masked = content
|
||||
.replace(
|
||||
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
|
||||
'[redacted-email]',
|
||||
)
|
||||
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, '[redacted-phone]')
|
||||
.replace(/\b(?:Bearer\s+|sk-|pk-)[A-Za-z0-9._-]{8,}\b/gu, '[redacted-token]')
|
||||
.replace(
|
||||
/\b(password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+/giu,
|
||||
'$1=[redacted-secret]',
|
||||
);
|
||||
|
||||
if (masked.length <= MAX_TRACE_CONTENT_CHARS) {
|
||||
return masked;
|
||||
}
|
||||
|
||||
return `${masked.slice(0, MAX_TRACE_CONTENT_CHARS)}...[truncated]`;
|
||||
}
|
||||
|
||||
export async function withSpan<T>(
|
||||
name: string,
|
||||
attributes: Attributes | undefined,
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
||||
import {
|
||||
ConsoleSpanExporter,
|
||||
ParentBasedSampler,
|
||||
TraceIdRatioBasedSampler,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
|
||||
import { getOtelExporter, getOtelSamplingRatio, getServiceName } from '../config.js';
|
||||
import {
|
||||
getOtelExporter,
|
||||
getOtelSamplingRatio,
|
||||
getPhoenixOtlpEndpoint,
|
||||
getServiceName,
|
||||
} from '../config.js';
|
||||
import { getLogger } from './logger.js';
|
||||
|
||||
let sdk: NodeSDK | null = null;
|
||||
@@ -18,7 +24,8 @@ export function initTracing(): void {
|
||||
}
|
||||
|
||||
const exporterType = getOtelExporter();
|
||||
if (exporterType === 'none') {
|
||||
const phoenixEndpoint = getPhoenixOtlpEndpoint();
|
||||
if (exporterType === 'none' && !phoenixEndpoint) {
|
||||
const logger = getLogger();
|
||||
logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init');
|
||||
started = true;
|
||||
@@ -28,7 +35,9 @@ export function initTracing(): void {
|
||||
process.env['OTEL_SERVICE_NAME'] ??= getServiceName();
|
||||
|
||||
const logger = getLogger();
|
||||
const exporter = new ConsoleSpanExporter();
|
||||
const exporter = phoenixEndpoint
|
||||
? new OTLPTraceExporter({ url: phoenixEndpoint })
|
||||
: new ConsoleSpanExporter();
|
||||
|
||||
sdk = new NodeSDK({
|
||||
instrumentations: [getNodeAutoInstrumentations()],
|
||||
@@ -44,6 +53,7 @@ export function initTracing(): void {
|
||||
logger.debug(
|
||||
{
|
||||
exporter: getOtelExporter(),
|
||||
phoenixOtlpEndpoint: phoenixEndpoint,
|
||||
samplingRatio: getOtelSamplingRatio(),
|
||||
},
|
||||
'[Tracing] Initialized',
|
||||
|
||||
Reference in new issue
Block a user