diff --git a/docs/service-evaluation-operations.md b/docs/service-evaluation-operations.md index 3358fc0..44e5fb6 100644 --- a/docs/service-evaluation-operations.md +++ b/docs/service-evaluation-operations.md @@ -461,6 +461,8 @@ npm test | `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 | | `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path | | `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 | +| `PHOENIX_OTLP_ENDPOINT` | unset | Phoenix OTLP trace endpoint | +| `PHOENIX_LOG_PROMPTS` | false | LLM prompt/completion content trace 기록 | | `METRICS_PORT` | 9464 | Prometheus `/metrics` HTTP endpoint port | | `SLACK_ALERT_WEBHOOK_URL` | unset | alert Slack incoming webhook URL | | `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | Slack webhook POST timeout | @@ -517,7 +519,32 @@ ALERT_THRESHOLD_DRIFT_PCT=15 예를 들어 onboarding latency alert와 moderation latency alert는 서로 throttle하지 않는다. skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한다. -### 16.2 Prometheus scrape +### 16.2 Phoenix trace + +LLM 호출, skill 실행, tool 실행 span은 OpenTelemetry trace로 남긴다. +`PHOENIX_OTLP_ENDPOINT`가 없으면 기존 설정을 따른다. 개발 환경 기본값은 trace exporter 비활성화이고, `OTEL_EXPORTER=stdout`이면 콘솔 exporter를 쓴다. + +Phoenix를 로컬에서 볼 때: + +```bash +docker run --rm -p 6006:6006 arizephoenix/phoenix:latest +``` + +Agent/worker env: + +```env +PHOENIX_OTLP_ENDPOINT=http://localhost:6006/v1/traces +OTEL_SAMPLING_RATIO=1 +PHOENIX_LOG_PROMPTS=false +``` + +`PHOENIX_OTLP_ENDPOINT`가 설정되면 `OTEL_EXPORTER=none`이어도 Phoenix OTLP exporter를 우선 사용한다. +Phoenix UI에서는 `service.name=lolympus-agent`, `gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.cost_usd`, `llm.token_count.prompt`, `llm.token_count.completion` 속성으로 LLM span을 필터링한다. + +Prompt와 completion 본문은 기본적으로 기록하지 않는다. +디버깅 때문에 필요할 때만 `PHOENIX_LOG_PROMPTS=true`를 켠다. 이 경우 email, 전화번호, token/API key 형태의 값은 span 기록 전에 masking된다. + +### 16.3 Prometheus scrape Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다. @@ -569,7 +596,7 @@ sum by (skill, stage) (rate(agent_validator_fail_total[5m])) | eval drift | 15% 이하 | | archive stalled | 3회 연속 실패 0건 | -### 16.3 Slack webhook 설정 +### 16.4 Slack webhook 설정 Slack incoming webhook URL을 env로 설정한다. diff --git a/package-lock.json b/package-lock.json index 70c5e82..0aebcf7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@infisical/sdk": "^5.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/auto-instrumentations-node": "^0.66.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.207.0", "@opentelemetry/sdk-node": "^0.207.0", "@opentelemetry/sdk-trace-base": "^2.1.0", "@opentelemetry/semantic-conventions": "^1.37.0", diff --git a/package.json b/package.json index 95dc145..423c24a 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "@infisical/sdk": "^5.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/auto-instrumentations-node": "^0.66.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.207.0", "@opentelemetry/sdk-node": "^0.207.0", "@opentelemetry/sdk-trace-base": "^2.1.0", "@opentelemetry/semantic-conventions": "^1.37.0", diff --git a/src/config.test.ts b/src/config.test.ts index f92a3e9..bbd2667 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -5,6 +5,8 @@ import { getAlertThresholdEvalScoreDropPct, getAlertThresholdLlmErrorRate, getAlertThresholdP95LatencyMs, + getPhoenixLogPrompts, + getPhoenixOtlpEndpoint, getSlackAlertWebhookUrl, getSlackWebhookTimeoutMs, getSupervisorLlmResponseEnabled, @@ -19,6 +21,8 @@ const ENV_KEYS = [ "ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING", "ALERT_THRESHOLD_P95_LATENCY_MS", "ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING", + "PHOENIX_LOG_PROMPTS", + "PHOENIX_OTLP_ENDPOINT", "SLACK_ALERT_WEBHOOK_URL", "SLACK_WEBHOOK_TIMEOUT_MS", "AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", @@ -62,6 +66,17 @@ describe("config", () => { expect(getSlackWebhookTimeoutMs()).toBe(1234); }); + it("reads Phoenix tracing configuration", () => { + expect(getPhoenixOtlpEndpoint()).toBeUndefined(); + expect(getPhoenixLogPrompts()).toBe(false); + + vi.stubEnv("PHOENIX_OTLP_ENDPOINT", " http://localhost:6006/v1/traces "); + vi.stubEnv("PHOENIX_LOG_PROMPTS", "true"); + + expect(getPhoenixOtlpEndpoint()).toBe("http://localhost:6006/v1/traces"); + expect(getPhoenixLogPrompts()).toBe(true); + }); + it("enables supervisor LLM user messages by default with env override", () => { expect(getSupervisorLlmResponseEnabled()).toBe(true); diff --git a/src/config.ts b/src/config.ts index 9037c2a..491363e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -121,6 +121,14 @@ export function getOtelExporter(): "stdout" | "none" { return getNodeEnv() === "dev" ? "none" : "stdout"; } +export function getPhoenixOtlpEndpoint(): string | undefined { + return process.env["PHOENIX_OTLP_ENDPOINT"]?.trim() || undefined; +} + +export function getPhoenixLogPrompts(): boolean { + return getBoolean("PHOENIX_LOG_PROMPTS", false); +} + export function getRequestSubject(): string { return ( process.env["AGENT_REQUEST_SUBJECT"]?.trim() || "lolympus-agent.request" diff --git a/src/llm/llm-client.test.ts b/src/llm/llm-client.test.ts index 33c9c93..439f74f 100644 --- a/src/llm/llm-client.test.ts +++ b/src/llm/llm-client.test.ts @@ -1,9 +1,36 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +const llmSpanMock = vi.hoisted(() => ({ + initialAttributes: [] as Array | undefined>, + setAttributes: vi.fn((attributes: Record) => attributes), +})); + +vi.mock("@obs/spans", () => ({ + maskTraceContent: (content: string) => + content + .replace( + /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, + "[redacted-email]", + ) + .replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, "[redacted-phone]"), + withLlmSpan: vi.fn( + ( + attributes: Record | undefined, + fn: (span: { setAttributes: typeof llmSpanMock.setAttributes }) => unknown, + ) => { + llmSpanMock.initialAttributes.push(attributes); + return Promise.resolve(fn({ setAttributes: llmSpanMock.setAttributes })); + }, + ), +})); + import { LlmClient } from "./llm-client.js"; import type { ChatCompletionResponse } from "./types.js"; -function okResponse(content = "ok"): Response { +function okResponse( + content = "ok", + overrides: Partial = {}, +): Response { const payload: ChatCompletionResponse = { choices: [ { @@ -15,6 +42,7 @@ function okResponse(content = "ok"): Response { }, ], model: "test-model", + ...overrides, }; return new Response(JSON.stringify(payload), { @@ -35,7 +63,10 @@ function stringifyFetchUrl(input: string | URL | Request): string { describe("LlmClient endpoint normalization", () => { afterEach(() => { + llmSpanMock.initialAttributes.length = 0; + llmSpanMock.setAttributes.mockClear(); vi.unstubAllGlobals(); + vi.unstubAllEnvs(); }); it("normalizes OpenRouter API base URLs to chat completions", async () => { @@ -86,4 +117,55 @@ describe("LlmClient endpoint normalization", () => { expect(result?.content).toBe("ok"); expect(urls).toEqual(["https://openrouter.ai/api/v1/chat/completions"]); }); + + it("records OpenInference token, cost, and masked prompt attributes", async () => { + vi.stubEnv("PHOENIX_LOG_PROMPTS", "true"); + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve( + okResponse("reply to 010-9999-8888", { + id: "chatcmpl-test", + usage: { + completion_tokens: 50, + prompt_tokens: 100, + total_tokens: 150, + }, + }), + ), + ), + ); + const client = new LlmClient({ + endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }], + providerApiKeys: { openrouter: "test-key" }, + }); + + const result = await client.chatCompletion( + [{ content: "hi a@example.com 010-1234-5678", role: "user" }], + { + maxTokens: 128, + modelOverride: "gpt-5.3-codex-spark", + temperature: 0.25, + }, + ); + + expect(result?.content).toBe("reply to 010-9999-8888"); + expect(llmSpanMock.initialAttributes.at(-1)).toMatchObject({ + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-5.3-codex-spark", + "gen_ai.request.temperature": 0.25, + "llm.input_messages.0.message.content": + "hi [redacted-email] [redacted-phone]", + "llm.model_name": "gpt-5.3-codex-spark", + }); + expect(llmSpanMock.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + "gen_ai.response.id": "chatcmpl-test", + "gen_ai.usage.cost_usd": 0.000045, + "llm.output_messages.0.message.content": "reply to [redacted-phone]", + "llm.token_count.completion": 50, + "llm.token_count.prompt": 100, + }), + ); + }); }); diff --git a/src/llm/llm-client.ts b/src/llm/llm-client.ts index aa582ac..c2d6b5b 100644 --- a/src/llm/llm-client.ts +++ b/src/llm/llm-client.ts @@ -8,11 +8,12 @@ import type { ProviderName, } from "./types.js"; import { getContext, getRequestLogger } from "@obs/context"; -import { withLlmSpan } from "@obs/spans"; +import { maskTraceContent, withLlmSpan } from "@obs/spans"; import { getLlmHealthTimeoutMs, getLlmRequestTimeoutMs, getLlmTemperature, + getPhoenixLogPrompts, } from "../config.js"; import { LlmCircuitBreaker } from "./circuit.js"; import { getProviderAdapter } from "./providers/index.js"; @@ -84,6 +85,33 @@ function getRequestSignal( return controller.signal; } +function buildPromptTraceAttributes( + messages: readonly ChatMessage[], +): Record { + return Object.fromEntries( + messages.flatMap((message, index) => [ + [`llm.input_messages.${index}.message.role`, message.role], + [ + `llm.input_messages.${index}.message.content`, + maskTraceContent(message.content), + ], + ]), + ); +} + +function buildCompletionTraceAttributes( + result: ChatCompletionResult | null, +): Record { + if (!result) { + return {}; + } + + return { + "llm.output_messages.0.message.content": maskTraceContent(result.content), + "llm.output_messages.0.message.role": result.message.role, + }; +} + export class LlmClient { private readonly circuit: LlmCircuitBreaker; private readonly endpoints: LlmEndpointConfig[]; @@ -182,11 +210,18 @@ export class LlmClient { temperature: options?.temperature ?? getLlmTemperature(), tools: options?.tools, }); + const shouldLogPrompts = getPhoenixLogPrompts(); const llmResult = await withLlmSpan( { + ...(shouldLogPrompts + ? buildPromptTraceAttributes(request.messages) + : {}), + "gen_ai.request.max_tokens": request.max_tokens, "gen_ai.request.model": request.model, + "gen_ai.request.temperature": request.temperature, "gen_ai.system": provider.name, + "llm.model_name": request.model, }, async (span) => { const response = await fetch(endpoint.completionUrl, { @@ -213,24 +248,29 @@ export class LlmClient { const payload = (await response.json()) as ChatCompletionResponse; const result = provider.toResult(payload, endpoint.completionUrl); - metrics.tokenCostUsd.observe( - estimateChatCost( - request.model, - payload.usage?.prompt_tokens, - payload.usage?.completion_tokens, - ), - { - model: request.model, - skill: getContext()?.skill, - }, + const costUsd = estimateChatCost( + request.model, + payload.usage?.prompt_tokens, + payload.usage?.completion_tokens, ); + metrics.tokenCostUsd.observe(costUsd, { + model: request.model, + skill: getContext()?.skill, + }); span.setAttributes({ + ...(shouldLogPrompts + ? buildCompletionTraceAttributes(result) + : {}), "gen_ai.response.finish_reasons": payload.choices ?.map((choice) => choice.finish_reason ?? "unknown") .filter((reason): reason is string => Boolean(reason)), + "gen_ai.response.id": payload.id, + "gen_ai.usage.cost_usd": costUsd, "gen_ai.usage.input_tokens": payload.usage?.prompt_tokens, "gen_ai.usage.output_tokens": payload.usage?.completion_tokens, + "llm.token_count.completion": payload.usage?.completion_tokens, + "llm.token_count.prompt": payload.usage?.prompt_tokens, }); if (!result) { diff --git a/src/llm/types.ts b/src/llm/types.ts index 26663f7..c5182e3 100644 --- a/src/llm/types.ts +++ b/src/llm/types.ts @@ -30,6 +30,7 @@ export interface ChatCompletionRequest { } export interface ChatCompletionResponse { + id?: string | undefined; model?: string; choices?: Array<{ finish_reason?: string | null; diff --git a/src/observability/spans.test.ts b/src/observability/spans.test.ts new file mode 100644 index 0000000..72d6cf1 --- /dev/null +++ b/src/observability/spans.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; + +import { maskTraceContent } from "./spans.js"; + +describe("maskTraceContent", () => { + it("redacts common PII and secrets before prompt tracing", () => { + const masked = maskTraceContent( + "email a@example.com phone 010-1234-5678 token=super-secret sk-testsecret123", + ); + + expect(masked).toContain("[redacted-email]"); + expect(masked).toContain("[redacted-phone]"); + expect(masked).toContain("token=[redacted-secret]"); + expect(masked).toContain("[redacted-token]"); + expect(masked).not.toContain("a@example.com"); + expect(masked).not.toContain("010-1234-5678"); + expect(masked).not.toContain("super-secret"); + expect(masked).not.toContain("sk-testsecret123"); + }); +}); diff --git a/src/observability/spans.ts b/src/observability/spans.ts index f29832d..6c51d51 100644 --- a/src/observability/spans.ts +++ b/src/observability/spans.ts @@ -7,6 +7,8 @@ import { import { getServiceName } from '../config.js'; +const MAX_TRACE_CONTENT_CHARS = 2_000; + function compactAttributes(attributes?: Attributes): Attributes { if (!attributes) { return {}; @@ -17,6 +19,26 @@ function compactAttributes(attributes?: Attributes): Attributes { ); } +export function maskTraceContent(content: string): string { + const masked = content + .replace( + /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, + '[redacted-email]', + ) + .replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, '[redacted-phone]') + .replace(/\b(?:Bearer\s+|sk-|pk-)[A-Za-z0-9._-]{8,}\b/gu, '[redacted-token]') + .replace( + /\b(password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+/giu, + '$1=[redacted-secret]', + ); + + if (masked.length <= MAX_TRACE_CONTENT_CHARS) { + return masked; + } + + return `${masked.slice(0, MAX_TRACE_CONTENT_CHARS)}...[truncated]`; +} + export async function withSpan( name: string, attributes: Attributes | undefined, diff --git a/src/observability/tracing.ts b/src/observability/tracing.ts index 8c08c32..6e0d5d9 100644 --- a/src/observability/tracing.ts +++ b/src/observability/tracing.ts @@ -1,12 +1,18 @@ import { NodeSDK } from '@opentelemetry/sdk-node'; import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; import { ConsoleSpanExporter, ParentBasedSampler, TraceIdRatioBasedSampler, } from '@opentelemetry/sdk-trace-base'; -import { getOtelExporter, getOtelSamplingRatio, getServiceName } from '../config.js'; +import { + getOtelExporter, + getOtelSamplingRatio, + getPhoenixOtlpEndpoint, + getServiceName, +} from '../config.js'; import { getLogger } from './logger.js'; let sdk: NodeSDK | null = null; @@ -18,7 +24,8 @@ export function initTracing(): void { } const exporterType = getOtelExporter(); - if (exporterType === 'none') { + const phoenixEndpoint = getPhoenixOtlpEndpoint(); + if (exporterType === 'none' && !phoenixEndpoint) { const logger = getLogger(); logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init'); started = true; @@ -28,7 +35,9 @@ export function initTracing(): void { process.env['OTEL_SERVICE_NAME'] ??= getServiceName(); const logger = getLogger(); - const exporter = new ConsoleSpanExporter(); + const exporter = phoenixEndpoint + ? new OTLPTraceExporter({ url: phoenixEndpoint }) + : new ConsoleSpanExporter(); sdk = new NodeSDK({ instrumentations: [getNodeAutoInstrumentations()], @@ -44,6 +53,7 @@ export function initTracing(): void { logger.debug( { exporter: getOtelExporter(), + phoenixOtlpEndpoint: phoenixEndpoint, samplingRatio: getOtelSamplingRatio(), }, '[Tracing] Initialized',