Add Phoenix OTLP tracing and LLM span attributes
This commit is contained in:
1 parent
f15459bb99
commit
1d0be56842
11 files changed
+240
-13
No files matched your search
@@ -461,6 +461,8 @@ npm test
|
|||||||
| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 |
|
| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 |
|
||||||
| `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path |
|
| `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path |
|
||||||
| `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 |
|
| `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 |
|
||||||
|
| `PHOENIX_OTLP_ENDPOINT` | unset | Phoenix OTLP trace endpoint |
|
||||||
|
| `PHOENIX_LOG_PROMPTS` | false | LLM prompt/completion content trace 기록 |
|
||||||
| `METRICS_PORT` | 9464 | Prometheus `/metrics` HTTP endpoint port |
|
| `METRICS_PORT` | 9464 | Prometheus `/metrics` HTTP endpoint port |
|
||||||
| `SLACK_ALERT_WEBHOOK_URL` | unset | alert Slack incoming webhook URL |
|
| `SLACK_ALERT_WEBHOOK_URL` | unset | alert Slack incoming webhook URL |
|
||||||
| `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | Slack webhook POST timeout |
|
| `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | Slack webhook POST timeout |
|
||||||
@@ -517,7 +519,32 @@ ALERT_THRESHOLD_DRIFT_PCT=15
|
|||||||
예를 들어 onboarding latency alert와 moderation latency alert는 서로 throttle하지 않는다.
|
예를 들어 onboarding latency alert와 moderation latency alert는 서로 throttle하지 않는다.
|
||||||
skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한다.
|
skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한다.
|
||||||
|
|
||||||
### 16.2 Prometheus scrape
|
### 16.2 Phoenix trace
|
||||||
|
|
||||||
|
LLM 호출, skill 실행, tool 실행 span은 OpenTelemetry trace로 남긴다.
|
||||||
|
`PHOENIX_OTLP_ENDPOINT`가 없으면 기존 설정을 따른다. 개발 환경 기본값은 trace exporter 비활성화이고, `OTEL_EXPORTER=stdout`이면 콘솔 exporter를 쓴다.
|
||||||
|
|
||||||
|
Phoenix를 로컬에서 볼 때:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --rm -p 6006:6006 arizephoenix/phoenix:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
Agent/worker env:
|
||||||
|
|
||||||
|
```env
|
||||||
|
PHOENIX_OTLP_ENDPOINT=http://localhost:6006/v1/traces
|
||||||
|
OTEL_SAMPLING_RATIO=1
|
||||||
|
PHOENIX_LOG_PROMPTS=false
|
||||||
|
```
|
||||||
|
|
||||||
|
`PHOENIX_OTLP_ENDPOINT`가 설정되면 `OTEL_EXPORTER=none`이어도 Phoenix OTLP exporter를 우선 사용한다.
|
||||||
|
Phoenix UI에서는 `service.name=lolympus-agent`, `gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.cost_usd`, `llm.token_count.prompt`, `llm.token_count.completion` 속성으로 LLM span을 필터링한다.
|
||||||
|
|
||||||
|
Prompt와 completion 본문은 기본적으로 기록하지 않는다.
|
||||||
|
디버깅 때문에 필요할 때만 `PHOENIX_LOG_PROMPTS=true`를 켠다. 이 경우 email, 전화번호, token/API key 형태의 값은 span 기록 전에 masking된다.
|
||||||
|
|
||||||
|
### 16.3 Prometheus scrape
|
||||||
|
|
||||||
Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다.
|
Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다.
|
||||||
|
|
||||||
@@ -569,7 +596,7 @@ sum by (skill, stage) (rate(agent_validator_fail_total[5m]))
|
|||||||
| eval drift | 15% 이하 |
|
| eval drift | 15% 이하 |
|
||||||
| archive stalled | 3회 연속 실패 0건 |
|
| archive stalled | 3회 연속 실패 0건 |
|
||||||
|
|
||||||
### 16.3 Slack webhook 설정
|
### 16.4 Slack webhook 설정
|
||||||
|
|
||||||
Slack incoming webhook URL을 env로 설정한다.
|
Slack incoming webhook URL을 env로 설정한다.
|
||||||
|
|
||||||
|
|||||||
Generated
+1
@@ -12,6 +12,7 @@
|
|||||||
"@infisical/sdk": "^5.0.0",
|
"@infisical/sdk": "^5.0.0",
|
||||||
"@opentelemetry/api": "^1.9.0",
|
"@opentelemetry/api": "^1.9.0",
|
||||||
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
|
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
|
||||||
|
"@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
|
||||||
"@opentelemetry/sdk-node": "^0.207.0",
|
"@opentelemetry/sdk-node": "^0.207.0",
|
||||||
"@opentelemetry/sdk-trace-base": "^2.1.0",
|
"@opentelemetry/sdk-trace-base": "^2.1.0",
|
||||||
"@opentelemetry/semantic-conventions": "^1.37.0",
|
"@opentelemetry/semantic-conventions": "^1.37.0",
|
||||||
|
|||||||
@@ -63,6 +63,7 @@
|
|||||||
"@infisical/sdk": "^5.0.0",
|
"@infisical/sdk": "^5.0.0",
|
||||||
"@opentelemetry/api": "^1.9.0",
|
"@opentelemetry/api": "^1.9.0",
|
||||||
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
|
"@opentelemetry/auto-instrumentations-node": "^0.66.0",
|
||||||
|
"@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
|
||||||
"@opentelemetry/sdk-node": "^0.207.0",
|
"@opentelemetry/sdk-node": "^0.207.0",
|
||||||
"@opentelemetry/sdk-trace-base": "^2.1.0",
|
"@opentelemetry/sdk-trace-base": "^2.1.0",
|
||||||
"@opentelemetry/semantic-conventions": "^1.37.0",
|
"@opentelemetry/semantic-conventions": "^1.37.0",
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import {
|
|||||||
getAlertThresholdEvalScoreDropPct,
|
getAlertThresholdEvalScoreDropPct,
|
||||||
getAlertThresholdLlmErrorRate,
|
getAlertThresholdLlmErrorRate,
|
||||||
getAlertThresholdP95LatencyMs,
|
getAlertThresholdP95LatencyMs,
|
||||||
|
getPhoenixLogPrompts,
|
||||||
|
getPhoenixOtlpEndpoint,
|
||||||
getSlackAlertWebhookUrl,
|
getSlackAlertWebhookUrl,
|
||||||
getSlackWebhookTimeoutMs,
|
getSlackWebhookTimeoutMs,
|
||||||
getSupervisorLlmResponseEnabled,
|
getSupervisorLlmResponseEnabled,
|
||||||
@@ -19,6 +21,8 @@ const ENV_KEYS = [
|
|||||||
"ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING",
|
"ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING",
|
||||||
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
||||||
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
||||||
|
"PHOENIX_LOG_PROMPTS",
|
||||||
|
"PHOENIX_OTLP_ENDPOINT",
|
||||||
"SLACK_ALERT_WEBHOOK_URL",
|
"SLACK_ALERT_WEBHOOK_URL",
|
||||||
"SLACK_WEBHOOK_TIMEOUT_MS",
|
"SLACK_WEBHOOK_TIMEOUT_MS",
|
||||||
"AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED",
|
"AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED",
|
||||||
@@ -62,6 +66,17 @@ describe("config", () => {
|
|||||||
expect(getSlackWebhookTimeoutMs()).toBe(1234);
|
expect(getSlackWebhookTimeoutMs()).toBe(1234);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reads Phoenix tracing configuration", () => {
|
||||||
|
expect(getPhoenixOtlpEndpoint()).toBeUndefined();
|
||||||
|
expect(getPhoenixLogPrompts()).toBe(false);
|
||||||
|
|
||||||
|
vi.stubEnv("PHOENIX_OTLP_ENDPOINT", " http://localhost:6006/v1/traces ");
|
||||||
|
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
|
||||||
|
|
||||||
|
expect(getPhoenixOtlpEndpoint()).toBe("http://localhost:6006/v1/traces");
|
||||||
|
expect(getPhoenixLogPrompts()).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("enables supervisor LLM user messages by default with env override", () => {
|
it("enables supervisor LLM user messages by default with env override", () => {
|
||||||
expect(getSupervisorLlmResponseEnabled()).toBe(true);
|
expect(getSupervisorLlmResponseEnabled()).toBe(true);
|
||||||
|
|
||||||
|
|||||||
@@ -121,6 +121,14 @@ export function getOtelExporter(): "stdout" | "none" {
|
|||||||
return getNodeEnv() === "dev" ? "none" : "stdout";
|
return getNodeEnv() === "dev" ? "none" : "stdout";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getPhoenixOtlpEndpoint(): string | undefined {
|
||||||
|
return process.env["PHOENIX_OTLP_ENDPOINT"]?.trim() || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getPhoenixLogPrompts(): boolean {
|
||||||
|
return getBoolean("PHOENIX_LOG_PROMPTS", false);
|
||||||
|
}
|
||||||
|
|
||||||
export function getRequestSubject(): string {
|
export function getRequestSubject(): string {
|
||||||
return (
|
return (
|
||||||
process.env["AGENT_REQUEST_SUBJECT"]?.trim() || "lolympus-agent.request"
|
process.env["AGENT_REQUEST_SUBJECT"]?.trim() || "lolympus-agent.request"
|
||||||
|
|||||||
@@ -1,9 +1,36 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
const llmSpanMock = vi.hoisted(() => ({
|
||||||
|
initialAttributes: [] as Array<Record<string, unknown> | undefined>,
|
||||||
|
setAttributes: vi.fn((attributes: Record<string, unknown>) => attributes),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@obs/spans", () => ({
|
||||||
|
maskTraceContent: (content: string) =>
|
||||||
|
content
|
||||||
|
.replace(
|
||||||
|
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
|
||||||
|
"[redacted-email]",
|
||||||
|
)
|
||||||
|
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, "[redacted-phone]"),
|
||||||
|
withLlmSpan: vi.fn(
|
||||||
|
(
|
||||||
|
attributes: Record<string, unknown> | undefined,
|
||||||
|
fn: (span: { setAttributes: typeof llmSpanMock.setAttributes }) => unknown,
|
||||||
|
) => {
|
||||||
|
llmSpanMock.initialAttributes.push(attributes);
|
||||||
|
return Promise.resolve(fn({ setAttributes: llmSpanMock.setAttributes }));
|
||||||
|
},
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
|
||||||
import { LlmClient } from "./llm-client.js";
|
import { LlmClient } from "./llm-client.js";
|
||||||
import type { ChatCompletionResponse } from "./types.js";
|
import type { ChatCompletionResponse } from "./types.js";
|
||||||
|
|
||||||
function okResponse(content = "ok"): Response {
|
function okResponse(
|
||||||
|
content = "ok",
|
||||||
|
overrides: Partial<ChatCompletionResponse> = {},
|
||||||
|
): Response {
|
||||||
const payload: ChatCompletionResponse = {
|
const payload: ChatCompletionResponse = {
|
||||||
choices: [
|
choices: [
|
||||||
{
|
{
|
||||||
@@ -15,6 +42,7 @@ function okResponse(content = "ok"): Response {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
model: "test-model",
|
model: "test-model",
|
||||||
|
...overrides,
|
||||||
};
|
};
|
||||||
|
|
||||||
return new Response(JSON.stringify(payload), {
|
return new Response(JSON.stringify(payload), {
|
||||||
@@ -35,7 +63,10 @@ function stringifyFetchUrl(input: string | URL | Request): string {
|
|||||||
|
|
||||||
describe("LlmClient endpoint normalization", () => {
|
describe("LlmClient endpoint normalization", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
llmSpanMock.initialAttributes.length = 0;
|
||||||
|
llmSpanMock.setAttributes.mockClear();
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
|
vi.unstubAllEnvs();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("normalizes OpenRouter API base URLs to chat completions", async () => {
|
it("normalizes OpenRouter API base URLs to chat completions", async () => {
|
||||||
@@ -86,4 +117,55 @@ describe("LlmClient endpoint normalization", () => {
|
|||||||
expect(result?.content).toBe("ok");
|
expect(result?.content).toBe("ok");
|
||||||
expect(urls).toEqual(["https://openrouter.ai/api/v1/chat/completions"]);
|
expect(urls).toEqual(["https://openrouter.ai/api/v1/chat/completions"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("records OpenInference token, cost, and masked prompt attributes", async () => {
|
||||||
|
vi.stubEnv("PHOENIX_LOG_PROMPTS", "true");
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn(() =>
|
||||||
|
Promise.resolve(
|
||||||
|
okResponse("reply to 010-9999-8888", {
|
||||||
|
id: "chatcmpl-test",
|
||||||
|
usage: {
|
||||||
|
completion_tokens: 50,
|
||||||
|
prompt_tokens: 100,
|
||||||
|
total_tokens: 150,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const client = new LlmClient({
|
||||||
|
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
|
||||||
|
providerApiKeys: { openrouter: "test-key" },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await client.chatCompletion(
|
||||||
|
[{ content: "hi a@example.com 010-1234-5678", role: "user" }],
|
||||||
|
{
|
||||||
|
maxTokens: 128,
|
||||||
|
modelOverride: "gpt-5.3-codex-spark",
|
||||||
|
temperature: 0.25,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result?.content).toBe("reply to 010-9999-8888");
|
||||||
|
expect(llmSpanMock.initialAttributes.at(-1)).toMatchObject({
|
||||||
|
"gen_ai.request.max_tokens": 128,
|
||||||
|
"gen_ai.request.model": "gpt-5.3-codex-spark",
|
||||||
|
"gen_ai.request.temperature": 0.25,
|
||||||
|
"llm.input_messages.0.message.content":
|
||||||
|
"hi [redacted-email] [redacted-phone]",
|
||||||
|
"llm.model_name": "gpt-5.3-codex-spark",
|
||||||
|
});
|
||||||
|
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
"gen_ai.response.id": "chatcmpl-test",
|
||||||
|
"gen_ai.usage.cost_usd": 0.000045,
|
||||||
|
"llm.output_messages.0.message.content": "reply to [redacted-phone]",
|
||||||
|
"llm.token_count.completion": 50,
|
||||||
|
"llm.token_count.prompt": 100,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
+47
-7
@@ -8,11 +8,12 @@ import type {
|
|||||||
ProviderName,
|
ProviderName,
|
||||||
} from "./types.js";
|
} from "./types.js";
|
||||||
import { getContext, getRequestLogger } from "@obs/context";
|
import { getContext, getRequestLogger } from "@obs/context";
|
||||||
import { withLlmSpan } from "@obs/spans";
|
import { maskTraceContent, withLlmSpan } from "@obs/spans";
|
||||||
import {
|
import {
|
||||||
getLlmHealthTimeoutMs,
|
getLlmHealthTimeoutMs,
|
||||||
getLlmRequestTimeoutMs,
|
getLlmRequestTimeoutMs,
|
||||||
getLlmTemperature,
|
getLlmTemperature,
|
||||||
|
getPhoenixLogPrompts,
|
||||||
} from "../config.js";
|
} from "../config.js";
|
||||||
import { LlmCircuitBreaker } from "./circuit.js";
|
import { LlmCircuitBreaker } from "./circuit.js";
|
||||||
import { getProviderAdapter } from "./providers/index.js";
|
import { getProviderAdapter } from "./providers/index.js";
|
||||||
@@ -84,6 +85,33 @@ function getRequestSignal(
|
|||||||
return controller.signal;
|
return controller.signal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildPromptTraceAttributes(
|
||||||
|
messages: readonly ChatMessage[],
|
||||||
|
): Record<string, string> {
|
||||||
|
return Object.fromEntries(
|
||||||
|
messages.flatMap((message, index) => [
|
||||||
|
[`llm.input_messages.${index}.message.role`, message.role],
|
||||||
|
[
|
||||||
|
`llm.input_messages.${index}.message.content`,
|
||||||
|
maskTraceContent(message.content),
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildCompletionTraceAttributes(
|
||||||
|
result: ChatCompletionResult | null,
|
||||||
|
): Record<string, string> {
|
||||||
|
if (!result) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"llm.output_messages.0.message.content": maskTraceContent(result.content),
|
||||||
|
"llm.output_messages.0.message.role": result.message.role,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export class LlmClient {
|
export class LlmClient {
|
||||||
private readonly circuit: LlmCircuitBreaker;
|
private readonly circuit: LlmCircuitBreaker;
|
||||||
private readonly endpoints: LlmEndpointConfig[];
|
private readonly endpoints: LlmEndpointConfig[];
|
||||||
@@ -182,11 +210,18 @@ export class LlmClient {
|
|||||||
temperature: options?.temperature ?? getLlmTemperature(),
|
temperature: options?.temperature ?? getLlmTemperature(),
|
||||||
tools: options?.tools,
|
tools: options?.tools,
|
||||||
});
|
});
|
||||||
|
const shouldLogPrompts = getPhoenixLogPrompts();
|
||||||
|
|
||||||
const llmResult = await withLlmSpan(
|
const llmResult = await withLlmSpan(
|
||||||
{
|
{
|
||||||
|
...(shouldLogPrompts
|
||||||
|
? buildPromptTraceAttributes(request.messages)
|
||||||
|
: {}),
|
||||||
|
"gen_ai.request.max_tokens": request.max_tokens,
|
||||||
"gen_ai.request.model": request.model,
|
"gen_ai.request.model": request.model,
|
||||||
|
"gen_ai.request.temperature": request.temperature,
|
||||||
"gen_ai.system": provider.name,
|
"gen_ai.system": provider.name,
|
||||||
|
"llm.model_name": request.model,
|
||||||
},
|
},
|
||||||
async (span) => {
|
async (span) => {
|
||||||
const response = await fetch(endpoint.completionUrl, {
|
const response = await fetch(endpoint.completionUrl, {
|
||||||
@@ -213,24 +248,29 @@ export class LlmClient {
|
|||||||
|
|
||||||
const payload = (await response.json()) as ChatCompletionResponse;
|
const payload = (await response.json()) as ChatCompletionResponse;
|
||||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||||
metrics.tokenCostUsd.observe(
|
const costUsd = estimateChatCost(
|
||||||
estimateChatCost(
|
|
||||||
request.model,
|
request.model,
|
||||||
payload.usage?.prompt_tokens,
|
payload.usage?.prompt_tokens,
|
||||||
payload.usage?.completion_tokens,
|
payload.usage?.completion_tokens,
|
||||||
),
|
);
|
||||||
{
|
metrics.tokenCostUsd.observe(costUsd, {
|
||||||
model: request.model,
|
model: request.model,
|
||||||
skill: getContext()?.skill,
|
skill: getContext()?.skill,
|
||||||
},
|
});
|
||||||
);
|
|
||||||
|
|
||||||
span.setAttributes({
|
span.setAttributes({
|
||||||
|
...(shouldLogPrompts
|
||||||
|
? buildCompletionTraceAttributes(result)
|
||||||
|
: {}),
|
||||||
"gen_ai.response.finish_reasons": payload.choices
|
"gen_ai.response.finish_reasons": payload.choices
|
||||||
?.map((choice) => choice.finish_reason ?? "unknown")
|
?.map((choice) => choice.finish_reason ?? "unknown")
|
||||||
.filter((reason): reason is string => Boolean(reason)),
|
.filter((reason): reason is string => Boolean(reason)),
|
||||||
|
"gen_ai.response.id": payload.id,
|
||||||
|
"gen_ai.usage.cost_usd": costUsd,
|
||||||
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
|
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
|
||||||
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
||||||
|
"llm.token_count.completion": payload.usage?.completion_tokens,
|
||||||
|
"llm.token_count.prompt": payload.usage?.prompt_tokens,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!result) {
|
if (!result) {
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ export interface ChatCompletionRequest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface ChatCompletionResponse {
|
export interface ChatCompletionResponse {
|
||||||
|
id?: string | undefined;
|
||||||
model?: string;
|
model?: string;
|
||||||
choices?: Array<{
|
choices?: Array<{
|
||||||
finish_reason?: string | null;
|
finish_reason?: string | null;
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { maskTraceContent } from "./spans.js";
|
||||||
|
|
||||||
|
describe("maskTraceContent", () => {
|
||||||
|
it("redacts common PII and secrets before prompt tracing", () => {
|
||||||
|
const masked = maskTraceContent(
|
||||||
|
"email a@example.com phone 010-1234-5678 token=super-secret sk-testsecret123",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(masked).toContain("[redacted-email]");
|
||||||
|
expect(masked).toContain("[redacted-phone]");
|
||||||
|
expect(masked).toContain("token=[redacted-secret]");
|
||||||
|
expect(masked).toContain("[redacted-token]");
|
||||||
|
expect(masked).not.toContain("a@example.com");
|
||||||
|
expect(masked).not.toContain("010-1234-5678");
|
||||||
|
expect(masked).not.toContain("super-secret");
|
||||||
|
expect(masked).not.toContain("sk-testsecret123");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -7,6 +7,8 @@ import {
|
|||||||
|
|
||||||
import { getServiceName } from '../config.js';
|
import { getServiceName } from '../config.js';
|
||||||
|
|
||||||
|
const MAX_TRACE_CONTENT_CHARS = 2_000;
|
||||||
|
|
||||||
function compactAttributes(attributes?: Attributes): Attributes {
|
function compactAttributes(attributes?: Attributes): Attributes {
|
||||||
if (!attributes) {
|
if (!attributes) {
|
||||||
return {};
|
return {};
|
||||||
@@ -17,6 +19,26 @@ function compactAttributes(attributes?: Attributes): Attributes {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function maskTraceContent(content: string): string {
|
||||||
|
const masked = content
|
||||||
|
.replace(
|
||||||
|
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
|
||||||
|
'[redacted-email]',
|
||||||
|
)
|
||||||
|
.replace(/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/gu, '[redacted-phone]')
|
||||||
|
.replace(/\b(?:Bearer\s+|sk-|pk-)[A-Za-z0-9._-]{8,}\b/gu, '[redacted-token]')
|
||||||
|
.replace(
|
||||||
|
/\b(password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+/giu,
|
||||||
|
'$1=[redacted-secret]',
|
||||||
|
);
|
||||||
|
|
||||||
|
if (masked.length <= MAX_TRACE_CONTENT_CHARS) {
|
||||||
|
return masked;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${masked.slice(0, MAX_TRACE_CONTENT_CHARS)}...[truncated]`;
|
||||||
|
}
|
||||||
|
|
||||||
export async function withSpan<T>(
|
export async function withSpan<T>(
|
||||||
name: string,
|
name: string,
|
||||||
attributes: Attributes | undefined,
|
attributes: Attributes | undefined,
|
||||||
|
|||||||
@@ -1,12 +1,18 @@
|
|||||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||||
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
|
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
|
||||||
|
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
||||||
import {
|
import {
|
||||||
ConsoleSpanExporter,
|
ConsoleSpanExporter,
|
||||||
ParentBasedSampler,
|
ParentBasedSampler,
|
||||||
TraceIdRatioBasedSampler,
|
TraceIdRatioBasedSampler,
|
||||||
} from '@opentelemetry/sdk-trace-base';
|
} from '@opentelemetry/sdk-trace-base';
|
||||||
|
|
||||||
import { getOtelExporter, getOtelSamplingRatio, getServiceName } from '../config.js';
|
import {
|
||||||
|
getOtelExporter,
|
||||||
|
getOtelSamplingRatio,
|
||||||
|
getPhoenixOtlpEndpoint,
|
||||||
|
getServiceName,
|
||||||
|
} from '../config.js';
|
||||||
import { getLogger } from './logger.js';
|
import { getLogger } from './logger.js';
|
||||||
|
|
||||||
let sdk: NodeSDK | null = null;
|
let sdk: NodeSDK | null = null;
|
||||||
@@ -18,7 +24,8 @@ export function initTracing(): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const exporterType = getOtelExporter();
|
const exporterType = getOtelExporter();
|
||||||
if (exporterType === 'none') {
|
const phoenixEndpoint = getPhoenixOtlpEndpoint();
|
||||||
|
if (exporterType === 'none' && !phoenixEndpoint) {
|
||||||
const logger = getLogger();
|
const logger = getLogger();
|
||||||
logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init');
|
logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init');
|
||||||
started = true;
|
started = true;
|
||||||
@@ -28,7 +35,9 @@ export function initTracing(): void {
|
|||||||
process.env['OTEL_SERVICE_NAME'] ??= getServiceName();
|
process.env['OTEL_SERVICE_NAME'] ??= getServiceName();
|
||||||
|
|
||||||
const logger = getLogger();
|
const logger = getLogger();
|
||||||
const exporter = new ConsoleSpanExporter();
|
const exporter = phoenixEndpoint
|
||||||
|
? new OTLPTraceExporter({ url: phoenixEndpoint })
|
||||||
|
: new ConsoleSpanExporter();
|
||||||
|
|
||||||
sdk = new NodeSDK({
|
sdk = new NodeSDK({
|
||||||
instrumentations: [getNodeAutoInstrumentations()],
|
instrumentations: [getNodeAutoInstrumentations()],
|
||||||
@@ -44,6 +53,7 @@ export function initTracing(): void {
|
|||||||
logger.debug(
|
logger.debug(
|
||||||
{
|
{
|
||||||
exporter: getOtelExporter(),
|
exporter: getOtelExporter(),
|
||||||
|
phoenixOtlpEndpoint: phoenixEndpoint,
|
||||||
samplingRatio: getOtelSamplingRatio(),
|
samplingRatio: getOtelSamplingRatio(),
|
||||||
},
|
},
|
||||||
'[Tracing] Initialized',
|
'[Tracing] Initialized',
|
||||||
|
|||||||
Reference in new issue
Block a user