Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+76 -71
View File
@@ -1,49 +1,37 @@
import type {
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionOptions,
ChatMessage,
ChatCompletionResult,
ChatCompletionResponse,
LlmClientOptions,
LlmEndpointConfig,
ProviderName,
} from "./types.js";
import { getRequestLogger } from "@obs/context";
import { withLlmSpan } from "@obs/spans";
import {
getLlmHealthTimeoutMs,
getLlmMaxTokens,
getLlmRequestTimeoutMs,
getLlmTemperature,
} from "../config.js";
import { LlmCircuitBreaker } from "./circuit.js";
import { getProviderAdapter } from "./providers/index.js";
import { selectMaxTokensForTier, selectModelForProvider, selectProviderOrder } from "./routing.js";
const DEFAULT_MODEL = "Qwen3.5-4B";
function buildCliProxyHealthUrl(endpoint: string): string {
try {
const url = new URL(endpoint);
url.pathname = "/";
url.search = "";
url.hash = "";
return url.toString();
} catch {
const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, "");
return `${normalized.replace(/\/+$/u, "")}/`;
}
}
function normalizeEndpointConfig(
endpoint: string | { url: string; provider?: "default" | "cliproxy" },
endpoint: string | { url: string; provider?: ProviderName | "default" | undefined },
): LlmEndpointConfig {
const raw =
typeof endpoint === "string"
? { provider: "default" as const, url: endpoint }
: { provider: endpoint.provider ?? "default", url: endpoint.url };
? { provider: "llama" as const, url: endpoint }
: { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url };
const provider = getProviderAdapter(raw.provider);
return {
completionUrl: buildCompletionUrl(raw.url),
healthUrl:
raw.provider === "cliproxy"
? buildCliProxyHealthUrl(raw.url)
: buildHealthUrl(raw.url),
healthUrl: provider.createHealthUrl(raw.url),
provider: raw.provider,
};
}
@@ -65,49 +53,68 @@ function buildCompletionUrl(endpoint: string): string {
}
}
function buildHealthUrl(endpoint: string): string {
try {
const url = new URL(endpoint);
url.pathname = "/health";
url.search = "";
url.hash = "";
return url.toString();
} catch {
if (endpoint.includes("/v1/chat/completions")) {
return `${endpoint.split("/v1/chat/completions")[0]}/health`;
}
return `${endpoint.replace(/\/+$/u, "")}/health`;
function getRequestSignal(
timeoutMs: number,
externalSignal?: AbortSignal,
): AbortSignal {
const timeoutSignal = AbortSignal.timeout(timeoutMs);
if (!externalSignal) {
return timeoutSignal;
}
if (typeof AbortSignal.any === "function") {
return AbortSignal.any([timeoutSignal, externalSignal]);
}
const controller = new AbortController();
const abort = (): void => controller.abort();
timeoutSignal.addEventListener("abort", abort, { once: true });
externalSignal.addEventListener("abort", abort, { once: true });
return controller.signal;
}
export class LlmClient {
private readonly circuit: LlmCircuitBreaker;
private readonly endpoints: LlmEndpointConfig[];
private readonly apiKey: string | undefined;
private readonly model: string;
private readonly llamaModel: string;
private readonly providerApiKeys: Partial<Record<ProviderName, string>>;
constructor(options: LlmClientOptions) {
this.circuit = new LlmCircuitBreaker(options.garnetClient);
this.endpoints = options.endpoints.map(normalizeEndpointConfig);
this.apiKey = options.apiKey;
this.model = options.model ?? DEFAULT_MODEL;
this.llamaModel = options.model ?? DEFAULT_MODEL;
this.providerApiKeys = options.providerApiKeys ?? {};
}
private getHeaders(includeJson = false): Record<string, string> {
private getHeaders(
provider: ProviderName,
includeJson = false,
): Record<string, string> {
const headers: Record<string, string> = {};
if (includeJson) {
headers["Content-Type"] = "application/json";
}
if (this.apiKey) {
headers["Authorization"] = `Bearer ${this.apiKey}`;
const apiKey = this.providerApiKeys[provider] ?? this.apiKey;
if (apiKey) {
headers["Authorization"] = `Bearer ${apiKey}`;
}
return headers;
}
private async checkHealth(endpoint: LlmEndpointConfig): Promise<boolean> {
const provider = getProviderAdapter(endpoint.provider);
if (!provider.supportsHealthCheck) {
return true;
}
try {
const response = await fetch(endpoint.healthUrl, {
headers: this.getHeaders(),
headers: this.getHeaders(endpoint.provider),
method: "GET",
signal: AbortSignal.timeout(getLlmHealthTimeoutMs()),
});
@@ -122,12 +129,8 @@ export class LlmClient {
async chatCompletion(
messages: ChatMessage[],
options?: {
jsonMode?: boolean | undefined;
maxTokens?: number | undefined;
temperature?: number | undefined;
},
): Promise<{ content: string; endpoint: string } | null> {
options?: ChatCompletionOptions,
): Promise<ChatCompletionResult | null> {
const logger = getRequestLogger();
if (this.endpoints.length === 0) {
@@ -135,18 +138,10 @@ export class LlmClient {
return null;
}
const request: ChatCompletionRequest = {
model: this.model,
messages,
max_tokens: options?.maxTokens ?? getLlmMaxTokens(),
temperature: options?.temperature ?? getLlmTemperature(),
};
const orderedEndpoints = selectProviderOrder(this.endpoints, options);
if (options?.jsonMode) {
request.response_format = { type: "json_object" };
}
for (const endpoint of this.endpoints) {
for (const endpoint of orderedEndpoints) {
const provider = getProviderAdapter(endpoint.provider);
const healthy = await this.circuit.canUse(
endpoint.completionUrl,
() => this.checkHealth(endpoint),
@@ -155,7 +150,7 @@ export class LlmClient {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
},
"[LLM] Endpoint unavailable via circuit breaker",
);
@@ -163,17 +158,30 @@ export class LlmClient {
}
try {
const requestTimeoutMs = Math.max(
1,
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
);
const request = provider.prepareRequest({
jsonMode: options?.jsonMode,
maxTokens: selectMaxTokensForTier(options),
messages,
model: selectModelForProvider(provider.name, options, this.llamaModel),
temperature: options?.temperature ?? getLlmTemperature(),
tools: options?.tools,
});
const llmResult = await withLlmSpan(
{
"gen_ai.request.model": request.model,
"gen_ai.system": endpoint.provider ?? 'default',
"gen_ai.system": provider.name,
},
async (span) => {
const response = await fetch(endpoint.completionUrl, {
body: JSON.stringify(request),
headers: this.getHeaders(true),
headers: this.getHeaders(provider.name, true),
method: "POST",
signal: AbortSignal.timeout(getLlmRequestTimeoutMs()),
signal: getRequestSignal(requestTimeoutMs, options?.signal),
});
if (!response.ok) {
@@ -181,7 +189,7 @@ export class LlmClient {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
status: response.status,
body,
},
@@ -191,7 +199,7 @@ export class LlmClient {
}
const payload = (await response.json()) as ChatCompletionResponse;
const content = payload.choices?.[0]?.message?.content;
const result = provider.toResult(payload, endpoint.completionUrl);
span.setAttributes({
"gen_ai.response.finish_reasons": payload.choices
@@ -201,21 +209,18 @@ export class LlmClient {
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
});
if (typeof content !== "string" || !content.trim()) {
if (!result) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
},
"[LLM] Empty response from endpoint",
);
return null;
}
return {
content: content.trim(),
endpoint: endpoint.completionUrl,
};
return result;
},
);
@@ -231,7 +236,7 @@ export class LlmClient {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
error: error instanceof Error ? error.message : String(error),
},
"[LLM] Request error",