Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
+76
-71
@@ -1,49 +1,37 @@
|
||||
import type {
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionOptions,
|
||||
ChatMessage,
|
||||
ChatCompletionResult,
|
||||
ChatCompletionResponse,
|
||||
LlmClientOptions,
|
||||
LlmEndpointConfig,
|
||||
ProviderName,
|
||||
} from "./types.js";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import { withLlmSpan } from "@obs/spans";
|
||||
import {
|
||||
getLlmHealthTimeoutMs,
|
||||
getLlmMaxTokens,
|
||||
getLlmRequestTimeoutMs,
|
||||
getLlmTemperature,
|
||||
} from "../config.js";
|
||||
import { LlmCircuitBreaker } from "./circuit.js";
|
||||
import { getProviderAdapter } from "./providers/index.js";
|
||||
import { selectMaxTokensForTier, selectModelForProvider, selectProviderOrder } from "./routing.js";
|
||||
|
||||
const DEFAULT_MODEL = "Qwen3.5-4B";
|
||||
|
||||
function buildCliProxyHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, "");
|
||||
return `${normalized.replace(/\/+$/u, "")}/`;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeEndpointConfig(
|
||||
endpoint: string | { url: string; provider?: "default" | "cliproxy" },
|
||||
endpoint: string | { url: string; provider?: ProviderName | "default" | undefined },
|
||||
): LlmEndpointConfig {
|
||||
const raw =
|
||||
typeof endpoint === "string"
|
||||
? { provider: "default" as const, url: endpoint }
|
||||
: { provider: endpoint.provider ?? "default", url: endpoint.url };
|
||||
? { provider: "llama" as const, url: endpoint }
|
||||
: { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url };
|
||||
const provider = getProviderAdapter(raw.provider);
|
||||
|
||||
return {
|
||||
completionUrl: buildCompletionUrl(raw.url),
|
||||
healthUrl:
|
||||
raw.provider === "cliproxy"
|
||||
? buildCliProxyHealthUrl(raw.url)
|
||||
: buildHealthUrl(raw.url),
|
||||
healthUrl: provider.createHealthUrl(raw.url),
|
||||
provider: raw.provider,
|
||||
};
|
||||
}
|
||||
@@ -65,49 +53,68 @@ function buildCompletionUrl(endpoint: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
function buildHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/health";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
if (endpoint.includes("/v1/chat/completions")) {
|
||||
return `${endpoint.split("/v1/chat/completions")[0]}/health`;
|
||||
}
|
||||
return `${endpoint.replace(/\/+$/u, "")}/health`;
|
||||
function getRequestSignal(
|
||||
timeoutMs: number,
|
||||
externalSignal?: AbortSignal,
|
||||
): AbortSignal {
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
|
||||
if (!externalSignal) {
|
||||
return timeoutSignal;
|
||||
}
|
||||
|
||||
if (typeof AbortSignal.any === "function") {
|
||||
return AbortSignal.any([timeoutSignal, externalSignal]);
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const abort = (): void => controller.abort();
|
||||
|
||||
timeoutSignal.addEventListener("abort", abort, { once: true });
|
||||
externalSignal.addEventListener("abort", abort, { once: true });
|
||||
|
||||
return controller.signal;
|
||||
}
|
||||
|
||||
export class LlmClient {
|
||||
private readonly circuit: LlmCircuitBreaker;
|
||||
private readonly endpoints: LlmEndpointConfig[];
|
||||
private readonly apiKey: string | undefined;
|
||||
private readonly model: string;
|
||||
private readonly llamaModel: string;
|
||||
private readonly providerApiKeys: Partial<Record<ProviderName, string>>;
|
||||
|
||||
constructor(options: LlmClientOptions) {
|
||||
this.circuit = new LlmCircuitBreaker(options.garnetClient);
|
||||
this.endpoints = options.endpoints.map(normalizeEndpointConfig);
|
||||
this.apiKey = options.apiKey;
|
||||
this.model = options.model ?? DEFAULT_MODEL;
|
||||
this.llamaModel = options.model ?? DEFAULT_MODEL;
|
||||
this.providerApiKeys = options.providerApiKeys ?? {};
|
||||
}
|
||||
|
||||
private getHeaders(includeJson = false): Record<string, string> {
|
||||
private getHeaders(
|
||||
provider: ProviderName,
|
||||
includeJson = false,
|
||||
): Record<string, string> {
|
||||
const headers: Record<string, string> = {};
|
||||
if (includeJson) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
if (this.apiKey) {
|
||||
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
||||
const apiKey = this.providerApiKeys[provider] ?? this.apiKey;
|
||||
if (apiKey) {
|
||||
headers["Authorization"] = `Bearer ${apiKey}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
private async checkHealth(endpoint: LlmEndpointConfig): Promise<boolean> {
|
||||
const provider = getProviderAdapter(endpoint.provider);
|
||||
if (!provider.supportsHealthCheck) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(endpoint.healthUrl, {
|
||||
headers: this.getHeaders(),
|
||||
headers: this.getHeaders(endpoint.provider),
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(getLlmHealthTimeoutMs()),
|
||||
});
|
||||
@@ -122,12 +129,8 @@ export class LlmClient {
|
||||
|
||||
async chatCompletion(
|
||||
messages: ChatMessage[],
|
||||
options?: {
|
||||
jsonMode?: boolean | undefined;
|
||||
maxTokens?: number | undefined;
|
||||
temperature?: number | undefined;
|
||||
},
|
||||
): Promise<{ content: string; endpoint: string } | null> {
|
||||
options?: ChatCompletionOptions,
|
||||
): Promise<ChatCompletionResult | null> {
|
||||
const logger = getRequestLogger();
|
||||
|
||||
if (this.endpoints.length === 0) {
|
||||
@@ -135,18 +138,10 @@ export class LlmClient {
|
||||
return null;
|
||||
}
|
||||
|
||||
const request: ChatCompletionRequest = {
|
||||
model: this.model,
|
||||
messages,
|
||||
max_tokens: options?.maxTokens ?? getLlmMaxTokens(),
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
};
|
||||
const orderedEndpoints = selectProviderOrder(this.endpoints, options);
|
||||
|
||||
if (options?.jsonMode) {
|
||||
request.response_format = { type: "json_object" };
|
||||
}
|
||||
|
||||
for (const endpoint of this.endpoints) {
|
||||
for (const endpoint of orderedEndpoints) {
|
||||
const provider = getProviderAdapter(endpoint.provider);
|
||||
const healthy = await this.circuit.canUse(
|
||||
endpoint.completionUrl,
|
||||
() => this.checkHealth(endpoint),
|
||||
@@ -155,7 +150,7 @@ export class LlmClient {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
},
|
||||
"[LLM] Endpoint unavailable via circuit breaker",
|
||||
);
|
||||
@@ -163,17 +158,30 @@ export class LlmClient {
|
||||
}
|
||||
|
||||
try {
|
||||
const requestTimeoutMs = Math.max(
|
||||
1,
|
||||
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
|
||||
);
|
||||
const request = provider.prepareRequest({
|
||||
jsonMode: options?.jsonMode,
|
||||
maxTokens: selectMaxTokensForTier(options),
|
||||
messages,
|
||||
model: selectModelForProvider(provider.name, options, this.llamaModel),
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
tools: options?.tools,
|
||||
});
|
||||
|
||||
const llmResult = await withLlmSpan(
|
||||
{
|
||||
"gen_ai.request.model": request.model,
|
||||
"gen_ai.system": endpoint.provider ?? 'default',
|
||||
"gen_ai.system": provider.name,
|
||||
},
|
||||
async (span) => {
|
||||
const response = await fetch(endpoint.completionUrl, {
|
||||
body: JSON.stringify(request),
|
||||
headers: this.getHeaders(true),
|
||||
headers: this.getHeaders(provider.name, true),
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(getLlmRequestTimeoutMs()),
|
||||
signal: getRequestSignal(requestTimeoutMs, options?.signal),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -181,7 +189,7 @@ export class LlmClient {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
status: response.status,
|
||||
body,
|
||||
},
|
||||
@@ -191,7 +199,7 @@ export class LlmClient {
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as ChatCompletionResponse;
|
||||
const content = payload.choices?.[0]?.message?.content;
|
||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||
|
||||
span.setAttributes({
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
@@ -201,21 +209,18 @@ export class LlmClient {
|
||||
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
||||
});
|
||||
|
||||
if (typeof content !== "string" || !content.trim()) {
|
||||
if (!result) {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
},
|
||||
"[LLM] Empty response from endpoint",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
content: content.trim(),
|
||||
endpoint: endpoint.completionUrl,
|
||||
};
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
||||
@@ -231,7 +236,7 @@ export class LlmClient {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"[LLM] Request error",
|
||||
|
||||
Reference in new issue
Block a user