import type { ChatCompletionOptions, ChatMessage, ChatCompletionResult, ChatCompletionResponse, LlmClientOptions, LlmEndpointConfig, ProviderName, } from "./types.js"; import { getRequestLogger } from "@obs/context"; import { withLlmSpan } from "@obs/spans"; import { getLlmHealthTimeoutMs, getLlmRequestTimeoutMs, getLlmTemperature, } from "../config.js"; import { LlmCircuitBreaker } from "./circuit.js"; import { getProviderAdapter } from "./providers/index.js"; import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js"; const DEFAULT_MODEL = "Qwen3.5-4B"; function normalizeEndpointConfig( endpoint: string | { url: string; provider?: ProviderName | "default" | undefined }, ): LlmEndpointConfig { const raw = typeof endpoint === "string" ? { provider: "llama" as const, url: endpoint } : { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url }; const provider = getProviderAdapter(raw.provider); return { completionUrl: buildCompletionUrl(raw.url), healthUrl: provider.createHealthUrl(raw.url), provider: raw.provider, }; } function buildCompletionUrl(endpoint: string): string { try { const url = new URL(endpoint); const normalizedPath = url.pathname.replace(/\/+$/u, ""); if (!normalizedPath) { url.pathname = "/v1/chat/completions"; } else if ( normalizedPath.endsWith("/v1") || normalizedPath.endsWith("/api/v1") ) { url.pathname = `${normalizedPath}/chat/completions`; } url.search = ""; url.hash = ""; return url.toString(); } catch { if (endpoint.includes("/v1/chat/completions")) { return endpoint; } return `${endpoint.replace(/\/+$/u, "")}/v1/chat/completions`; } } function getRequestSignal( timeoutMs: number, externalSignal?: AbortSignal, ): AbortSignal { const timeoutSignal = AbortSignal.timeout(timeoutMs); if (!externalSignal) { return timeoutSignal; } if (typeof AbortSignal.any === "function") { return AbortSignal.any([timeoutSignal, externalSignal]); } const controller = new AbortController(); const abort = (): void => controller.abort(); timeoutSignal.addEventListener("abort", abort, { once: true }); externalSignal.addEventListener("abort", abort, { once: true }); return controller.signal; } export class LlmClient { private readonly circuit: LlmCircuitBreaker; private readonly endpoints: LlmEndpointConfig[]; private readonly apiKey: string | undefined; private readonly llamaModel: string; private readonly providerApiKeys: Partial>; constructor(options: LlmClientOptions) { this.circuit = new LlmCircuitBreaker(options.garnetClient); this.endpoints = options.endpoints.map(normalizeEndpointConfig); this.apiKey = options.apiKey; this.llamaModel = options.model ?? DEFAULT_MODEL; this.providerApiKeys = options.providerApiKeys ?? {}; } private getHeaders( provider: ProviderName, includeJson = false, ): Record { const headers: Record = {}; if (includeJson) { headers["Content-Type"] = "application/json"; } const apiKey = this.providerApiKeys[provider] ?? this.apiKey; if (apiKey) { headers["Authorization"] = `Bearer ${apiKey}`; } return headers; } private async checkHealth(endpoint: LlmEndpointConfig): Promise { const provider = getProviderAdapter(endpoint.provider); if (!provider.supportsHealthCheck) { return true; } try { const response = await fetch(endpoint.healthUrl, { headers: this.getHeaders(endpoint.provider), method: "GET", signal: AbortSignal.timeout(getLlmHealthTimeoutMs()), }); if (endpoint.provider === "cliproxy") { return response.status === 200; } return response.ok; } catch { return false; } } async chatCompletion( messages: ChatMessage[], options?: ChatCompletionOptions, ): Promise { const logger = getRequestLogger(); if (this.endpoints.length === 0) { logger.warn("[LLM] No endpoints configured"); return null; } const orderedEndpoints = selectProviderOrder(this.endpoints, options); for (const endpoint of orderedEndpoints) { const provider = getProviderAdapter(endpoint.provider); const healthy = await this.circuit.canUse( endpoint.completionUrl, () => this.checkHealth(endpoint), ); if (!healthy) { logger.warn( { endpoint: endpoint.completionUrl, provider: provider.name, }, "[LLM] Endpoint unavailable via circuit breaker", ); continue; } const requestTimeoutMs = Math.max( 1, Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY), ); const models = selectModelsForProvider(provider.name, options, this.llamaModel); let endpointSucceeded = false; for (const model of models) { try { const request = provider.prepareRequest({ jsonMode: options?.jsonMode, maxTokens: selectMaxTokensForTier(options), messages, model, temperature: options?.temperature ?? getLlmTemperature(), tools: options?.tools, }); const llmResult = await withLlmSpan( { "gen_ai.request.model": request.model, "gen_ai.system": provider.name, }, async (span) => { const response = await fetch(endpoint.completionUrl, { body: JSON.stringify(request), headers: this.getHeaders(provider.name, true), method: "POST", signal: getRequestSignal(requestTimeoutMs, options?.signal), }); if (!response.ok) { const body = (await response.text()).slice(0, 500); logger.warn( { endpoint: endpoint.completionUrl, provider: provider.name, model: request.model, status: response.status, body, }, "[LLM] Request failed", ); return null; } const payload = (await response.json()) as ChatCompletionResponse; const result = provider.toResult(payload, endpoint.completionUrl); span.setAttributes({ "gen_ai.response.finish_reasons": payload.choices ?.map((choice) => choice.finish_reason ?? "unknown") .filter((reason): reason is string => Boolean(reason)), "gen_ai.usage.input_tokens": payload.usage?.prompt_tokens, "gen_ai.usage.output_tokens": payload.usage?.completion_tokens, }); if (!result) { logger.warn( { endpoint: endpoint.completionUrl, provider: provider.name, model: request.model, }, "[LLM] Empty response from endpoint", ); return null; } return result; }, ); if (llmResult) { await this.circuit.recordSuccess(endpoint.completionUrl); endpointSucceeded = true; return llmResult; } } catch (error) { logger.warn( { endpoint: endpoint.completionUrl, provider: provider.name, model, error: error instanceof Error ? error.message : String(error), }, "[LLM] Request error", ); } } if (!endpointSucceeded) { await this.circuit.recordFailure(endpoint.completionUrl); } } logger.warn("[LLM] All endpoints failed"); return null; } hasEndpoints(): boolean { return this.endpoints.length > 0; } }