270 lines
8.1 KiB
TypeScript
270 lines
8.1 KiB
TypeScript
import type {
|
|
ChatCompletionOptions,
|
|
ChatMessage,
|
|
ChatCompletionResult,
|
|
ChatCompletionResponse,
|
|
LlmClientOptions,
|
|
LlmEndpointConfig,
|
|
ProviderName,
|
|
} from "./types.js";
|
|
import { getRequestLogger } from "@obs/context";
|
|
import { withLlmSpan } from "@obs/spans";
|
|
import {
|
|
getLlmHealthTimeoutMs,
|
|
getLlmRequestTimeoutMs,
|
|
getLlmTemperature,
|
|
} from "../config.js";
|
|
import { LlmCircuitBreaker } from "./circuit.js";
|
|
import { getProviderAdapter } from "./providers/index.js";
|
|
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
|
|
|
|
const DEFAULT_MODEL = "Qwen3.5-4B";
|
|
|
|
function normalizeEndpointConfig(
|
|
endpoint: string | { url: string; provider?: ProviderName | "default" | undefined },
|
|
): LlmEndpointConfig {
|
|
const raw =
|
|
typeof endpoint === "string"
|
|
? { provider: "llama" as const, url: endpoint }
|
|
: { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url };
|
|
const provider = getProviderAdapter(raw.provider);
|
|
|
|
return {
|
|
completionUrl: buildCompletionUrl(raw.url),
|
|
healthUrl: provider.createHealthUrl(raw.url),
|
|
provider: raw.provider,
|
|
};
|
|
}
|
|
|
|
function buildCompletionUrl(endpoint: string): string {
|
|
try {
|
|
const url = new URL(endpoint);
|
|
const normalizedPath = url.pathname.replace(/\/+$/u, "");
|
|
if (!normalizedPath) {
|
|
url.pathname = "/v1/chat/completions";
|
|
} else if (
|
|
normalizedPath.endsWith("/v1") ||
|
|
normalizedPath.endsWith("/api/v1")
|
|
) {
|
|
url.pathname = `${normalizedPath}/chat/completions`;
|
|
}
|
|
url.search = "";
|
|
url.hash = "";
|
|
return url.toString();
|
|
} catch {
|
|
if (endpoint.includes("/v1/chat/completions")) {
|
|
return endpoint;
|
|
}
|
|
return `${endpoint.replace(/\/+$/u, "")}/v1/chat/completions`;
|
|
}
|
|
}
|
|
|
|
function getRequestSignal(
|
|
timeoutMs: number,
|
|
externalSignal?: AbortSignal,
|
|
): AbortSignal {
|
|
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
|
|
if (!externalSignal) {
|
|
return timeoutSignal;
|
|
}
|
|
|
|
if (typeof AbortSignal.any === "function") {
|
|
return AbortSignal.any([timeoutSignal, externalSignal]);
|
|
}
|
|
|
|
const controller = new AbortController();
|
|
const abort = (): void => controller.abort();
|
|
|
|
timeoutSignal.addEventListener("abort", abort, { once: true });
|
|
externalSignal.addEventListener("abort", abort, { once: true });
|
|
|
|
return controller.signal;
|
|
}
|
|
|
|
export class LlmClient {
|
|
private readonly circuit: LlmCircuitBreaker;
|
|
private readonly endpoints: LlmEndpointConfig[];
|
|
private readonly apiKey: string | undefined;
|
|
private readonly llamaModel: string;
|
|
private readonly providerApiKeys: Partial<Record<ProviderName, string>>;
|
|
|
|
constructor(options: LlmClientOptions) {
|
|
this.circuit = new LlmCircuitBreaker(options.garnetClient);
|
|
this.endpoints = options.endpoints.map(normalizeEndpointConfig);
|
|
this.apiKey = options.apiKey;
|
|
this.llamaModel = options.model ?? DEFAULT_MODEL;
|
|
this.providerApiKeys = options.providerApiKeys ?? {};
|
|
}
|
|
|
|
private getHeaders(
|
|
provider: ProviderName,
|
|
includeJson = false,
|
|
): Record<string, string> {
|
|
const headers: Record<string, string> = {};
|
|
if (includeJson) {
|
|
headers["Content-Type"] = "application/json";
|
|
}
|
|
const apiKey = this.providerApiKeys[provider] ?? this.apiKey;
|
|
if (apiKey) {
|
|
headers["Authorization"] = `Bearer ${apiKey}`;
|
|
}
|
|
return headers;
|
|
}
|
|
|
|
private async checkHealth(endpoint: LlmEndpointConfig): Promise<boolean> {
|
|
const provider = getProviderAdapter(endpoint.provider);
|
|
if (!provider.supportsHealthCheck) {
|
|
return true;
|
|
}
|
|
|
|
try {
|
|
const response = await fetch(endpoint.healthUrl, {
|
|
headers: this.getHeaders(endpoint.provider),
|
|
method: "GET",
|
|
signal: AbortSignal.timeout(getLlmHealthTimeoutMs()),
|
|
});
|
|
if (endpoint.provider === "cliproxy") {
|
|
return response.status === 200;
|
|
}
|
|
return response.ok;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async chatCompletion(
|
|
messages: ChatMessage[],
|
|
options?: ChatCompletionOptions,
|
|
): Promise<ChatCompletionResult | null> {
|
|
const logger = getRequestLogger();
|
|
|
|
if (this.endpoints.length === 0) {
|
|
logger.warn("[LLM] No endpoints configured");
|
|
return null;
|
|
}
|
|
|
|
const orderedEndpoints = selectProviderOrder(this.endpoints, options);
|
|
|
|
for (const endpoint of orderedEndpoints) {
|
|
const provider = getProviderAdapter(endpoint.provider);
|
|
const healthy = await this.circuit.canUse(
|
|
endpoint.completionUrl,
|
|
() => this.checkHealth(endpoint),
|
|
);
|
|
if (!healthy) {
|
|
logger.warn(
|
|
{
|
|
endpoint: endpoint.completionUrl,
|
|
provider: provider.name,
|
|
},
|
|
"[LLM] Endpoint unavailable via circuit breaker",
|
|
);
|
|
continue;
|
|
}
|
|
|
|
const requestTimeoutMs = Math.max(
|
|
1,
|
|
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
|
|
);
|
|
const models = selectModelsForProvider(provider.name, options, this.llamaModel);
|
|
let endpointSucceeded = false;
|
|
|
|
for (const model of models) {
|
|
try {
|
|
const request = provider.prepareRequest({
|
|
jsonMode: options?.jsonMode,
|
|
maxTokens: selectMaxTokensForTier(options),
|
|
messages,
|
|
model,
|
|
temperature: options?.temperature ?? getLlmTemperature(),
|
|
tools: options?.tools,
|
|
});
|
|
|
|
const llmResult = await withLlmSpan(
|
|
{
|
|
"gen_ai.request.model": request.model,
|
|
"gen_ai.system": provider.name,
|
|
},
|
|
async (span) => {
|
|
const response = await fetch(endpoint.completionUrl, {
|
|
body: JSON.stringify(request),
|
|
headers: this.getHeaders(provider.name, true),
|
|
method: "POST",
|
|
signal: getRequestSignal(requestTimeoutMs, options?.signal),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const body = (await response.text()).slice(0, 500);
|
|
logger.warn(
|
|
{
|
|
endpoint: endpoint.completionUrl,
|
|
provider: provider.name,
|
|
model: request.model,
|
|
status: response.status,
|
|
body,
|
|
},
|
|
"[LLM] Request failed",
|
|
);
|
|
return null;
|
|
}
|
|
|
|
const payload = (await response.json()) as ChatCompletionResponse;
|
|
const result = provider.toResult(payload, endpoint.completionUrl);
|
|
|
|
span.setAttributes({
|
|
"gen_ai.response.finish_reasons": payload.choices
|
|
?.map((choice) => choice.finish_reason ?? "unknown")
|
|
.filter((reason): reason is string => Boolean(reason)),
|
|
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
|
|
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
|
});
|
|
|
|
if (!result) {
|
|
logger.warn(
|
|
{
|
|
endpoint: endpoint.completionUrl,
|
|
provider: provider.name,
|
|
model: request.model,
|
|
},
|
|
"[LLM] Empty response from endpoint",
|
|
);
|
|
return null;
|
|
}
|
|
|
|
return result;
|
|
},
|
|
);
|
|
|
|
if (llmResult) {
|
|
await this.circuit.recordSuccess(endpoint.completionUrl);
|
|
endpointSucceeded = true;
|
|
return llmResult;
|
|
}
|
|
} catch (error) {
|
|
logger.warn(
|
|
{
|
|
endpoint: endpoint.completionUrl,
|
|
provider: provider.name,
|
|
model,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
},
|
|
"[LLM] Request error",
|
|
);
|
|
}
|
|
}
|
|
|
|
if (!endpointSucceeded) {
|
|
await this.circuit.recordFailure(endpoint.completionUrl);
|
|
}
|
|
}
|
|
|
|
logger.warn("[LLM] All endpoints failed");
|
|
return null;
|
|
}
|
|
|
|
hasEndpoints(): boolean {
|
|
return this.endpoints.length > 0;
|
|
}
|
|
}
|