Files
lolympus-agent/src/llm/llm-client.ts
T

270 lines
8.1 KiB
TypeScript

import type {
ChatCompletionOptions,
ChatMessage,
ChatCompletionResult,
ChatCompletionResponse,
LlmClientOptions,
LlmEndpointConfig,
ProviderName,
} from "./types.js";
import { getRequestLogger } from "@obs/context";
import { withLlmSpan } from "@obs/spans";
import {
getLlmHealthTimeoutMs,
getLlmRequestTimeoutMs,
getLlmTemperature,
} from "../config.js";
import { LlmCircuitBreaker } from "./circuit.js";
import { getProviderAdapter } from "./providers/index.js";
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
const DEFAULT_MODEL = "Qwen3.5-4B";
function normalizeEndpointConfig(
endpoint: string | { url: string; provider?: ProviderName | "default" | undefined },
): LlmEndpointConfig {
const raw =
typeof endpoint === "string"
? { provider: "llama" as const, url: endpoint }
: { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url };
const provider = getProviderAdapter(raw.provider);
return {
completionUrl: buildCompletionUrl(raw.url),
healthUrl: provider.createHealthUrl(raw.url),
provider: raw.provider,
};
}
function buildCompletionUrl(endpoint: string): string {
try {
const url = new URL(endpoint);
const normalizedPath = url.pathname.replace(/\/+$/u, "");
if (!normalizedPath) {
url.pathname = "/v1/chat/completions";
} else if (
normalizedPath.endsWith("/v1") ||
normalizedPath.endsWith("/api/v1")
) {
url.pathname = `${normalizedPath}/chat/completions`;
}
url.search = "";
url.hash = "";
return url.toString();
} catch {
if (endpoint.includes("/v1/chat/completions")) {
return endpoint;
}
return `${endpoint.replace(/\/+$/u, "")}/v1/chat/completions`;
}
}
function getRequestSignal(
timeoutMs: number,
externalSignal?: AbortSignal,
): AbortSignal {
const timeoutSignal = AbortSignal.timeout(timeoutMs);
if (!externalSignal) {
return timeoutSignal;
}
if (typeof AbortSignal.any === "function") {
return AbortSignal.any([timeoutSignal, externalSignal]);
}
const controller = new AbortController();
const abort = (): void => controller.abort();
timeoutSignal.addEventListener("abort", abort, { once: true });
externalSignal.addEventListener("abort", abort, { once: true });
return controller.signal;
}
export class LlmClient {
private readonly circuit: LlmCircuitBreaker;
private readonly endpoints: LlmEndpointConfig[];
private readonly apiKey: string | undefined;
private readonly llamaModel: string;
private readonly providerApiKeys: Partial<Record<ProviderName, string>>;
constructor(options: LlmClientOptions) {
this.circuit = new LlmCircuitBreaker(options.garnetClient);
this.endpoints = options.endpoints.map(normalizeEndpointConfig);
this.apiKey = options.apiKey;
this.llamaModel = options.model ?? DEFAULT_MODEL;
this.providerApiKeys = options.providerApiKeys ?? {};
}
private getHeaders(
provider: ProviderName,
includeJson = false,
): Record<string, string> {
const headers: Record<string, string> = {};
if (includeJson) {
headers["Content-Type"] = "application/json";
}
const apiKey = this.providerApiKeys[provider] ?? this.apiKey;
if (apiKey) {
headers["Authorization"] = `Bearer ${apiKey}`;
}
return headers;
}
private async checkHealth(endpoint: LlmEndpointConfig): Promise<boolean> {
const provider = getProviderAdapter(endpoint.provider);
if (!provider.supportsHealthCheck) {
return true;
}
try {
const response = await fetch(endpoint.healthUrl, {
headers: this.getHeaders(endpoint.provider),
method: "GET",
signal: AbortSignal.timeout(getLlmHealthTimeoutMs()),
});
if (endpoint.provider === "cliproxy") {
return response.status === 200;
}
return response.ok;
} catch {
return false;
}
}
async chatCompletion(
messages: ChatMessage[],
options?: ChatCompletionOptions,
): Promise<ChatCompletionResult | null> {
const logger = getRequestLogger();
if (this.endpoints.length === 0) {
logger.warn("[LLM] No endpoints configured");
return null;
}
const orderedEndpoints = selectProviderOrder(this.endpoints, options);
for (const endpoint of orderedEndpoints) {
const provider = getProviderAdapter(endpoint.provider);
const healthy = await this.circuit.canUse(
endpoint.completionUrl,
() => this.checkHealth(endpoint),
);
if (!healthy) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
},
"[LLM] Endpoint unavailable via circuit breaker",
);
continue;
}
const requestTimeoutMs = Math.max(
1,
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
);
const models = selectModelsForProvider(provider.name, options, this.llamaModel);
let endpointSucceeded = false;
for (const model of models) {
try {
const request = provider.prepareRequest({
jsonMode: options?.jsonMode,
maxTokens: selectMaxTokensForTier(options),
messages,
model,
temperature: options?.temperature ?? getLlmTemperature(),
tools: options?.tools,
});
const llmResult = await withLlmSpan(
{
"gen_ai.request.model": request.model,
"gen_ai.system": provider.name,
},
async (span) => {
const response = await fetch(endpoint.completionUrl, {
body: JSON.stringify(request),
headers: this.getHeaders(provider.name, true),
method: "POST",
signal: getRequestSignal(requestTimeoutMs, options?.signal),
});
if (!response.ok) {
const body = (await response.text()).slice(0, 500);
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
model: request.model,
status: response.status,
body,
},
"[LLM] Request failed",
);
return null;
}
const payload = (await response.json()) as ChatCompletionResponse;
const result = provider.toResult(payload, endpoint.completionUrl);
span.setAttributes({
"gen_ai.response.finish_reasons": payload.choices
?.map((choice) => choice.finish_reason ?? "unknown")
.filter((reason): reason is string => Boolean(reason)),
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
});
if (!result) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
model: request.model,
},
"[LLM] Empty response from endpoint",
);
return null;
}
return result;
},
);
if (llmResult) {
await this.circuit.recordSuccess(endpoint.completionUrl);
endpointSucceeded = true;
return llmResult;
}
} catch (error) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
model,
error: error instanceof Error ? error.message : String(error),
},
"[LLM] Request error",
);
}
}
if (!endpointSucceeded) {
await this.circuit.recordFailure(endpoint.completionUrl);
}
}
logger.warn("[LLM] All endpoints failed");
return null;
}
hasEndpoints(): boolean {
return this.endpoints.length > 0;
}
}