Initialize onboarding agent scaffold
This commit is contained in:
commit
4294f3c92e
57 files changed
+14388
No files matched your search
@@ -0,0 +1,250 @@
|
||||
import type {
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatMessage,
|
||||
LlmClientOptions,
|
||||
LlmEndpointConfig,
|
||||
} from "./types.js";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import { withLlmSpan } from "@obs/spans";
|
||||
import {
|
||||
getLlmHealthTimeoutMs,
|
||||
getLlmMaxTokens,
|
||||
getLlmRequestTimeoutMs,
|
||||
getLlmTemperature,
|
||||
} from "../config.js";
|
||||
import { LlmCircuitBreaker } from "./circuit.js";
|
||||
|
||||
const DEFAULT_MODEL = "Qwen3.5-4B";
|
||||
|
||||
function buildCliProxyHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, "");
|
||||
return `${normalized.replace(/\/+$/u, "")}/`;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeEndpointConfig(
|
||||
endpoint: string | { url: string; provider?: "default" | "cliproxy" },
|
||||
): LlmEndpointConfig {
|
||||
const raw =
|
||||
typeof endpoint === "string"
|
||||
? { provider: "default" as const, url: endpoint }
|
||||
: { provider: endpoint.provider ?? "default", url: endpoint.url };
|
||||
|
||||
return {
|
||||
completionUrl: buildCompletionUrl(raw.url),
|
||||
healthUrl:
|
||||
raw.provider === "cliproxy"
|
||||
? buildCliProxyHealthUrl(raw.url)
|
||||
: buildHealthUrl(raw.url),
|
||||
provider: raw.provider,
|
||||
};
|
||||
}
|
||||
|
||||
function buildCompletionUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
if (!url.pathname || url.pathname === "/") {
|
||||
url.pathname = "/v1/chat/completions";
|
||||
}
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
if (endpoint.includes("/v1/chat/completions")) {
|
||||
return endpoint;
|
||||
}
|
||||
return `${endpoint.replace(/\/+$/u, "")}/v1/chat/completions`;
|
||||
}
|
||||
}
|
||||
|
||||
function buildHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/health";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
if (endpoint.includes("/v1/chat/completions")) {
|
||||
return `${endpoint.split("/v1/chat/completions")[0]}/health`;
|
||||
}
|
||||
return `${endpoint.replace(/\/+$/u, "")}/health`;
|
||||
}
|
||||
}
|
||||
|
||||
export class LlmClient {
|
||||
private readonly circuit: LlmCircuitBreaker;
|
||||
private readonly endpoints: LlmEndpointConfig[];
|
||||
private readonly apiKey: string | undefined;
|
||||
private readonly model: string;
|
||||
|
||||
constructor(options: LlmClientOptions) {
|
||||
this.circuit = new LlmCircuitBreaker(options.garnetClient);
|
||||
this.endpoints = options.endpoints.map(normalizeEndpointConfig);
|
||||
this.apiKey = options.apiKey;
|
||||
this.model = options.model ?? DEFAULT_MODEL;
|
||||
}
|
||||
|
||||
private getHeaders(includeJson = false): Record<string, string> {
|
||||
const headers: Record<string, string> = {};
|
||||
if (includeJson) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
if (this.apiKey) {
|
||||
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
private async checkHealth(endpoint: LlmEndpointConfig): Promise<boolean> {
|
||||
try {
|
||||
const response = await fetch(endpoint.healthUrl, {
|
||||
headers: this.getHeaders(),
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(getLlmHealthTimeoutMs()),
|
||||
});
|
||||
if (endpoint.provider === "cliproxy") {
|
||||
return response.status === 200;
|
||||
}
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async chatCompletion(
|
||||
messages: ChatMessage[],
|
||||
options?: {
|
||||
jsonMode?: boolean | undefined;
|
||||
maxTokens?: number | undefined;
|
||||
temperature?: number | undefined;
|
||||
},
|
||||
): Promise<{ content: string; endpoint: string } | null> {
|
||||
const logger = getRequestLogger();
|
||||
|
||||
if (this.endpoints.length === 0) {
|
||||
logger.warn("[LLM] No endpoints configured");
|
||||
return null;
|
||||
}
|
||||
|
||||
const request: ChatCompletionRequest = {
|
||||
model: this.model,
|
||||
messages,
|
||||
max_tokens: options?.maxTokens ?? getLlmMaxTokens(),
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
};
|
||||
|
||||
if (options?.jsonMode) {
|
||||
request.response_format = { type: "json_object" };
|
||||
}
|
||||
|
||||
for (const endpoint of this.endpoints) {
|
||||
const healthy = await this.circuit.canUse(
|
||||
endpoint.completionUrl,
|
||||
() => this.checkHealth(endpoint),
|
||||
);
|
||||
if (!healthy) {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
},
|
||||
"[LLM] Endpoint unavailable via circuit breaker",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const llmResult = await withLlmSpan(
|
||||
{
|
||||
"gen_ai.request.model": request.model,
|
||||
"gen_ai.system": endpoint.provider ?? 'default',
|
||||
},
|
||||
async (span) => {
|
||||
const response = await fetch(endpoint.completionUrl, {
|
||||
body: JSON.stringify(request),
|
||||
headers: this.getHeaders(true),
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(getLlmRequestTimeoutMs()),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = (await response.text()).slice(0, 500);
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
status: response.status,
|
||||
body,
|
||||
},
|
||||
"[LLM] Request failed",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as ChatCompletionResponse;
|
||||
const content = payload.choices?.[0]?.message?.content;
|
||||
|
||||
span.setAttributes({
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
?.map((choice) => choice.finish_reason ?? "unknown")
|
||||
.filter((reason): reason is string => Boolean(reason)),
|
||||
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
|
||||
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
||||
});
|
||||
|
||||
if (typeof content !== "string" || !content.trim()) {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
},
|
||||
"[LLM] Empty response from endpoint",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
content: content.trim(),
|
||||
endpoint: endpoint.completionUrl,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
if (!llmResult) {
|
||||
await this.circuit.recordFailure(endpoint.completionUrl);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.circuit.recordSuccess(endpoint.completionUrl);
|
||||
return llmResult;
|
||||
} catch (error) {
|
||||
await this.circuit.recordFailure(endpoint.completionUrl);
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"[LLM] Request error",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn("[LLM] All endpoints failed");
|
||||
return null;
|
||||
}
|
||||
|
||||
hasEndpoints(): boolean {
|
||||
return this.endpoints.length > 0;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user