Tune LLM routing and prompts

This commit is contained in:
dahyeon committed 2026-04-21 15:24:21 -07:00
1 parent 9c013f7d4f
commit 60f33b8f4b
19 files changed
+736 -803

No files matched your search

+84 -76
View File
@@ -16,7 +16,7 @@ import {
} from "../config.js";
import { LlmCircuitBreaker } from "./circuit.js";
import { getProviderAdapter } from "./providers/index.js";
import { selectMaxTokensForTier, selectModelForProvider, selectProviderOrder } from "./routing.js";
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
const DEFAULT_MODEL = "Qwen3.5-4B";
@@ -157,91 +157,99 @@ export class LlmClient {
continue;
}
try {
const requestTimeoutMs = Math.max(
1,
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
);
const request = provider.prepareRequest({
jsonMode: options?.jsonMode,
maxTokens: selectMaxTokensForTier(options),
messages,
model: selectModelForProvider(provider.name, options, this.llamaModel),
temperature: options?.temperature ?? getLlmTemperature(),
tools: options?.tools,
});
const requestTimeoutMs = Math.max(
1,
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
);
const models = selectModelsForProvider(provider.name, options, this.llamaModel);
let endpointSucceeded = false;
const llmResult = await withLlmSpan(
{
"gen_ai.request.model": request.model,
"gen_ai.system": provider.name,
},
async (span) => {
const response = await fetch(endpoint.completionUrl, {
body: JSON.stringify(request),
headers: this.getHeaders(provider.name, true),
method: "POST",
signal: getRequestSignal(requestTimeoutMs, options?.signal),
});
for (const model of models) {
try {
const request = provider.prepareRequest({
jsonMode: options?.jsonMode,
maxTokens: selectMaxTokensForTier(options),
messages,
model,
temperature: options?.temperature ?? getLlmTemperature(),
tools: options?.tools,
});
if (!response.ok) {
const body = (await response.text()).slice(0, 500);
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
status: response.status,
body,
},
"[LLM] Request failed",
);
return null;
}
const llmResult = await withLlmSpan(
{
"gen_ai.request.model": request.model,
"gen_ai.system": provider.name,
},
async (span) => {
const response = await fetch(endpoint.completionUrl, {
body: JSON.stringify(request),
headers: this.getHeaders(provider.name, true),
method: "POST",
signal: getRequestSignal(requestTimeoutMs, options?.signal),
});
const payload = (await response.json()) as ChatCompletionResponse;
const result = provider.toResult(payload, endpoint.completionUrl);
if (!response.ok) {
const body = (await response.text()).slice(0, 500);
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
model: request.model,
status: response.status,
body,
},
"[LLM] Request failed",
);
return null;
}
span.setAttributes({
"gen_ai.response.finish_reasons": payload.choices
?.map((choice) => choice.finish_reason ?? "unknown")
.filter((reason): reason is string => Boolean(reason)),
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
});
const payload = (await response.json()) as ChatCompletionResponse;
const result = provider.toResult(payload, endpoint.completionUrl);
if (!result) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
},
"[LLM] Empty response from endpoint",
);
return null;
}
span.setAttributes({
"gen_ai.response.finish_reasons": payload.choices
?.map((choice) => choice.finish_reason ?? "unknown")
.filter((reason): reason is string => Boolean(reason)),
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
});
return result;
},
);
if (!result) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
model: request.model,
},
"[LLM] Empty response from endpoint",
);
return null;
}
if (!llmResult) {
await this.circuit.recordFailure(endpoint.completionUrl);
continue;
return result;
},
);
if (llmResult) {
await this.circuit.recordSuccess(endpoint.completionUrl);
endpointSucceeded = true;
return llmResult;
}
} catch (error) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
model,
error: error instanceof Error ? error.message : String(error),
},
"[LLM] Request error",
);
}
}
await this.circuit.recordSuccess(endpoint.completionUrl);
return llmResult;
} catch (error) {
if (!endpointSucceeded) {
await this.circuit.recordFailure(endpoint.completionUrl);
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: provider.name,
error: error instanceof Error ? error.message : String(error),
},
"[LLM] Request error",
);
continue;
}
}