Tune LLM routing and prompts
This commit is contained in:
1 parent
9c013f7d4f
commit
60f33b8f4b
19 files changed
+736
-803
No files matched your search
+84
-76
@@ -16,7 +16,7 @@ import {
|
||||
} from "../config.js";
|
||||
import { LlmCircuitBreaker } from "./circuit.js";
|
||||
import { getProviderAdapter } from "./providers/index.js";
|
||||
import { selectMaxTokensForTier, selectModelForProvider, selectProviderOrder } from "./routing.js";
|
||||
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
|
||||
|
||||
const DEFAULT_MODEL = "Qwen3.5-4B";
|
||||
|
||||
@@ -157,91 +157,99 @@ export class LlmClient {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const requestTimeoutMs = Math.max(
|
||||
1,
|
||||
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
|
||||
);
|
||||
const request = provider.prepareRequest({
|
||||
jsonMode: options?.jsonMode,
|
||||
maxTokens: selectMaxTokensForTier(options),
|
||||
messages,
|
||||
model: selectModelForProvider(provider.name, options, this.llamaModel),
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
tools: options?.tools,
|
||||
});
|
||||
const requestTimeoutMs = Math.max(
|
||||
1,
|
||||
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
|
||||
);
|
||||
const models = selectModelsForProvider(provider.name, options, this.llamaModel);
|
||||
let endpointSucceeded = false;
|
||||
|
||||
const llmResult = await withLlmSpan(
|
||||
{
|
||||
"gen_ai.request.model": request.model,
|
||||
"gen_ai.system": provider.name,
|
||||
},
|
||||
async (span) => {
|
||||
const response = await fetch(endpoint.completionUrl, {
|
||||
body: JSON.stringify(request),
|
||||
headers: this.getHeaders(provider.name, true),
|
||||
method: "POST",
|
||||
signal: getRequestSignal(requestTimeoutMs, options?.signal),
|
||||
});
|
||||
for (const model of models) {
|
||||
try {
|
||||
const request = provider.prepareRequest({
|
||||
jsonMode: options?.jsonMode,
|
||||
maxTokens: selectMaxTokensForTier(options),
|
||||
messages,
|
||||
model,
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
tools: options?.tools,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = (await response.text()).slice(0, 500);
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: provider.name,
|
||||
status: response.status,
|
||||
body,
|
||||
},
|
||||
"[LLM] Request failed",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const llmResult = await withLlmSpan(
|
||||
{
|
||||
"gen_ai.request.model": request.model,
|
||||
"gen_ai.system": provider.name,
|
||||
},
|
||||
async (span) => {
|
||||
const response = await fetch(endpoint.completionUrl, {
|
||||
body: JSON.stringify(request),
|
||||
headers: this.getHeaders(provider.name, true),
|
||||
method: "POST",
|
||||
signal: getRequestSignal(requestTimeoutMs, options?.signal),
|
||||
});
|
||||
|
||||
const payload = (await response.json()) as ChatCompletionResponse;
|
||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||
if (!response.ok) {
|
||||
const body = (await response.text()).slice(0, 500);
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: provider.name,
|
||||
model: request.model,
|
||||
status: response.status,
|
||||
body,
|
||||
},
|
||||
"[LLM] Request failed",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
span.setAttributes({
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
?.map((choice) => choice.finish_reason ?? "unknown")
|
||||
.filter((reason): reason is string => Boolean(reason)),
|
||||
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
|
||||
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
||||
});
|
||||
const payload = (await response.json()) as ChatCompletionResponse;
|
||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||
|
||||
if (!result) {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: provider.name,
|
||||
},
|
||||
"[LLM] Empty response from endpoint",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
span.setAttributes({
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
?.map((choice) => choice.finish_reason ?? "unknown")
|
||||
.filter((reason): reason is string => Boolean(reason)),
|
||||
"gen_ai.usage.input_tokens": payload.usage?.prompt_tokens,
|
||||
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
||||
});
|
||||
|
||||
return result;
|
||||
},
|
||||
);
|
||||
if (!result) {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: provider.name,
|
||||
model: request.model,
|
||||
},
|
||||
"[LLM] Empty response from endpoint",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!llmResult) {
|
||||
await this.circuit.recordFailure(endpoint.completionUrl);
|
||||
continue;
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
||||
if (llmResult) {
|
||||
await this.circuit.recordSuccess(endpoint.completionUrl);
|
||||
endpointSucceeded = true;
|
||||
return llmResult;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: provider.name,
|
||||
model,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"[LLM] Request error",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.circuit.recordSuccess(endpoint.completionUrl);
|
||||
return llmResult;
|
||||
} catch (error) {
|
||||
if (!endpointSucceeded) {
|
||||
await this.circuit.recordFailure(endpoint.completionUrl);
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: provider.name,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"[LLM] Request error",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user