Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+33 -11
View File
@@ -1,5 +1,5 @@
import { loadEnvironmentFromNodeEnv } from "./common/env-loader.js";
import { getServiceName } from "./config.js";
import { getOpenRouterApiUrl, getOpenRouterEnabled, getServiceName } from "./config.js";
import type { GarnetClient } from "./common/garnet.js";
import type { NatsClient } from "./common/nats.js";
@@ -21,6 +21,7 @@ const { NatsClient: RuntimeNatsClient } = await import("./common/nats.js");
const { Agent } = await import("./agent/index.js");
const { ConversationStore } = await import("./conversation/index.js");
const { OnboardingSkill } = await import("./skills/onboarding/index.js");
const { ShadowEvalBus } = await import("./eval/shadow-bus.js");
const { LlmClient } = await import("./llm/index.js");
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
@@ -53,22 +54,22 @@ function parseEndpointList(
function toProviderEndpoints(
urls: string[],
provider?: "default" | "cliproxy",
): Array<string | { url: string; provider?: "default" | "cliproxy" }> {
if (!provider || provider === "default") {
provider?: "llama" | "cliproxy" | "openrouter",
): Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }> {
if (!provider) {
return urls;
}
return urls.map((url) => ({ url, provider }));
}
function formatEndpointList(
endpoints: Array<string | { url: string; provider?: "default" | "cliproxy" }>,
endpoints: Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }>,
): string {
return endpoints
.map((endpoint) =>
typeof endpoint === "string"
? endpoint
: `${endpoint.provider ?? "default"}:${endpoint.url}`,
: `${endpoint.provider}:${endpoint.url}`,
)
.join(", ");
}
@@ -135,30 +136,50 @@ async function bootstrap(): Promise<void> {
getRawSecret("LLAMA_SERVER_API_URL"),
process.env["LLAMA_SERVER_API_URL"],
);
const openRouterEndpoints = getOpenRouterEnabled()
? parseEndpointList(
getRawSecret("OPENROUTER_API_URL"),
process.env["OPENROUTER_API_URL"],
getOpenRouterApiUrl(),
)
: [];
const llmEndpoints = [
...toProviderEndpoints(cliProxyEndpoints, "cliproxy"),
...toProviderEndpoints(generalLlmEndpoints),
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
...toProviderEndpoints(generalLlmEndpoints, "llama"),
];
const llmApiKey = firstNonEmpty(
const cliproxyApiKey = firstNonEmpty(
getRawSecret("CLI_PROXY_API_KEY"),
process.env["CLI_PROXY_API_KEY"],
);
const openRouterApiKey = firstNonEmpty(
getRawSecret("OPENROUTER_API_KEY"),
process.env["OPENROUTER_API_KEY"],
);
const llmApiKey = firstNonEmpty(
getRawSecret("AGENT_LLM_API_KEY"),
process.env["AGENT_LLM_API_KEY"],
getRawSecret("LLAMA_SERVER_API_KEY"),
process.env["LLAMA_SERVER_API_KEY"],
);
const llmModel = firstNonEmpty(
getRawSecret("CLI_PROXY_LLM_MODEL"),
process.env["CLI_PROXY_LLM_MODEL"],
getRawSecret("AGENT_LLM_MODEL"),
process.env["AGENT_LLM_MODEL"],
);
const providerApiKeys = Object.fromEntries(
[
["cliproxy", cliproxyApiKey],
["llama", llmApiKey],
["openrouter", openRouterApiKey],
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
) as Partial<Record<"cliproxy" | "llama" | "openrouter", string>>;
const llmClient = new LlmClient({
endpoints: llmEndpoints,
garnetClient,
apiKey: llmApiKey,
model: llmModel,
providerApiKeys,
});
if (llmEndpoints.length > 0) {
@@ -197,7 +218,8 @@ async function bootstrap(): Promise<void> {
const agent = new Agent(natsClient, conversationStore);
// Register skills
const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService);
const shadowEvalBus = new ShadowEvalBus(garnetClient);
const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService, shadowEvalBus);
agent.registerSkill(onboardingSkill);
installShutdownHooks(async () => {