Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+207
View File
@@ -0,0 +1,207 @@
import { loadEnvironmentFromNodeEnv } from "../common/env-loader.js";
import {
getOpenRouterApiUrl,
getOpenRouterEnabled,
} from "../config.js";
loadEnvironmentFromNodeEnv();
const baseServiceName = process.env["SERVICE_NAME"]?.trim() || "lolympus-agent";
process.env["SERVICE_NAME"] = `${baseServiceName}-worker`;
const { getLogger, initLogger } = await import("../observability/logger.js");
initLogger();
const logger = getLogger();
const { initTracing, shutdownTracing } = await import(
"../observability/tracing.js"
);
initTracing();
const { GarnetClient } = await import("../common/garnet.js");
const { NatsClient } = await import("../common/nats.js");
const { LlmClient } = await import("../llm/index.js");
const { getRawSecret } = await import("../common/secret-manager.js");
const { ConversationStore } = await import("../conversation/index.js");
const { ShadowWorker } = await import("../eval/shadow-worker.js");
const { AlertService } = await import("../observability/alert.js");
function firstNonEmpty(
...values: Array<string | null | undefined>
): string | undefined {
for (const value of values) {
if (value?.trim()) {
return value.trim();
}
}
return undefined;
}
function parseEndpointList(
...values: Array<string | null | undefined>
): string[] {
const items: string[] = [];
for (const value of values) {
if (!value?.trim()) {
continue;
}
for (const item of value.split(/[\n,]/u)) {
const trimmed = item.trim();
if (trimmed) {
items.push(trimmed);
}
}
}
return [...new Set(items)];
}
function toProviderEndpoints(
urls: string[],
provider?: "llama" | "cliproxy" | "openrouter",
): Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }> {
if (!provider) {
return urls;
}
return urls.map((url) => ({ url, provider }));
}
function installShutdownHooks(
shutdown: (reason: string, error?: unknown) => Promise<void>,
): void {
let shuttingDown = false;
const runShutdown = async (
reason: string,
error?: unknown,
): Promise<void> => {
if (shuttingDown) {
return;
}
shuttingDown = true;
if (error) {
logger.error({ err: error }, `[Background] ${reason}`);
} else {
logger.info(`[Background] ${reason}`);
}
try {
await shutdown(reason, error);
} catch (shutdownError) {
logger.error({ err: shutdownError }, "[Background] Cleanup failed");
} finally {
process.exit(error ? 1 : 0);
}
};
for (const signal of ["SIGINT", "SIGTERM"] as const) {
process.once(signal, () => {
void runShutdown(signal);
});
}
process.once("uncaughtException", (error) => {
void runShutdown("uncaughtException", error);
});
process.once("unhandledRejection", (reason) => {
void runShutdown("unhandledRejection", reason);
});
}
async function bootstrap(): Promise<void> {
const serviceName = process.env["SERVICE_NAME"] ?? "lolympus-agent-worker";
const garnetClient = new GarnetClient();
await garnetClient.connect();
const natsClient = new NatsClient();
await natsClient.connect();
const cliproxyEndpoints = parseEndpointList(
getRawSecret("CLI_PROXY_API_URLS"),
process.env["CLI_PROXY_API_URLS"],
);
const generalLlmEndpoints = parseEndpointList(
getRawSecret("AGENT_LLM_API_URLS"),
process.env["AGENT_LLM_API_URLS"],
getRawSecret("LLAMA_SERVER_API_URL"),
process.env["LLAMA_SERVER_API_URL"],
);
const openRouterEndpoints = getOpenRouterEnabled()
? parseEndpointList(
getRawSecret("OPENROUTER_API_URL"),
process.env["OPENROUTER_API_URL"],
getOpenRouterApiUrl(),
)
: [];
const llmApiKey = firstNonEmpty(
getRawSecret("AGENT_LLM_API_KEY"),
process.env["AGENT_LLM_API_KEY"],
getRawSecret("LLAMA_SERVER_API_KEY"),
process.env["LLAMA_SERVER_API_KEY"],
);
const providerApiKeys = Object.fromEntries(
[
[
"cliproxy",
firstNonEmpty(
getRawSecret("CLI_PROXY_API_KEY"),
process.env["CLI_PROXY_API_KEY"],
),
],
["llama", llmApiKey],
[
"openrouter",
firstNonEmpty(
getRawSecret("OPENROUTER_API_KEY"),
process.env["OPENROUTER_API_KEY"],
),
],
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
) as Partial<Record<"cliproxy" | "llama" | "openrouter", string>>;
const llmClient = new LlmClient({
endpoints: [
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
...toProviderEndpoints(generalLlmEndpoints, "llama"),
],
garnetClient,
apiKey: llmApiKey,
model: firstNonEmpty(
getRawSecret("AGENT_LLM_MODEL"),
process.env["AGENT_LLM_MODEL"],
),
providerApiKeys,
});
const conversationStore = new ConversationStore(garnetClient);
const alertService = new AlertService(garnetClient, natsClient);
const worker = new ShadowWorker(
garnetClient,
llmClient,
conversationStore,
alertService,
);
installShutdownHooks(async () => {
await worker.stop();
await natsClient.close();
await garnetClient.close();
await shutdownTracing();
});
await worker.start();
logger.info({ serviceName }, "✓ Background worker bootstrapped");
}
try {
await bootstrap();
} catch (error) {
logger.error({ err: error }, "[Background] Fatal startup error");
await shutdownTracing();
process.exitCode = 1;
}