Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+146 -2
View File
@@ -65,8 +65,12 @@ export function getOtelSamplingRatio(): number {
return getRatioNumber("OTEL_SAMPLING_RATIO", fallback);
}
export function getOtelExporter(): "stdout" {
return "stdout";
export function getOtelExporter(): "stdout" | "none" {
const env = process.env["OTEL_EXPORTER"]?.trim().toLowerCase();
if (env === "none" || env === "stdout") {
return env;
}
return getNodeEnv() === "dev" ? "none" : "stdout";
}
export function getRequestSubject(): string {
@@ -75,6 +79,10 @@ export function getRequestSubject(): string {
);
}
export function getAgentRequestTimeoutMs(): number {
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
}
export function getOnboardingSessionKeyPrefix(): string {
return process.env["ONBOARDING_SESSION_KEY_PREFIX"]?.trim() || "lolympus";
}
@@ -95,6 +103,10 @@ export function getBackgroundStreamMaxLen(): number {
return getPositiveNumber("BACKGROUND_STREAM_MAXLEN", 5_000);
}
export function getEvalStreamMaxLen(): number {
return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000);
}
export function getMaxConversationMessages(): number {
return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20);
}
@@ -123,14 +135,110 @@ export function getClassifierMaxTokens(): number {
return getPositiveNumber("CLASSIFIER_MAX_TOKENS", 16);
}
export function getLlmJudgeModel(): string {
return process.env["LLM_JUDGE_MODEL"]?.trim() || "gpt-5.3-codex-spark";
}
export function getLlmJudgeMaxTokens(): number {
return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128);
}
export function getSafetyMarginMs(): number {
return getPositiveNumber("SAFETY_MARGIN_MS", 500);
}
export function getDeadlineNearThresholdMin(): number {
return getPositiveNumber("DEADLINE_NEAR_THRESHOLD_MIN", 30);
}
export function getOpenRouterEnabled(): boolean {
const value = process.env["OPENROUTER_ENABLED"]?.trim().toLowerCase();
return value === "1" || value === "true" || value === "yes";
}
export function getOpenRouterApiUrl(): string {
return process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1";
}
export function getTier0Model(): string {
return (
process.env["TIER0_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.3-codex-spark"
);
}
export function getTier1PlannerModel(): string {
return (
process.env["TIER1_PLANNER_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.4"
);
}
export function getTier1ResponderModel(): string {
return (
process.env["TIER1_RESPONDER_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.3-codex-spark"
);
}
export function getTier2CriticModel(): string {
return (
process.env["TIER2_CRITIC_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.4"
);
}
export function getOpenRouterTier1Model(): string {
return (
process.env["OPENROUTER_TIER1_MODEL"]?.trim() ||
"openai/gpt-5.3-codex-spark"
);
}
export function getOpenRouterTier2Model(): string {
return process.env["OPENROUTER_TIER2_MODEL"]?.trim() || "openai/gpt-5.4";
}
export function getLlmMaxTokens(): number {
return getPositiveNumber("LLM_MAX_TOKENS", 768);
}
export function getTier0MaxTokens(): number {
return getPositiveNumber("TIER0_MAX_TOKENS", 384);
}
export function getTier1MaxTokens(): number {
return getPositiveNumber("TIER1_MAX_TOKENS", 512);
}
export function getTier2MaxTokens(): number {
return getPositiveNumber("TIER2_MAX_TOKENS", 512);
}
export function getPlannerMaxTokens(): number {
return getPositiveNumber("PLANNER_MAX_TOKENS", 256);
}
export function getCriticMaxTokens(): number {
return getPositiveNumber("CRITIC_MAX_TOKENS", 512);
}
export function getTier0EstimatedCallMs(): number {
return getPositiveNumber("TIER0_ESTIMATED_CALL_MS", 600);
}
export function getTier1EstimatedCallMs(): number {
return getPositiveNumber("TIER1_ESTIMATED_CALL_MS", 1200);
}
export function getTier2EstimatedCallMs(): number {
return getPositiveNumber("TIER2_ESTIMATED_CALL_MS", 1400);
}
export function getLlmTemperature(): number {
const raw = process.env["LLM_TEMPERATURE"]?.trim();
if (raw) {
@@ -151,3 +259,39 @@ export function getMongoDbProjectId(): string | undefined {
export function getRoomContextLimit(): number {
return getPositiveNumber("ROOM_CONTEXT_LIMIT", 8);
}
export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
switch (tier) {
case 0:
return getRatioNumber(
"SHADOW_EVAL_SAMPLING_RATE_TIER0",
getNodeEnv() === "prod" ? 0.1 : 1.0,
);
case 1:
return getRatioNumber(
"SHADOW_EVAL_SAMPLING_RATE_TIER1",
getNodeEnv() === "prod" ? 0.5 : 1.0,
);
case 2:
return getRatioNumber(
"SHADOW_EVAL_SAMPLING_RATE_TIER2",
1.0,
);
}
}
export function getAlertThresholdP95LatencyMs(): number {
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
}
export function getAlertThresholdEvalScoreDropPct(): number {
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
}
export function getAlertThresholdLlmErrorRate(): number {
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
}
export function getAlertRateLimitSec(): number {
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
}