Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
+146
-2
@@ -65,8 +65,12 @@ export function getOtelSamplingRatio(): number {
|
||||
return getRatioNumber("OTEL_SAMPLING_RATIO", fallback);
|
||||
}
|
||||
|
||||
export function getOtelExporter(): "stdout" {
|
||||
return "stdout";
|
||||
export function getOtelExporter(): "stdout" | "none" {
|
||||
const env = process.env["OTEL_EXPORTER"]?.trim().toLowerCase();
|
||||
if (env === "none" || env === "stdout") {
|
||||
return env;
|
||||
}
|
||||
return getNodeEnv() === "dev" ? "none" : "stdout";
|
||||
}
|
||||
|
||||
export function getRequestSubject(): string {
|
||||
@@ -75,6 +79,10 @@ export function getRequestSubject(): string {
|
||||
);
|
||||
}
|
||||
|
||||
export function getAgentRequestTimeoutMs(): number {
|
||||
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
|
||||
}
|
||||
|
||||
export function getOnboardingSessionKeyPrefix(): string {
|
||||
return process.env["ONBOARDING_SESSION_KEY_PREFIX"]?.trim() || "lolympus";
|
||||
}
|
||||
@@ -95,6 +103,10 @@ export function getBackgroundStreamMaxLen(): number {
|
||||
return getPositiveNumber("BACKGROUND_STREAM_MAXLEN", 5_000);
|
||||
}
|
||||
|
||||
export function getEvalStreamMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000);
|
||||
}
|
||||
|
||||
export function getMaxConversationMessages(): number {
|
||||
return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20);
|
||||
}
|
||||
@@ -123,14 +135,110 @@ export function getClassifierMaxTokens(): number {
|
||||
return getPositiveNumber("CLASSIFIER_MAX_TOKENS", 16);
|
||||
}
|
||||
|
||||
export function getLlmJudgeModel(): string {
|
||||
return process.env["LLM_JUDGE_MODEL"]?.trim() || "gpt-5.3-codex-spark";
|
||||
}
|
||||
|
||||
export function getLlmJudgeMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128);
|
||||
}
|
||||
|
||||
export function getSafetyMarginMs(): number {
|
||||
return getPositiveNumber("SAFETY_MARGIN_MS", 500);
|
||||
}
|
||||
|
||||
export function getDeadlineNearThresholdMin(): number {
|
||||
return getPositiveNumber("DEADLINE_NEAR_THRESHOLD_MIN", 30);
|
||||
}
|
||||
|
||||
export function getOpenRouterEnabled(): boolean {
|
||||
const value = process.env["OPENROUTER_ENABLED"]?.trim().toLowerCase();
|
||||
return value === "1" || value === "true" || value === "yes";
|
||||
}
|
||||
|
||||
export function getOpenRouterApiUrl(): string {
|
||||
return process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1";
|
||||
}
|
||||
|
||||
export function getTier0Model(): string {
|
||||
return (
|
||||
process.env["TIER0_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.3-codex-spark"
|
||||
);
|
||||
}
|
||||
|
||||
export function getTier1PlannerModel(): string {
|
||||
return (
|
||||
process.env["TIER1_PLANNER_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.4"
|
||||
);
|
||||
}
|
||||
|
||||
export function getTier1ResponderModel(): string {
|
||||
return (
|
||||
process.env["TIER1_RESPONDER_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.3-codex-spark"
|
||||
);
|
||||
}
|
||||
|
||||
export function getTier2CriticModel(): string {
|
||||
return (
|
||||
process.env["TIER2_CRITIC_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.4"
|
||||
);
|
||||
}
|
||||
|
||||
export function getOpenRouterTier1Model(): string {
|
||||
return (
|
||||
process.env["OPENROUTER_TIER1_MODEL"]?.trim() ||
|
||||
"openai/gpt-5.3-codex-spark"
|
||||
);
|
||||
}
|
||||
|
||||
export function getOpenRouterTier2Model(): string {
|
||||
return process.env["OPENROUTER_TIER2_MODEL"]?.trim() || "openai/gpt-5.4";
|
||||
}
|
||||
|
||||
export function getLlmMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_MAX_TOKENS", 768);
|
||||
}
|
||||
|
||||
export function getTier0MaxTokens(): number {
|
||||
return getPositiveNumber("TIER0_MAX_TOKENS", 384);
|
||||
}
|
||||
|
||||
export function getTier1MaxTokens(): number {
|
||||
return getPositiveNumber("TIER1_MAX_TOKENS", 512);
|
||||
}
|
||||
|
||||
export function getTier2MaxTokens(): number {
|
||||
return getPositiveNumber("TIER2_MAX_TOKENS", 512);
|
||||
}
|
||||
|
||||
export function getPlannerMaxTokens(): number {
|
||||
return getPositiveNumber("PLANNER_MAX_TOKENS", 256);
|
||||
}
|
||||
|
||||
export function getCriticMaxTokens(): number {
|
||||
return getPositiveNumber("CRITIC_MAX_TOKENS", 512);
|
||||
}
|
||||
|
||||
export function getTier0EstimatedCallMs(): number {
|
||||
return getPositiveNumber("TIER0_ESTIMATED_CALL_MS", 600);
|
||||
}
|
||||
|
||||
export function getTier1EstimatedCallMs(): number {
|
||||
return getPositiveNumber("TIER1_ESTIMATED_CALL_MS", 1200);
|
||||
}
|
||||
|
||||
export function getTier2EstimatedCallMs(): number {
|
||||
return getPositiveNumber("TIER2_ESTIMATED_CALL_MS", 1400);
|
||||
}
|
||||
|
||||
export function getLlmTemperature(): number {
|
||||
const raw = process.env["LLM_TEMPERATURE"]?.trim();
|
||||
if (raw) {
|
||||
@@ -151,3 +259,39 @@ export function getMongoDbProjectId(): string | undefined {
|
||||
export function getRoomContextLimit(): number {
|
||||
return getPositiveNumber("ROOM_CONTEXT_LIMIT", 8);
|
||||
}
|
||||
|
||||
export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
|
||||
switch (tier) {
|
||||
case 0:
|
||||
return getRatioNumber(
|
||||
"SHADOW_EVAL_SAMPLING_RATE_TIER0",
|
||||
getNodeEnv() === "prod" ? 0.1 : 1.0,
|
||||
);
|
||||
case 1:
|
||||
return getRatioNumber(
|
||||
"SHADOW_EVAL_SAMPLING_RATE_TIER1",
|
||||
getNodeEnv() === "prod" ? 0.5 : 1.0,
|
||||
);
|
||||
case 2:
|
||||
return getRatioNumber(
|
||||
"SHADOW_EVAL_SAMPLING_RATE_TIER2",
|
||||
1.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function getAlertThresholdP95LatencyMs(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
|
||||
}
|
||||
|
||||
export function getAlertThresholdEvalScoreDropPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
|
||||
}
|
||||
|
||||
export function getAlertThresholdLlmErrorRate(): number {
|
||||
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
|
||||
}
|
||||
|
||||
export function getAlertRateLimitSec(): number {
|
||||
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
|
||||
}
|
||||
Reference in new issue
Block a user