Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit 6f09408423
57 files changed
+7060 -142

No files matched your search

+112
View File
@@ -29,6 +29,14 @@ function getRatioNumber(
: fallback;
}
function getBoolean(envKey: string, fallback: boolean): boolean {
const raw = process.env[envKey]?.trim().toLowerCase();
if (!raw) return fallback;
if (raw === "1" || raw === "true" || raw === "yes") return true;
if (raw === "0" || raw === "false" || raw === "no") return false;
return fallback;
}
export function getNodeEnv(): "dev" | "test" | "prod" {
const value = process.env["NODE_ENV"]?.trim();
return value === "test" || value === "prod" ? value : "dev";
@@ -118,6 +126,94 @@ export function getEvalStreamMaxLen(): number {
return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000);
}
export function getEvalAggTickMs(): number {
return getPositiveNumber("EVAL_AGG_TICK_MS", 300_000);
}
export function getEvalAggTtlSec(): number {
return getPositiveNumber("EVAL_AGG_TTL_SEC", 604_800);
}
export function getEvalAggLatencySampleCap(): number {
return getPositiveNumber("EVAL_AGG_LATENCY_SAMPLE_CAP", 200);
}
export function getEvalArchiveTickMs(): number {
return getPositiveNumber("EVAL_ARCHIVE_TICK_MS", 3_600_000);
}
export function getEvalArchiveTtlDays(): number {
return getPositiveNumber("EVAL_ARCHIVE_TTL_DAYS", 30);
}
export function getEvalArchiveBatchSize(): number {
return getPositiveNumber("EVAL_ARCHIVE_BATCH_SIZE", 500);
}
export function getEvalDriftTickMs(): number {
return getPositiveNumber("EVAL_DRIFT_TICK_MS", 86_400_000);
}
export function getEvalPromoteTickMs(): number {
return getPositiveNumber("EVAL_PROMOTE_TICK_MS", 86_400_000);
}
export function getEvalPromoteLowScore(): number {
return getPositiveNumber("EVAL_PROMOTE_LOW_SCORE", 2);
}
export function getEvalPromoteRareThreshold(): number {
return getPositiveNumber("EVAL_PROMOTE_RARE_THRESHOLD", 5);
}
export function getEvalPromotePerCategoryLimit(): number {
return getPositiveNumber("EVAL_PROMOTE_PER_CATEGORY_LIMIT", 10);
}
export function getEvalReviewTickMs(): number {
return getPositiveNumber("EVAL_REVIEW_TICK_MS", 300_000);
}
export function getEvalReviewQueueMaxLen(): number {
return getPositiveNumber("EVAL_REVIEW_QUEUE_MAXLEN", 500);
}
export function getEvalReviewLabelMaxLen(): number {
return getPositiveNumber("EVAL_REVIEW_LABEL_MAXLEN", 5_000);
}
export function getEvalReviewPerCategoryLimit(): number {
return getPositiveNumber("EVAL_REVIEW_PER_CATEGORY_LIMIT", 5);
}
export function getEvalReviewDedupeTtlSec(): number {
return getPositiveNumber("EVAL_REVIEW_DEDUPE_TTL_SEC", 604_800);
}
export function getEvalCalibrationMinCorr(): number {
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
}
export function getLlmProposeModel(): string {
return process.env["LLM_PROPOSE_MODEL"]?.trim() || getLlmJudgeModel();
}
export function getLlmProposeMaxTokens(): number {
return getPositiveNumber("LLM_PROPOSE_MAX_TOKENS", 1_024);
}
export function getEvalTournamentMaxMatches(): number {
return getPositiveNumber("EVAL_TOURNAMENT_MAX_MATCHES", 50);
}
export function getEvalTournamentMinCases(): number {
return getPositiveNumber("EVAL_TOURNAMENT_MIN_CASES", 10);
}
export function getPromptVariantId(): string | undefined {
return process.env["PROMPT_VARIANT_ID"]?.trim() || undefined;
}
export function getMaxConversationMessages(): number {
return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20);
}
@@ -154,6 +250,18 @@ export function getLlmJudgeMaxTokens(): number {
return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128);
}
export function getEvalProcessLlmJudgeEnabled(): boolean {
return getBoolean("EVAL_PROCESS_LLM_JUDGE_ENABLED", false);
}
export function getEvalProcessLlmJudgeSamplingRate(): number {
return getRatioNumber("EVAL_PROCESS_LLM_JUDGE_SAMPLING_RATE", 0.1);
}
export function getEvalProcessLlmJudgeMaxTokens(): number {
return getPositiveNumber("EVAL_PROCESS_LLM_JUDGE_MAX_TOKENS", 256);
}
export function getSafetyMarginMs(): number {
return getPositiveNumber("SAFETY_MARGIN_MS", 500);
}
@@ -303,6 +411,10 @@ export function getAlertThresholdLlmErrorRate(): number {
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
}
export function getAlertThresholdDriftPct(): number {
return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15);
}
export function getAlertRateLimitSec(): number {
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
}