Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
+112
@@ -29,6 +29,14 @@ function getRatioNumber(
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function getBoolean(envKey: string, fallback: boolean): boolean {
|
||||
const raw = process.env[envKey]?.trim().toLowerCase();
|
||||
if (!raw) return fallback;
|
||||
if (raw === "1" || raw === "true" || raw === "yes") return true;
|
||||
if (raw === "0" || raw === "false" || raw === "no") return false;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function getNodeEnv(): "dev" | "test" | "prod" {
|
||||
const value = process.env["NODE_ENV"]?.trim();
|
||||
return value === "test" || value === "prod" ? value : "dev";
|
||||
@@ -118,6 +126,94 @@ export function getEvalStreamMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000);
|
||||
}
|
||||
|
||||
export function getEvalAggTickMs(): number {
|
||||
return getPositiveNumber("EVAL_AGG_TICK_MS", 300_000);
|
||||
}
|
||||
|
||||
export function getEvalAggTtlSec(): number {
|
||||
return getPositiveNumber("EVAL_AGG_TTL_SEC", 604_800);
|
||||
}
|
||||
|
||||
export function getEvalAggLatencySampleCap(): number {
|
||||
return getPositiveNumber("EVAL_AGG_LATENCY_SAMPLE_CAP", 200);
|
||||
}
|
||||
|
||||
export function getEvalArchiveTickMs(): number {
|
||||
return getPositiveNumber("EVAL_ARCHIVE_TICK_MS", 3_600_000);
|
||||
}
|
||||
|
||||
export function getEvalArchiveTtlDays(): number {
|
||||
return getPositiveNumber("EVAL_ARCHIVE_TTL_DAYS", 30);
|
||||
}
|
||||
|
||||
export function getEvalArchiveBatchSize(): number {
|
||||
return getPositiveNumber("EVAL_ARCHIVE_BATCH_SIZE", 500);
|
||||
}
|
||||
|
||||
export function getEvalDriftTickMs(): number {
|
||||
return getPositiveNumber("EVAL_DRIFT_TICK_MS", 86_400_000);
|
||||
}
|
||||
|
||||
export function getEvalPromoteTickMs(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_TICK_MS", 86_400_000);
|
||||
}
|
||||
|
||||
export function getEvalPromoteLowScore(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_LOW_SCORE", 2);
|
||||
}
|
||||
|
||||
export function getEvalPromoteRareThreshold(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_RARE_THRESHOLD", 5);
|
||||
}
|
||||
|
||||
export function getEvalPromotePerCategoryLimit(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_PER_CATEGORY_LIMIT", 10);
|
||||
}
|
||||
|
||||
export function getEvalReviewTickMs(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_TICK_MS", 300_000);
|
||||
}
|
||||
|
||||
export function getEvalReviewQueueMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_QUEUE_MAXLEN", 500);
|
||||
}
|
||||
|
||||
export function getEvalReviewLabelMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_LABEL_MAXLEN", 5_000);
|
||||
}
|
||||
|
||||
export function getEvalReviewPerCategoryLimit(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_PER_CATEGORY_LIMIT", 5);
|
||||
}
|
||||
|
||||
export function getEvalReviewDedupeTtlSec(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_DEDUPE_TTL_SEC", 604_800);
|
||||
}
|
||||
|
||||
export function getEvalCalibrationMinCorr(): number {
|
||||
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
||||
}
|
||||
|
||||
export function getLlmProposeModel(): string {
|
||||
return process.env["LLM_PROPOSE_MODEL"]?.trim() || getLlmJudgeModel();
|
||||
}
|
||||
|
||||
export function getLlmProposeMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_PROPOSE_MAX_TOKENS", 1_024);
|
||||
}
|
||||
|
||||
export function getEvalTournamentMaxMatches(): number {
|
||||
return getPositiveNumber("EVAL_TOURNAMENT_MAX_MATCHES", 50);
|
||||
}
|
||||
|
||||
export function getEvalTournamentMinCases(): number {
|
||||
return getPositiveNumber("EVAL_TOURNAMENT_MIN_CASES", 10);
|
||||
}
|
||||
|
||||
export function getPromptVariantId(): string | undefined {
|
||||
return process.env["PROMPT_VARIANT_ID"]?.trim() || undefined;
|
||||
}
|
||||
|
||||
export function getMaxConversationMessages(): number {
|
||||
return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20);
|
||||
}
|
||||
@@ -154,6 +250,18 @@ export function getLlmJudgeMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128);
|
||||
}
|
||||
|
||||
export function getEvalProcessLlmJudgeEnabled(): boolean {
|
||||
return getBoolean("EVAL_PROCESS_LLM_JUDGE_ENABLED", false);
|
||||
}
|
||||
|
||||
export function getEvalProcessLlmJudgeSamplingRate(): number {
|
||||
return getRatioNumber("EVAL_PROCESS_LLM_JUDGE_SAMPLING_RATE", 0.1);
|
||||
}
|
||||
|
||||
export function getEvalProcessLlmJudgeMaxTokens(): number {
|
||||
return getPositiveNumber("EVAL_PROCESS_LLM_JUDGE_MAX_TOKENS", 256);
|
||||
}
|
||||
|
||||
export function getSafetyMarginMs(): number {
|
||||
return getPositiveNumber("SAFETY_MARGIN_MS", 500);
|
||||
}
|
||||
@@ -303,6 +411,10 @@ export function getAlertThresholdLlmErrorRate(): number {
|
||||
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
|
||||
}
|
||||
|
||||
export function getAlertThresholdDriftPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15);
|
||||
}
|
||||
|
||||
export function getAlertRateLimitSec(): number {
|
||||
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
|
||||
}
|
||||
Reference in new issue
Block a user