import type { GarnetClient, StreamEntry } from "@common/garnet"; import { getEvalAggLatencySampleCap, getEvalAggTtlSec, } from "../config.js"; export const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results"; export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor"; const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:"; export interface SliceKey { tier: 0 | 1 | 2; intent: string; phase: string; routingDecision?: string | undefined; abstentionAction?: string | undefined; abstentionReason?: string | undefined; } export interface SliceMetrics { abstentionCount: number; count: number; llmOverallSum: number; llmOverallCount: number; validatorRejectCount: number; validatorSoftWarnCount: number; latencySumMs: number; latencySamples: number[]; processJudgeSum: number; processJudgeCount: number; updatedAt: string; } export interface SliceReportRow { slice: SliceKey; abstentionRate: number; count: number; llmOverallAvg: number | null; processJudgeAvg: number | null; validatorRejectRate: number; validatorSoftWarnRate: number; p50LatencyMs: number | null; p95LatencyMs: number | null; updatedAt: string; } type AggregatorGarnetClient = Pick< GarnetClient, | "expireKey" | "getHash" | "readStreamRange" | "scanKeys" | "setHash" | "setString" >; interface EvalResultObservation { abstentionAction: string | null; latencyMs: number | null; llmOverall: number | null; processJudgeScore: number | null; slice: SliceKey; validatorStatus: string | null; } function asRecord(value: unknown): Record { return typeof value === "object" && value !== null ? (value as Record) : {}; } function parseJsonRecord(value: string | undefined): Record { if (!value?.trim()) { return {}; } try { return asRecord(JSON.parse(value)); } catch { return {}; } } function parseFiniteNumber(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) { return value; } if (typeof value !== "string" || !value.trim()) { return null; } const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function parseTier(value: unknown): 0 | 1 | 2 | null { const parsed = parseFiniteNumber(value); if (parsed === 0 || parsed === 1 || parsed === 2) { return parsed; } return null; } function getNestedString( record: Record, first: string, second: string, ): string | null { const nested = asRecord(record[first]); const value = nested[second]; return typeof value === "string" && value.trim() ? value : null; } function extractLlmOverall(message: Record): number | null { const direct = parseFiniteNumber(message["llmOverall"]); if (direct !== null) { return direct; } const llmJudge = parseJsonRecord(message["llmJudge"]); const scores = asRecord(llmJudge["scores"]); return parseFiniteNumber(scores["overall"]); } function extractProcessJudgeScore( message: Record, ): number | null { const processJudge = parseJsonRecord(message["processJudge"]); return parseFiniteNumber(processJudge["score"]); } function extractValidatorStatus(message: Record): string | null { if (message["validatorStatus"]?.trim()) { return message["validatorStatus"] ?? null; } const validation = parseJsonRecord(message["validation"]); const status = validation["status"]; return typeof status === "string" && status.trim() ? status : null; } function extractAbstention( message: Record, ): { action: string | null; reason: string | null } { const directAction = message["abstentionAction"]?.trim(); const directReason = message["abstentionReason"]?.trim(); if (directAction) { return { action: directAction, reason: directReason || null, }; } const process = parseJsonRecord(message["process"]); const steps = asRecord(process["steps"]); const classify = asRecord(steps["classify"]); const output = asRecord(classify["output"]); const abstention = asRecord(output["abstention"]); const action = asRecord(abstention["action"]); const kind = action["kind"]; const reason = abstention["reason"]; return { action: typeof kind === "string" && kind.trim() ? kind : null, reason: typeof reason === "string" && reason.trim() ? reason : null, }; } function extractRoutingDecision(derived: Record): string | null { const routingDecision = asRecord(derived["routingDecision"]); const kind = routingDecision["kind"]; return typeof kind === "string" && kind.trim() ? kind : null; } function parseObservation(entry: StreamEntry): EvalResultObservation | null { const derived = parseJsonRecord(entry.message["derived"]); const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]); const intent = entry.message["intent"]?.trim() || getNestedString(derived, "intent", "kind"); const phase = entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind"); if (tier === null || !intent || !phase) { return null; } const abstention = extractAbstention(entry.message); const routingDecision = extractRoutingDecision(derived); return { abstentionAction: abstention.action, latencyMs: parseFiniteNumber(entry.message["latencyMs"]), llmOverall: extractLlmOverall(entry.message), processJudgeScore: extractProcessJudgeScore(entry.message), slice: { tier, intent, phase, ...(routingDecision ? { routingDecision } : {}), ...(abstention.action ? { abstentionAction: abstention.action } : {}), ...(abstention.reason ? { abstentionReason: abstention.reason } : {}), }, validatorStatus: extractValidatorStatus(entry.message), }; } function parseNumberField( hash: Record, field: keyof SliceMetrics, ): number { const value = parseFiniteNumber(hash[field]); return value ?? 0; } function parseLatencySamples(hash: Record): number[] { const raw = hash["latencySamples"]; if (!raw?.trim()) { return []; } try { const parsed = JSON.parse(raw) as unknown; if (!Array.isArray(parsed)) { return []; } return parsed.flatMap((item) => { const value = parseFiniteNumber(item); return value === null ? [] : [value]; }); } catch { return []; } } function metricsFromHash(hash: Record): SliceMetrics { return { abstentionCount: parseNumberField(hash, "abstentionCount"), count: parseNumberField(hash, "count"), llmOverallSum: parseNumberField(hash, "llmOverallSum"), llmOverallCount: parseNumberField(hash, "llmOverallCount"), validatorRejectCount: parseNumberField(hash, "validatorRejectCount"), validatorSoftWarnCount: parseNumberField(hash, "validatorSoftWarnCount"), latencySumMs: parseNumberField(hash, "latencySumMs"), latencySamples: parseLatencySamples(hash), processJudgeSum: parseNumberField(hash, "processJudgeSum"), processJudgeCount: parseNumberField(hash, "processJudgeCount"), updatedAt: hash["updatedAt"] ?? new Date(0).toISOString(), }; } function metricsToHash(metrics: SliceMetrics): Record { return { abstentionCount: String(metrics.abstentionCount), count: String(metrics.count), llmOverallSum: String(metrics.llmOverallSum), llmOverallCount: String(metrics.llmOverallCount), validatorRejectCount: String(metrics.validatorRejectCount), validatorSoftWarnCount: String(metrics.validatorSoftWarnCount), latencySumMs: String(metrics.latencySumMs), latencySamples: JSON.stringify(metrics.latencySamples), processJudgeSum: String(metrics.processJudgeSum), processJudgeCount: String(metrics.processJudgeCount), updatedAt: metrics.updatedAt, }; } function updateMetrics( current: SliceMetrics, observation: EvalResultObservation, nowMs: number, ): SliceMetrics { const latencySamples = [...current.latencySamples]; const latencySampleCap = getEvalAggLatencySampleCap(); if (observation.latencyMs !== null) { latencySamples.push(observation.latencyMs); if (latencySamples.length > latencySampleCap) { latencySamples.splice(0, latencySamples.length - latencySampleCap); } } return { abstentionCount: current.abstentionCount + (observation.abstentionAction && observation.abstentionAction !== "proceed" ? 1 : 0), count: current.count + 1, llmOverallSum: current.llmOverallSum + (observation.llmOverall ?? 0), llmOverallCount: current.llmOverallCount + (observation.llmOverall === null ? 0 : 1), validatorRejectCount: current.validatorRejectCount + (observation.validatorStatus === "reject" ? 1 : 0), validatorSoftWarnCount: current.validatorSoftWarnCount + (observation.validatorStatus === "soft-warn" ? 1 : 0), latencySumMs: current.latencySumMs + (observation.latencyMs ?? 0), latencySamples, processJudgeSum: current.processJudgeSum + (observation.processJudgeScore ?? 0), processJudgeCount: current.processJudgeCount + (observation.processJudgeScore === null ? 0 : 1), updatedAt: new Date(nowMs).toISOString(), }; } function streamRangeStart(sinceId: string): string { const trimmed = sinceId.trim(); if (!trimmed || trimmed === "0-0" || trimmed === "-") { return "-"; } return `(${trimmed}`; } function round(value: number): number { return Number(value.toFixed(3)); } function calculatePercentile(values: number[], percentile: number): number | null { if (values.length === 0) { return null; } const sorted = [...values].sort((left, right) => left - right); const index = Math.min( sorted.length - 1, Math.max(0, Math.ceil(sorted.length * percentile) - 1), ); const selected = sorted[index]; return selected === undefined ? null : round(selected); } function average(sum: number, count: number): number | null { return count > 0 ? round(sum / count) : null; } function sliceKeyFromString(value: string): SliceKey | null { const parts = value.split(":"); const [tierRaw, intent, phase] = parts; const tier = parseTier(tierRaw); if (tier === null || !intent || !phase) { return null; } const routingDecision = parts.length >= 6 ? parts[3] : undefined; const abstentionAction = parts.length >= 6 ? parts[4] : parts[3]; const abstentionReason = parts.length >= 6 ? parts[5] : parts[4]; return { tier, intent, phase, ...(routingDecision && routingDecision !== "-" ? { routingDecision } : {}), ...(abstentionAction && abstentionAction !== "-" ? { abstentionAction } : {}), ...(abstentionReason && abstentionReason !== "-" ? { abstentionReason } : {}), }; } function sliceKeyFromRedisKey(key: string): SliceKey | null { if (!key.startsWith(EVAL_AGG_SLICE_KEY_PREFIX)) { return null; } return sliceKeyFromString(key.slice(EVAL_AGG_SLICE_KEY_PREFIX.length)); } export function sliceKeyToString(key: SliceKey): string { const base = `${key.tier}:${key.intent}:${key.phase}`; if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) { return base; } if (key.routingDecision) { return [ base, key.routingDecision, key.abstentionAction ?? "-", key.abstentionReason ?? "-", ].join(":"); } return [ base, key.abstentionAction ?? "-", key.abstentionReason ?? "-", ].join(":"); } export function getSliceAggregateKey(key: SliceKey): string { return `${EVAL_AGG_SLICE_KEY_PREFIX}${sliceKeyToString(key)}`; } export function sliceMetricsToReportRow( slice: SliceKey, metrics: SliceMetrics, ): SliceReportRow { return { slice, abstentionRate: metrics.count === 0 ? 0 : round(metrics.abstentionCount / metrics.count), count: metrics.count, llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount), processJudgeAvg: average( metrics.processJudgeSum, metrics.processJudgeCount, ), validatorRejectRate: metrics.count === 0 ? 0 : round(metrics.validatorRejectCount / metrics.count), validatorSoftWarnRate: metrics.count === 0 ? 0 : round(metrics.validatorSoftWarnCount / metrics.count), p50LatencyMs: calculatePercentile(metrics.latencySamples, 0.5), p95LatencyMs: calculatePercentile(metrics.latencySamples, 0.95), updatedAt: metrics.updatedAt, }; } export async function aggregateRecentEvalResults( garnetClient: AggregatorGarnetClient, options: { sinceId: string; nowMs: number }, ): Promise<{ lastSeenId: string; updatedSlices: number }> { const entries = await garnetClient.readStreamRange( EVAL_RESULTS_STREAM_KEY, streamRangeStart(options.sinceId), "+", ); if (entries.length === 0) { return { lastSeenId: options.sinceId, updatedSlices: 0, }; } const updatedKeys = new Set(); let lastSeenId = options.sinceId; for (const entry of entries) { lastSeenId = entry.id; const observation = parseObservation(entry); if (!observation) { continue; } const key = getSliceAggregateKey(observation.slice); const current = metricsFromHash(await garnetClient.getHash(key)); const next = updateMetrics(current, observation, options.nowMs); await garnetClient.setHash(key, metricsToHash(next)); await garnetClient.expireKey(key, getEvalAggTtlSec()); updatedKeys.add(key); } await garnetClient.setString(EVAL_AGG_CURSOR_KEY, lastSeenId); return { lastSeenId, updatedSlices: updatedKeys.size, }; } export async function readAggregatedSliceReportRows( garnetClient: AggregatorGarnetClient, ): Promise { const keys = await garnetClient.scanKeys(`${EVAL_AGG_SLICE_KEY_PREFIX}*`); const rows: SliceReportRow[] = []; for (const key of keys) { const slice = sliceKeyFromRedisKey(key); if (!slice) { continue; } const metrics = metricsFromHash(await garnetClient.getHash(key)); rows.push(sliceMetricsToReportRow(slice, metrics)); } return rows; }