Files
lolympus-agent/src/eval/aggregator.ts
T

493 lines
14 KiB
TypeScript

import type { GarnetClient, StreamEntry } from "@common/garnet";
import {
getEvalAggLatencySampleCap,
getEvalAggTtlSec,
} from "../config.js";
export const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor";
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
export interface SliceKey {
tier: 0 | 1 | 2;
intent: string;
phase: string;
routingDecision?: string | undefined;
abstentionAction?: string | undefined;
abstentionReason?: string | undefined;
}
export interface SliceMetrics {
abstentionCount: number;
count: number;
llmOverallSum: number;
llmOverallCount: number;
validatorRejectCount: number;
validatorSoftWarnCount: number;
latencySumMs: number;
latencySamples: number[];
processJudgeSum: number;
processJudgeCount: number;
updatedAt: string;
}
export interface SliceReportRow {
slice: SliceKey;
abstentionRate: number;
count: number;
llmOverallAvg: number | null;
processJudgeAvg: number | null;
validatorRejectRate: number;
validatorSoftWarnRate: number;
p50LatencyMs: number | null;
p95LatencyMs: number | null;
updatedAt: string;
}
type AggregatorGarnetClient = Pick<
GarnetClient,
| "expireKey"
| "getHash"
| "readStreamRange"
| "scanKeys"
| "setHash"
| "setString"
>;
interface EvalResultObservation {
abstentionAction: string | null;
latencyMs: number | null;
llmOverall: number | null;
processJudgeScore: number | null;
slice: SliceKey;
validatorStatus: string | null;
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function parseJsonRecord(value: string | undefined): Record<string, unknown> {
if (!value?.trim()) {
return {};
}
try {
return asRecord(JSON.parse(value));
} catch {
return {};
}
}
function parseFiniteNumber(value: unknown): number | null {
if (typeof value === "number" && Number.isFinite(value)) {
return value;
}
if (typeof value !== "string" || !value.trim()) {
return null;
}
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function parseTier(value: unknown): 0 | 1 | 2 | null {
const parsed = parseFiniteNumber(value);
if (parsed === 0 || parsed === 1 || parsed === 2) {
return parsed;
}
return null;
}
function getNestedString(
record: Record<string, unknown>,
first: string,
second: string,
): string | null {
const nested = asRecord(record[first]);
const value = nested[second];
return typeof value === "string" && value.trim() ? value : null;
}
function extractLlmOverall(message: Record<string, string>): number | null {
const direct = parseFiniteNumber(message["llmOverall"]);
if (direct !== null) {
return direct;
}
const llmJudge = parseJsonRecord(message["llmJudge"]);
const scores = asRecord(llmJudge["scores"]);
return parseFiniteNumber(scores["overall"]);
}
function extractProcessJudgeScore(
message: Record<string, string>,
): number | null {
const processJudge = parseJsonRecord(message["processJudge"]);
return parseFiniteNumber(processJudge["score"]);
}
function extractValidatorStatus(message: Record<string, string>): string | null {
if (message["validatorStatus"]?.trim()) {
return message["validatorStatus"] ?? null;
}
const validation = parseJsonRecord(message["validation"]);
const status = validation["status"];
return typeof status === "string" && status.trim() ? status : null;
}
function extractAbstention(
message: Record<string, string>,
): { action: string | null; reason: string | null } {
const directAction = message["abstentionAction"]?.trim();
const directReason = message["abstentionReason"]?.trim();
if (directAction) {
return {
action: directAction,
reason: directReason || null,
};
}
const process = parseJsonRecord(message["process"]);
const steps = asRecord(process["steps"]);
const classify = asRecord(steps["classify"]);
const output = asRecord(classify["output"]);
const abstention = asRecord(output["abstention"]);
const action = asRecord(abstention["action"]);
const kind = action["kind"];
const reason = abstention["reason"];
return {
action: typeof kind === "string" && kind.trim() ? kind : null,
reason: typeof reason === "string" && reason.trim() ? reason : null,
};
}
function extractRoutingDecision(derived: Record<string, unknown>): string | null {
const routingDecision = asRecord(derived["routingDecision"]);
const kind = routingDecision["kind"];
return typeof kind === "string" && kind.trim() ? kind : null;
}
function parseObservation(entry: StreamEntry): EvalResultObservation | null {
const derived = parseJsonRecord(entry.message["derived"]);
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
const intent =
entry.message["intent"]?.trim() ||
getNestedString(derived, "intent", "kind");
const phase =
entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind");
if (tier === null || !intent || !phase) {
return null;
}
const abstention = extractAbstention(entry.message);
const routingDecision = extractRoutingDecision(derived);
return {
abstentionAction: abstention.action,
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
llmOverall: extractLlmOverall(entry.message),
processJudgeScore: extractProcessJudgeScore(entry.message),
slice: {
tier,
intent,
phase,
...(routingDecision ? { routingDecision } : {}),
...(abstention.action ? { abstentionAction: abstention.action } : {}),
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
},
validatorStatus: extractValidatorStatus(entry.message),
};
}
function parseNumberField(
hash: Record<string, string>,
field: keyof SliceMetrics,
): number {
const value = parseFiniteNumber(hash[field]);
return value ?? 0;
}
function parseLatencySamples(hash: Record<string, string>): number[] {
const raw = hash["latencySamples"];
if (!raw?.trim()) {
return [];
}
try {
const parsed = JSON.parse(raw) as unknown;
if (!Array.isArray(parsed)) {
return [];
}
return parsed.flatMap((item) => {
const value = parseFiniteNumber(item);
return value === null ? [] : [value];
});
} catch {
return [];
}
}
function metricsFromHash(hash: Record<string, string>): SliceMetrics {
return {
abstentionCount: parseNumberField(hash, "abstentionCount"),
count: parseNumberField(hash, "count"),
llmOverallSum: parseNumberField(hash, "llmOverallSum"),
llmOverallCount: parseNumberField(hash, "llmOverallCount"),
validatorRejectCount: parseNumberField(hash, "validatorRejectCount"),
validatorSoftWarnCount: parseNumberField(hash, "validatorSoftWarnCount"),
latencySumMs: parseNumberField(hash, "latencySumMs"),
latencySamples: parseLatencySamples(hash),
processJudgeSum: parseNumberField(hash, "processJudgeSum"),
processJudgeCount: parseNumberField(hash, "processJudgeCount"),
updatedAt: hash["updatedAt"] ?? new Date(0).toISOString(),
};
}
function metricsToHash(metrics: SliceMetrics): Record<string, string> {
return {
abstentionCount: String(metrics.abstentionCount),
count: String(metrics.count),
llmOverallSum: String(metrics.llmOverallSum),
llmOverallCount: String(metrics.llmOverallCount),
validatorRejectCount: String(metrics.validatorRejectCount),
validatorSoftWarnCount: String(metrics.validatorSoftWarnCount),
latencySumMs: String(metrics.latencySumMs),
latencySamples: JSON.stringify(metrics.latencySamples),
processJudgeSum: String(metrics.processJudgeSum),
processJudgeCount: String(metrics.processJudgeCount),
updatedAt: metrics.updatedAt,
};
}
function updateMetrics(
current: SliceMetrics,
observation: EvalResultObservation,
nowMs: number,
): SliceMetrics {
const latencySamples = [...current.latencySamples];
const latencySampleCap = getEvalAggLatencySampleCap();
if (observation.latencyMs !== null) {
latencySamples.push(observation.latencyMs);
if (latencySamples.length > latencySampleCap) {
latencySamples.splice(0, latencySamples.length - latencySampleCap);
}
}
return {
abstentionCount:
current.abstentionCount +
(observation.abstentionAction &&
observation.abstentionAction !== "proceed"
? 1
: 0),
count: current.count + 1,
llmOverallSum:
current.llmOverallSum + (observation.llmOverall ?? 0),
llmOverallCount:
current.llmOverallCount + (observation.llmOverall === null ? 0 : 1),
validatorRejectCount:
current.validatorRejectCount +
(observation.validatorStatus === "reject" ? 1 : 0),
validatorSoftWarnCount:
current.validatorSoftWarnCount +
(observation.validatorStatus === "soft-warn" ? 1 : 0),
latencySumMs: current.latencySumMs + (observation.latencyMs ?? 0),
latencySamples,
processJudgeSum:
current.processJudgeSum + (observation.processJudgeScore ?? 0),
processJudgeCount:
current.processJudgeCount +
(observation.processJudgeScore === null ? 0 : 1),
updatedAt: new Date(nowMs).toISOString(),
};
}
function streamRangeStart(sinceId: string): string {
const trimmed = sinceId.trim();
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
return "-";
}
return `(${trimmed}`;
}
function round(value: number): number {
return Number(value.toFixed(3));
}
function calculatePercentile(values: number[], percentile: number): number | null {
if (values.length === 0) {
return null;
}
const sorted = [...values].sort((left, right) => left - right);
const index = Math.min(
sorted.length - 1,
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
);
const selected = sorted[index];
return selected === undefined ? null : round(selected);
}
function average(sum: number, count: number): number | null {
return count > 0 ? round(sum / count) : null;
}
function sliceKeyFromString(value: string): SliceKey | null {
const parts = value.split(":");
const [tierRaw, intent, phase] = parts;
const tier = parseTier(tierRaw);
if (tier === null || !intent || !phase) {
return null;
}
const routingDecision = parts.length >= 6 ? parts[3] : undefined;
const abstentionAction = parts.length >= 6 ? parts[4] : parts[3];
const abstentionReason = parts.length >= 6 ? parts[5] : parts[4];
return {
tier,
intent,
phase,
...(routingDecision && routingDecision !== "-"
? { routingDecision }
: {}),
...(abstentionAction && abstentionAction !== "-"
? { abstentionAction }
: {}),
...(abstentionReason && abstentionReason !== "-"
? { abstentionReason }
: {}),
};
}
function sliceKeyFromRedisKey(key: string): SliceKey | null {
if (!key.startsWith(EVAL_AGG_SLICE_KEY_PREFIX)) {
return null;
}
return sliceKeyFromString(key.slice(EVAL_AGG_SLICE_KEY_PREFIX.length));
}
export function sliceKeyToString(key: SliceKey): string {
const base = `${key.tier}:${key.intent}:${key.phase}`;
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
return base;
}
if (key.routingDecision) {
return [
base,
key.routingDecision,
key.abstentionAction ?? "-",
key.abstentionReason ?? "-",
].join(":");
}
return [
base,
key.abstentionAction ?? "-",
key.abstentionReason ?? "-",
].join(":");
}
export function getSliceAggregateKey(key: SliceKey): string {
return `${EVAL_AGG_SLICE_KEY_PREFIX}${sliceKeyToString(key)}`;
}
export function sliceMetricsToReportRow(
slice: SliceKey,
metrics: SliceMetrics,
): SliceReportRow {
return {
slice,
abstentionRate:
metrics.count === 0 ? 0 : round(metrics.abstentionCount / metrics.count),
count: metrics.count,
llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount),
processJudgeAvg: average(
metrics.processJudgeSum,
metrics.processJudgeCount,
),
validatorRejectRate:
metrics.count === 0
? 0
: round(metrics.validatorRejectCount / metrics.count),
validatorSoftWarnRate:
metrics.count === 0
? 0
: round(metrics.validatorSoftWarnCount / metrics.count),
p50LatencyMs: calculatePercentile(metrics.latencySamples, 0.5),
p95LatencyMs: calculatePercentile(metrics.latencySamples, 0.95),
updatedAt: metrics.updatedAt,
};
}
export async function aggregateRecentEvalResults(
garnetClient: AggregatorGarnetClient,
options: { sinceId: string; nowMs: number },
): Promise<{ lastSeenId: string; updatedSlices: number }> {
const entries = await garnetClient.readStreamRange(
EVAL_RESULTS_STREAM_KEY,
streamRangeStart(options.sinceId),
"+",
);
if (entries.length === 0) {
return {
lastSeenId: options.sinceId,
updatedSlices: 0,
};
}
const updatedKeys = new Set<string>();
let lastSeenId = options.sinceId;
for (const entry of entries) {
lastSeenId = entry.id;
const observation = parseObservation(entry);
if (!observation) {
continue;
}
const key = getSliceAggregateKey(observation.slice);
const current = metricsFromHash(await garnetClient.getHash(key));
const next = updateMetrics(current, observation, options.nowMs);
await garnetClient.setHash(key, metricsToHash(next));
await garnetClient.expireKey(key, getEvalAggTtlSec());
updatedKeys.add(key);
}
await garnetClient.setString(EVAL_AGG_CURSOR_KEY, lastSeenId);
return {
lastSeenId,
updatedSlices: updatedKeys.size,
};
}
export async function readAggregatedSliceReportRows(
garnetClient: AggregatorGarnetClient,
): Promise<SliceReportRow[]> {
const keys = await garnetClient.scanKeys(`${EVAL_AGG_SLICE_KEY_PREFIX}*`);
const rows: SliceReportRow[] = [];
for (const key of keys) {
const slice = sliceKeyFromRedisKey(key);
if (!slice) {
continue;
}
const metrics = metricsFromHash(await garnetClient.getHash(key));
rows.push(sliceMetricsToReportRow(slice, metrics));
}
return rows;
}