493 lines
14 KiB
TypeScript
493 lines
14 KiB
TypeScript
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
|
import {
|
|
getEvalAggLatencySampleCap,
|
|
getEvalAggTtlSec,
|
|
} from "../config.js";
|
|
|
|
export const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
|
export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor";
|
|
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
|
|
|
|
export interface SliceKey {
|
|
tier: 0 | 1 | 2;
|
|
intent: string;
|
|
phase: string;
|
|
routingDecision?: string | undefined;
|
|
abstentionAction?: string | undefined;
|
|
abstentionReason?: string | undefined;
|
|
}
|
|
|
|
export interface SliceMetrics {
|
|
abstentionCount: number;
|
|
count: number;
|
|
llmOverallSum: number;
|
|
llmOverallCount: number;
|
|
validatorRejectCount: number;
|
|
validatorSoftWarnCount: number;
|
|
latencySumMs: number;
|
|
latencySamples: number[];
|
|
processJudgeSum: number;
|
|
processJudgeCount: number;
|
|
updatedAt: string;
|
|
}
|
|
|
|
export interface SliceReportRow {
|
|
slice: SliceKey;
|
|
abstentionRate: number;
|
|
count: number;
|
|
llmOverallAvg: number | null;
|
|
processJudgeAvg: number | null;
|
|
validatorRejectRate: number;
|
|
validatorSoftWarnRate: number;
|
|
p50LatencyMs: number | null;
|
|
p95LatencyMs: number | null;
|
|
updatedAt: string;
|
|
}
|
|
|
|
type AggregatorGarnetClient = Pick<
|
|
GarnetClient,
|
|
| "expireKey"
|
|
| "getHash"
|
|
| "readStreamRange"
|
|
| "scanKeys"
|
|
| "setHash"
|
|
| "setString"
|
|
>;
|
|
|
|
interface EvalResultObservation {
|
|
abstentionAction: string | null;
|
|
latencyMs: number | null;
|
|
llmOverall: number | null;
|
|
processJudgeScore: number | null;
|
|
slice: SliceKey;
|
|
validatorStatus: string | null;
|
|
}
|
|
|
|
function asRecord(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function parseJsonRecord(value: string | undefined): Record<string, unknown> {
|
|
if (!value?.trim()) {
|
|
return {};
|
|
}
|
|
|
|
try {
|
|
return asRecord(JSON.parse(value));
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
function parseFiniteNumber(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) {
|
|
return value;
|
|
}
|
|
if (typeof value !== "string" || !value.trim()) {
|
|
return null;
|
|
}
|
|
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
|
const parsed = parseFiniteNumber(value);
|
|
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
|
return parsed;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function getNestedString(
|
|
record: Record<string, unknown>,
|
|
first: string,
|
|
second: string,
|
|
): string | null {
|
|
const nested = asRecord(record[first]);
|
|
const value = nested[second];
|
|
return typeof value === "string" && value.trim() ? value : null;
|
|
}
|
|
|
|
function extractLlmOverall(message: Record<string, string>): number | null {
|
|
const direct = parseFiniteNumber(message["llmOverall"]);
|
|
if (direct !== null) {
|
|
return direct;
|
|
}
|
|
|
|
const llmJudge = parseJsonRecord(message["llmJudge"]);
|
|
const scores = asRecord(llmJudge["scores"]);
|
|
return parseFiniteNumber(scores["overall"]);
|
|
}
|
|
|
|
function extractProcessJudgeScore(
|
|
message: Record<string, string>,
|
|
): number | null {
|
|
const processJudge = parseJsonRecord(message["processJudge"]);
|
|
return parseFiniteNumber(processJudge["score"]);
|
|
}
|
|
|
|
function extractValidatorStatus(message: Record<string, string>): string | null {
|
|
if (message["validatorStatus"]?.trim()) {
|
|
return message["validatorStatus"] ?? null;
|
|
}
|
|
|
|
const validation = parseJsonRecord(message["validation"]);
|
|
const status = validation["status"];
|
|
return typeof status === "string" && status.trim() ? status : null;
|
|
}
|
|
|
|
function extractAbstention(
|
|
message: Record<string, string>,
|
|
): { action: string | null; reason: string | null } {
|
|
const directAction = message["abstentionAction"]?.trim();
|
|
const directReason = message["abstentionReason"]?.trim();
|
|
if (directAction) {
|
|
return {
|
|
action: directAction,
|
|
reason: directReason || null,
|
|
};
|
|
}
|
|
|
|
const process = parseJsonRecord(message["process"]);
|
|
const steps = asRecord(process["steps"]);
|
|
const classify = asRecord(steps["classify"]);
|
|
const output = asRecord(classify["output"]);
|
|
const abstention = asRecord(output["abstention"]);
|
|
const action = asRecord(abstention["action"]);
|
|
const kind = action["kind"];
|
|
const reason = abstention["reason"];
|
|
|
|
return {
|
|
action: typeof kind === "string" && kind.trim() ? kind : null,
|
|
reason: typeof reason === "string" && reason.trim() ? reason : null,
|
|
};
|
|
}
|
|
|
|
function extractRoutingDecision(derived: Record<string, unknown>): string | null {
|
|
const routingDecision = asRecord(derived["routingDecision"]);
|
|
const kind = routingDecision["kind"];
|
|
return typeof kind === "string" && kind.trim() ? kind : null;
|
|
}
|
|
|
|
function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
|
const derived = parseJsonRecord(entry.message["derived"]);
|
|
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
|
|
const intent =
|
|
entry.message["intent"]?.trim() ||
|
|
getNestedString(derived, "intent", "kind");
|
|
const phase =
|
|
entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind");
|
|
|
|
if (tier === null || !intent || !phase) {
|
|
return null;
|
|
}
|
|
|
|
const abstention = extractAbstention(entry.message);
|
|
const routingDecision = extractRoutingDecision(derived);
|
|
|
|
return {
|
|
abstentionAction: abstention.action,
|
|
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
|
|
llmOverall: extractLlmOverall(entry.message),
|
|
processJudgeScore: extractProcessJudgeScore(entry.message),
|
|
slice: {
|
|
tier,
|
|
intent,
|
|
phase,
|
|
...(routingDecision ? { routingDecision } : {}),
|
|
...(abstention.action ? { abstentionAction: abstention.action } : {}),
|
|
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
|
|
},
|
|
validatorStatus: extractValidatorStatus(entry.message),
|
|
};
|
|
}
|
|
|
|
function parseNumberField(
|
|
hash: Record<string, string>,
|
|
field: keyof SliceMetrics,
|
|
): number {
|
|
const value = parseFiniteNumber(hash[field]);
|
|
return value ?? 0;
|
|
}
|
|
|
|
function parseLatencySamples(hash: Record<string, string>): number[] {
|
|
const raw = hash["latencySamples"];
|
|
if (!raw?.trim()) {
|
|
return [];
|
|
}
|
|
|
|
try {
|
|
const parsed = JSON.parse(raw) as unknown;
|
|
if (!Array.isArray(parsed)) {
|
|
return [];
|
|
}
|
|
return parsed.flatMap((item) => {
|
|
const value = parseFiniteNumber(item);
|
|
return value === null ? [] : [value];
|
|
});
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function metricsFromHash(hash: Record<string, string>): SliceMetrics {
|
|
return {
|
|
abstentionCount: parseNumberField(hash, "abstentionCount"),
|
|
count: parseNumberField(hash, "count"),
|
|
llmOverallSum: parseNumberField(hash, "llmOverallSum"),
|
|
llmOverallCount: parseNumberField(hash, "llmOverallCount"),
|
|
validatorRejectCount: parseNumberField(hash, "validatorRejectCount"),
|
|
validatorSoftWarnCount: parseNumberField(hash, "validatorSoftWarnCount"),
|
|
latencySumMs: parseNumberField(hash, "latencySumMs"),
|
|
latencySamples: parseLatencySamples(hash),
|
|
processJudgeSum: parseNumberField(hash, "processJudgeSum"),
|
|
processJudgeCount: parseNumberField(hash, "processJudgeCount"),
|
|
updatedAt: hash["updatedAt"] ?? new Date(0).toISOString(),
|
|
};
|
|
}
|
|
|
|
function metricsToHash(metrics: SliceMetrics): Record<string, string> {
|
|
return {
|
|
abstentionCount: String(metrics.abstentionCount),
|
|
count: String(metrics.count),
|
|
llmOverallSum: String(metrics.llmOverallSum),
|
|
llmOverallCount: String(metrics.llmOverallCount),
|
|
validatorRejectCount: String(metrics.validatorRejectCount),
|
|
validatorSoftWarnCount: String(metrics.validatorSoftWarnCount),
|
|
latencySumMs: String(metrics.latencySumMs),
|
|
latencySamples: JSON.stringify(metrics.latencySamples),
|
|
processJudgeSum: String(metrics.processJudgeSum),
|
|
processJudgeCount: String(metrics.processJudgeCount),
|
|
updatedAt: metrics.updatedAt,
|
|
};
|
|
}
|
|
|
|
function updateMetrics(
|
|
current: SliceMetrics,
|
|
observation: EvalResultObservation,
|
|
nowMs: number,
|
|
): SliceMetrics {
|
|
const latencySamples = [...current.latencySamples];
|
|
const latencySampleCap = getEvalAggLatencySampleCap();
|
|
|
|
if (observation.latencyMs !== null) {
|
|
latencySamples.push(observation.latencyMs);
|
|
if (latencySamples.length > latencySampleCap) {
|
|
latencySamples.splice(0, latencySamples.length - latencySampleCap);
|
|
}
|
|
}
|
|
|
|
return {
|
|
abstentionCount:
|
|
current.abstentionCount +
|
|
(observation.abstentionAction &&
|
|
observation.abstentionAction !== "proceed"
|
|
? 1
|
|
: 0),
|
|
count: current.count + 1,
|
|
llmOverallSum:
|
|
current.llmOverallSum + (observation.llmOverall ?? 0),
|
|
llmOverallCount:
|
|
current.llmOverallCount + (observation.llmOverall === null ? 0 : 1),
|
|
validatorRejectCount:
|
|
current.validatorRejectCount +
|
|
(observation.validatorStatus === "reject" ? 1 : 0),
|
|
validatorSoftWarnCount:
|
|
current.validatorSoftWarnCount +
|
|
(observation.validatorStatus === "soft-warn" ? 1 : 0),
|
|
latencySumMs: current.latencySumMs + (observation.latencyMs ?? 0),
|
|
latencySamples,
|
|
processJudgeSum:
|
|
current.processJudgeSum + (observation.processJudgeScore ?? 0),
|
|
processJudgeCount:
|
|
current.processJudgeCount +
|
|
(observation.processJudgeScore === null ? 0 : 1),
|
|
updatedAt: new Date(nowMs).toISOString(),
|
|
};
|
|
}
|
|
|
|
function streamRangeStart(sinceId: string): string {
|
|
const trimmed = sinceId.trim();
|
|
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
|
return "-";
|
|
}
|
|
return `(${trimmed}`;
|
|
}
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(3));
|
|
}
|
|
|
|
function calculatePercentile(values: number[], percentile: number): number | null {
|
|
if (values.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
const sorted = [...values].sort((left, right) => left - right);
|
|
const index = Math.min(
|
|
sorted.length - 1,
|
|
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
|
);
|
|
const selected = sorted[index];
|
|
return selected === undefined ? null : round(selected);
|
|
}
|
|
|
|
function average(sum: number, count: number): number | null {
|
|
return count > 0 ? round(sum / count) : null;
|
|
}
|
|
|
|
function sliceKeyFromString(value: string): SliceKey | null {
|
|
const parts = value.split(":");
|
|
const [tierRaw, intent, phase] = parts;
|
|
const tier = parseTier(tierRaw);
|
|
if (tier === null || !intent || !phase) {
|
|
return null;
|
|
}
|
|
|
|
const routingDecision = parts.length >= 6 ? parts[3] : undefined;
|
|
const abstentionAction = parts.length >= 6 ? parts[4] : parts[3];
|
|
const abstentionReason = parts.length >= 6 ? parts[5] : parts[4];
|
|
|
|
return {
|
|
tier,
|
|
intent,
|
|
phase,
|
|
...(routingDecision && routingDecision !== "-"
|
|
? { routingDecision }
|
|
: {}),
|
|
...(abstentionAction && abstentionAction !== "-"
|
|
? { abstentionAction }
|
|
: {}),
|
|
...(abstentionReason && abstentionReason !== "-"
|
|
? { abstentionReason }
|
|
: {}),
|
|
};
|
|
}
|
|
|
|
function sliceKeyFromRedisKey(key: string): SliceKey | null {
|
|
if (!key.startsWith(EVAL_AGG_SLICE_KEY_PREFIX)) {
|
|
return null;
|
|
}
|
|
|
|
return sliceKeyFromString(key.slice(EVAL_AGG_SLICE_KEY_PREFIX.length));
|
|
}
|
|
|
|
export function sliceKeyToString(key: SliceKey): string {
|
|
const base = `${key.tier}:${key.intent}:${key.phase}`;
|
|
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
|
|
return base;
|
|
}
|
|
|
|
if (key.routingDecision) {
|
|
return [
|
|
base,
|
|
key.routingDecision,
|
|
key.abstentionAction ?? "-",
|
|
key.abstentionReason ?? "-",
|
|
].join(":");
|
|
}
|
|
|
|
return [
|
|
base,
|
|
key.abstentionAction ?? "-",
|
|
key.abstentionReason ?? "-",
|
|
].join(":");
|
|
}
|
|
|
|
export function getSliceAggregateKey(key: SliceKey): string {
|
|
return `${EVAL_AGG_SLICE_KEY_PREFIX}${sliceKeyToString(key)}`;
|
|
}
|
|
|
|
export function sliceMetricsToReportRow(
|
|
slice: SliceKey,
|
|
metrics: SliceMetrics,
|
|
): SliceReportRow {
|
|
return {
|
|
slice,
|
|
abstentionRate:
|
|
metrics.count === 0 ? 0 : round(metrics.abstentionCount / metrics.count),
|
|
count: metrics.count,
|
|
llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount),
|
|
processJudgeAvg: average(
|
|
metrics.processJudgeSum,
|
|
metrics.processJudgeCount,
|
|
),
|
|
validatorRejectRate:
|
|
metrics.count === 0
|
|
? 0
|
|
: round(metrics.validatorRejectCount / metrics.count),
|
|
validatorSoftWarnRate:
|
|
metrics.count === 0
|
|
? 0
|
|
: round(metrics.validatorSoftWarnCount / metrics.count),
|
|
p50LatencyMs: calculatePercentile(metrics.latencySamples, 0.5),
|
|
p95LatencyMs: calculatePercentile(metrics.latencySamples, 0.95),
|
|
updatedAt: metrics.updatedAt,
|
|
};
|
|
}
|
|
|
|
export async function aggregateRecentEvalResults(
|
|
garnetClient: AggregatorGarnetClient,
|
|
options: { sinceId: string; nowMs: number },
|
|
): Promise<{ lastSeenId: string; updatedSlices: number }> {
|
|
const entries = await garnetClient.readStreamRange(
|
|
EVAL_RESULTS_STREAM_KEY,
|
|
streamRangeStart(options.sinceId),
|
|
"+",
|
|
);
|
|
|
|
if (entries.length === 0) {
|
|
return {
|
|
lastSeenId: options.sinceId,
|
|
updatedSlices: 0,
|
|
};
|
|
}
|
|
|
|
const updatedKeys = new Set<string>();
|
|
let lastSeenId = options.sinceId;
|
|
|
|
for (const entry of entries) {
|
|
lastSeenId = entry.id;
|
|
const observation = parseObservation(entry);
|
|
if (!observation) {
|
|
continue;
|
|
}
|
|
|
|
const key = getSliceAggregateKey(observation.slice);
|
|
const current = metricsFromHash(await garnetClient.getHash(key));
|
|
const next = updateMetrics(current, observation, options.nowMs);
|
|
await garnetClient.setHash(key, metricsToHash(next));
|
|
await garnetClient.expireKey(key, getEvalAggTtlSec());
|
|
updatedKeys.add(key);
|
|
}
|
|
|
|
await garnetClient.setString(EVAL_AGG_CURSOR_KEY, lastSeenId);
|
|
|
|
return {
|
|
lastSeenId,
|
|
updatedSlices: updatedKeys.size,
|
|
};
|
|
}
|
|
|
|
export async function readAggregatedSliceReportRows(
|
|
garnetClient: AggregatorGarnetClient,
|
|
): Promise<SliceReportRow[]> {
|
|
const keys = await garnetClient.scanKeys(`${EVAL_AGG_SLICE_KEY_PREFIX}*`);
|
|
const rows: SliceReportRow[] = [];
|
|
|
|
for (const key of keys) {
|
|
const slice = sliceKeyFromRedisKey(key);
|
|
if (!slice) {
|
|
continue;
|
|
}
|
|
|
|
const metrics = metricsFromHash(await garnetClient.getHash(key));
|
|
rows.push(sliceMetricsToReportRow(slice, metrics));
|
|
}
|
|
|
|
return rows;
|
|
}
|