Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -0,0 +1,396 @@
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getEvalAggLatencySampleCap,
|
||||
getEvalAggTtlSec,
|
||||
} from "../config.js";
|
||||
|
||||
export const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
||||
export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor";
|
||||
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
|
||||
|
||||
export interface SliceKey {
|
||||
tier: 0 | 1 | 2;
|
||||
intent: string;
|
||||
phase: string;
|
||||
}
|
||||
|
||||
export interface SliceMetrics {
|
||||
count: number;
|
||||
llmOverallSum: number;
|
||||
llmOverallCount: number;
|
||||
validatorRejectCount: number;
|
||||
validatorSoftWarnCount: number;
|
||||
latencySumMs: number;
|
||||
latencySamples: number[];
|
||||
processJudgeSum: number;
|
||||
processJudgeCount: number;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface SliceReportRow {
|
||||
slice: SliceKey;
|
||||
count: number;
|
||||
llmOverallAvg: number | null;
|
||||
processJudgeAvg: number | null;
|
||||
validatorRejectRate: number;
|
||||
validatorSoftWarnRate: number;
|
||||
p50LatencyMs: number | null;
|
||||
p95LatencyMs: number | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
type AggregatorGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
| "expireKey"
|
||||
| "getHash"
|
||||
| "readStreamRange"
|
||||
| "scanKeys"
|
||||
| "setHash"
|
||||
| "setString"
|
||||
>;
|
||||
|
||||
interface EvalResultObservation {
|
||||
latencyMs: number | null;
|
||||
llmOverall: number | null;
|
||||
processJudgeScore: number | null;
|
||||
slice: SliceKey;
|
||||
validatorStatus: string | null;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function parseJsonRecord(value: string | undefined): Record<string, unknown> {
|
||||
if (!value?.trim()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
return asRecord(JSON.parse(value));
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function parseFiniteNumber(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
||||
const parsed = parseFiniteNumber(value);
|
||||
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
||||
return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getNestedString(
|
||||
record: Record<string, unknown>,
|
||||
first: string,
|
||||
second: string,
|
||||
): string | null {
|
||||
const nested = asRecord(record[first]);
|
||||
const value = nested[second];
|
||||
return typeof value === "string" && value.trim() ? value : null;
|
||||
}
|
||||
|
||||
function extractLlmOverall(message: Record<string, string>): number | null {
|
||||
const direct = parseFiniteNumber(message["llmOverall"]);
|
||||
if (direct !== null) {
|
||||
return direct;
|
||||
}
|
||||
|
||||
const llmJudge = parseJsonRecord(message["llmJudge"]);
|
||||
const scores = asRecord(llmJudge["scores"]);
|
||||
return parseFiniteNumber(scores["overall"]);
|
||||
}
|
||||
|
||||
function extractProcessJudgeScore(
|
||||
message: Record<string, string>,
|
||||
): number | null {
|
||||
const processJudge = parseJsonRecord(message["processJudge"]);
|
||||
return parseFiniteNumber(processJudge["score"]);
|
||||
}
|
||||
|
||||
function extractValidatorStatus(message: Record<string, string>): string | null {
|
||||
if (message["validatorStatus"]?.trim()) {
|
||||
return message["validatorStatus"] ?? null;
|
||||
}
|
||||
|
||||
const validation = parseJsonRecord(message["validation"]);
|
||||
const status = validation["status"];
|
||||
return typeof status === "string" && status.trim() ? status : null;
|
||||
}
|
||||
|
||||
function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
||||
const derived = parseJsonRecord(entry.message["derived"]);
|
||||
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
|
||||
const intent =
|
||||
entry.message["intent"]?.trim() ||
|
||||
getNestedString(derived, "intent", "kind");
|
||||
const phase =
|
||||
entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind");
|
||||
|
||||
if (tier === null || !intent || !phase) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
|
||||
llmOverall: extractLlmOverall(entry.message),
|
||||
processJudgeScore: extractProcessJudgeScore(entry.message),
|
||||
slice: { tier, intent, phase },
|
||||
validatorStatus: extractValidatorStatus(entry.message),
|
||||
};
|
||||
}
|
||||
|
||||
function parseNumberField(
|
||||
hash: Record<string, string>,
|
||||
field: keyof SliceMetrics,
|
||||
): number {
|
||||
const value = parseFiniteNumber(hash[field]);
|
||||
return value ?? 0;
|
||||
}
|
||||
|
||||
function parseLatencySamples(hash: Record<string, string>): number[] {
|
||||
const raw = hash["latencySamples"];
|
||||
if (!raw?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!Array.isArray(parsed)) {
|
||||
return [];
|
||||
}
|
||||
return parsed.flatMap((item) => {
|
||||
const value = parseFiniteNumber(item);
|
||||
return value === null ? [] : [value];
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function metricsFromHash(hash: Record<string, string>): SliceMetrics {
|
||||
return {
|
||||
count: parseNumberField(hash, "count"),
|
||||
llmOverallSum: parseNumberField(hash, "llmOverallSum"),
|
||||
llmOverallCount: parseNumberField(hash, "llmOverallCount"),
|
||||
validatorRejectCount: parseNumberField(hash, "validatorRejectCount"),
|
||||
validatorSoftWarnCount: parseNumberField(hash, "validatorSoftWarnCount"),
|
||||
latencySumMs: parseNumberField(hash, "latencySumMs"),
|
||||
latencySamples: parseLatencySamples(hash),
|
||||
processJudgeSum: parseNumberField(hash, "processJudgeSum"),
|
||||
processJudgeCount: parseNumberField(hash, "processJudgeCount"),
|
||||
updatedAt: hash["updatedAt"] ?? new Date(0).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function metricsToHash(metrics: SliceMetrics): Record<string, string> {
|
||||
return {
|
||||
count: String(metrics.count),
|
||||
llmOverallSum: String(metrics.llmOverallSum),
|
||||
llmOverallCount: String(metrics.llmOverallCount),
|
||||
validatorRejectCount: String(metrics.validatorRejectCount),
|
||||
validatorSoftWarnCount: String(metrics.validatorSoftWarnCount),
|
||||
latencySumMs: String(metrics.latencySumMs),
|
||||
latencySamples: JSON.stringify(metrics.latencySamples),
|
||||
processJudgeSum: String(metrics.processJudgeSum),
|
||||
processJudgeCount: String(metrics.processJudgeCount),
|
||||
updatedAt: metrics.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function updateMetrics(
|
||||
current: SliceMetrics,
|
||||
observation: EvalResultObservation,
|
||||
nowMs: number,
|
||||
): SliceMetrics {
|
||||
const latencySamples = [...current.latencySamples];
|
||||
const latencySampleCap = getEvalAggLatencySampleCap();
|
||||
|
||||
if (observation.latencyMs !== null) {
|
||||
latencySamples.push(observation.latencyMs);
|
||||
if (latencySamples.length > latencySampleCap) {
|
||||
latencySamples.splice(0, latencySamples.length - latencySampleCap);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
count: current.count + 1,
|
||||
llmOverallSum:
|
||||
current.llmOverallSum + (observation.llmOverall ?? 0),
|
||||
llmOverallCount:
|
||||
current.llmOverallCount + (observation.llmOverall === null ? 0 : 1),
|
||||
validatorRejectCount:
|
||||
current.validatorRejectCount +
|
||||
(observation.validatorStatus === "reject" ? 1 : 0),
|
||||
validatorSoftWarnCount:
|
||||
current.validatorSoftWarnCount +
|
||||
(observation.validatorStatus === "soft-warn" ? 1 : 0),
|
||||
latencySumMs: current.latencySumMs + (observation.latencyMs ?? 0),
|
||||
latencySamples,
|
||||
processJudgeSum:
|
||||
current.processJudgeSum + (observation.processJudgeScore ?? 0),
|
||||
processJudgeCount:
|
||||
current.processJudgeCount +
|
||||
(observation.processJudgeScore === null ? 0 : 1),
|
||||
updatedAt: new Date(nowMs).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function streamRangeStart(sinceId: string): string {
|
||||
const trimmed = sinceId.trim();
|
||||
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
||||
return "-";
|
||||
}
|
||||
return `(${trimmed}`;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function calculatePercentile(values: number[], percentile: number): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.min(
|
||||
sorted.length - 1,
|
||||
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
||||
);
|
||||
const selected = sorted[index];
|
||||
return selected === undefined ? null : round(selected);
|
||||
}
|
||||
|
||||
function average(sum: number, count: number): number | null {
|
||||
return count > 0 ? round(sum / count) : null;
|
||||
}
|
||||
|
||||
function sliceKeyFromString(value: string): SliceKey | null {
|
||||
const [tierRaw, intent, phase] = value.split(":");
|
||||
const tier = parseTier(tierRaw);
|
||||
if (tier === null || !intent || !phase) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { tier, intent, phase };
|
||||
}
|
||||
|
||||
function sliceKeyFromRedisKey(key: string): SliceKey | null {
|
||||
if (!key.startsWith(EVAL_AGG_SLICE_KEY_PREFIX)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return sliceKeyFromString(key.slice(EVAL_AGG_SLICE_KEY_PREFIX.length));
|
||||
}
|
||||
|
||||
export function sliceKeyToString(key: SliceKey): string {
|
||||
return `${key.tier}:${key.intent}:${key.phase}`;
|
||||
}
|
||||
|
||||
export function getSliceAggregateKey(key: SliceKey): string {
|
||||
return `${EVAL_AGG_SLICE_KEY_PREFIX}${sliceKeyToString(key)}`;
|
||||
}
|
||||
|
||||
export function sliceMetricsToReportRow(
|
||||
slice: SliceKey,
|
||||
metrics: SliceMetrics,
|
||||
): SliceReportRow {
|
||||
return {
|
||||
slice,
|
||||
count: metrics.count,
|
||||
llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount),
|
||||
processJudgeAvg: average(
|
||||
metrics.processJudgeSum,
|
||||
metrics.processJudgeCount,
|
||||
),
|
||||
validatorRejectRate:
|
||||
metrics.count === 0
|
||||
? 0
|
||||
: round(metrics.validatorRejectCount / metrics.count),
|
||||
validatorSoftWarnRate:
|
||||
metrics.count === 0
|
||||
? 0
|
||||
: round(metrics.validatorSoftWarnCount / metrics.count),
|
||||
p50LatencyMs: calculatePercentile(metrics.latencySamples, 0.5),
|
||||
p95LatencyMs: calculatePercentile(metrics.latencySamples, 0.95),
|
||||
updatedAt: metrics.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export async function aggregateRecentEvalResults(
|
||||
garnetClient: AggregatorGarnetClient,
|
||||
options: { sinceId: string; nowMs: number },
|
||||
): Promise<{ lastSeenId: string; updatedSlices: number }> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
streamRangeStart(options.sinceId),
|
||||
"+",
|
||||
);
|
||||
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
lastSeenId: options.sinceId,
|
||||
updatedSlices: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const updatedKeys = new Set<string>();
|
||||
let lastSeenId = options.sinceId;
|
||||
|
||||
for (const entry of entries) {
|
||||
lastSeenId = entry.id;
|
||||
const observation = parseObservation(entry);
|
||||
if (!observation) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = getSliceAggregateKey(observation.slice);
|
||||
const current = metricsFromHash(await garnetClient.getHash(key));
|
||||
const next = updateMetrics(current, observation, options.nowMs);
|
||||
await garnetClient.setHash(key, metricsToHash(next));
|
||||
await garnetClient.expireKey(key, getEvalAggTtlSec());
|
||||
updatedKeys.add(key);
|
||||
}
|
||||
|
||||
await garnetClient.setString(EVAL_AGG_CURSOR_KEY, lastSeenId);
|
||||
|
||||
return {
|
||||
lastSeenId,
|
||||
updatedSlices: updatedKeys.size,
|
||||
};
|
||||
}
|
||||
|
||||
export async function readAggregatedSliceReportRows(
|
||||
garnetClient: AggregatorGarnetClient,
|
||||
): Promise<SliceReportRow[]> {
|
||||
const keys = await garnetClient.scanKeys(`${EVAL_AGG_SLICE_KEY_PREFIX}*`);
|
||||
const rows: SliceReportRow[] = [];
|
||||
|
||||
for (const key of keys) {
|
||||
const slice = sliceKeyFromRedisKey(key);
|
||||
if (!slice) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const metrics = metricsFromHash(await garnetClient.getHash(key));
|
||||
rows.push(sliceMetricsToReportRow(slice, metrics));
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
Reference in new issue
Block a user