Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,404 @@
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ValidationResult } from "@skills/onboarding/runtime/validator";
|
||||
import {
|
||||
getEvalArchiveBatchSize,
|
||||
getEvalArchiveTtlDays,
|
||||
} from "../config.js";
|
||||
import type { LlmJudgeResult } from "./types.js";
|
||||
import type { ProcessJudgeResult } from "./types.js";
|
||||
import {
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
type SliceKey,
|
||||
type SliceReportRow,
|
||||
} from "./aggregator.js";
|
||||
|
||||
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
|
||||
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
|
||||
|
||||
export interface EvalArchiveEntry {
|
||||
streamId: string;
|
||||
evaluatedAt: Date;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
derived: DerivedState;
|
||||
classified: ClassifiedIntent;
|
||||
validation: ValidationResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
process?: unknown;
|
||||
latencyMs: number | null;
|
||||
tier: 0 | 1 | 2;
|
||||
}
|
||||
|
||||
interface BulkWriteResultLike {
|
||||
upsertedCount?: number | undefined;
|
||||
}
|
||||
|
||||
interface FindCursorLike<T> {
|
||||
limit(value: number): FindCursorLike<T>;
|
||||
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
|
||||
toArray(): Promise<T[]>;
|
||||
}
|
||||
|
||||
interface EvalArchiveCollectionLike {
|
||||
bulkWrite(
|
||||
operations: unknown[],
|
||||
options?: { ordered?: boolean | undefined },
|
||||
): Promise<BulkWriteResultLike>;
|
||||
createIndex(
|
||||
keys: Record<string, unknown>,
|
||||
options?: Record<string, unknown>,
|
||||
): Promise<unknown>;
|
||||
find(filter: Record<string, unknown>): FindCursorLike<EvalArchiveEntry>;
|
||||
}
|
||||
|
||||
export interface MongoConnectionLike {
|
||||
collection(name: string): unknown;
|
||||
}
|
||||
|
||||
type ArchiveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"readStreamRange" | "setString"
|
||||
>;
|
||||
|
||||
function parseJsonField<T>(value: string | undefined, field: string): T {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`Missing ${field} field`);
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseFiniteNumber(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
||||
const parsed = parseFiniteNumber(value);
|
||||
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
||||
return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
|
||||
if (!value?.trim()) {
|
||||
return new Date(nowMs);
|
||||
}
|
||||
|
||||
const parsed = new Date(value);
|
||||
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
|
||||
}
|
||||
|
||||
function streamRangeStart(sinceId: string): string {
|
||||
const trimmed = sinceId.trim();
|
||||
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
||||
return "-";
|
||||
}
|
||||
return `(${trimmed}`;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function calculatePercentile(values: number[], percentile: number): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.min(
|
||||
sorted.length - 1,
|
||||
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
||||
);
|
||||
const selected = sorted[index];
|
||||
return selected === undefined ? null : round(selected);
|
||||
}
|
||||
|
||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
return {
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
};
|
||||
}
|
||||
|
||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
if (entry.llmJudge.skipped) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
function sliceKeyToMapKey(slice: SliceKey): string {
|
||||
return `${slice.tier}:${slice.intent}:${slice.phase}`;
|
||||
}
|
||||
|
||||
function parseEvalArchiveEntry(
|
||||
entry: StreamEntry,
|
||||
nowMs: number,
|
||||
): EvalArchiveEntry {
|
||||
const input = parseJsonField<{ request: AgentRequest }>(
|
||||
entry.message["input"],
|
||||
"input",
|
||||
);
|
||||
const derived = parseJsonField<DerivedState>(
|
||||
entry.message["derived"],
|
||||
"derived",
|
||||
);
|
||||
const processJudge = parseOptionalJsonField<ProcessJudgeResult>(
|
||||
entry.message["processJudge"],
|
||||
);
|
||||
const process = parseOptionalJsonField(entry.message["process"]);
|
||||
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived.tier);
|
||||
|
||||
if (tier === null) {
|
||||
throw new Error("Missing valid tier");
|
||||
}
|
||||
|
||||
const archiveEntry: EvalArchiveEntry = {
|
||||
streamId: entry.id,
|
||||
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
|
||||
request: input.request,
|
||||
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
|
||||
derived,
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
entry.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
validation: parseJsonField<ValidationResult>(
|
||||
entry.message["validation"],
|
||||
"validation",
|
||||
),
|
||||
llmJudge: parseJsonField<LlmJudgeResult>(
|
||||
entry.message["llmJudge"],
|
||||
"llmJudge",
|
||||
),
|
||||
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
|
||||
tier,
|
||||
};
|
||||
|
||||
if (processJudge !== undefined) {
|
||||
archiveEntry.processJudge = processJudge;
|
||||
}
|
||||
if (process !== undefined) {
|
||||
archiveEntry.process = process;
|
||||
}
|
||||
|
||||
return archiveEntry;
|
||||
}
|
||||
|
||||
async function ensureEvalArchiveIndexes(
|
||||
collection: EvalArchiveCollectionLike,
|
||||
): Promise<void> {
|
||||
await Promise.all([
|
||||
collection.createIndex({ streamId: 1 }, { unique: true }),
|
||||
collection.createIndex(
|
||||
{ evaluatedAt: 1 },
|
||||
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
|
||||
),
|
||||
collection.createIndex({
|
||||
tier: 1,
|
||||
"derived.intent.kind": 1,
|
||||
"derived.phase.kind": 1,
|
||||
evaluatedAt: -1,
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
export function getEvalArchiveCollection(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
): EvalArchiveCollectionLike {
|
||||
return mongoConnection.collection(EVAL_ARCHIVE_COLLECTION) as EvalArchiveCollectionLike;
|
||||
}
|
||||
|
||||
export async function archiveEvalResults(
|
||||
garnetClient: ArchiveGarnetClient,
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options: {
|
||||
batchSize?: number | undefined;
|
||||
nowMs: number;
|
||||
sinceId: string;
|
||||
},
|
||||
): Promise<{ lastArchivedId: string; archivedCount: number }> {
|
||||
const collection = getEvalArchiveCollection(mongoConnection);
|
||||
await ensureEvalArchiveIndexes(collection);
|
||||
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
streamRangeStart(options.sinceId),
|
||||
"+",
|
||||
{
|
||||
count: options.batchSize ?? getEvalArchiveBatchSize(),
|
||||
},
|
||||
);
|
||||
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
lastArchivedId: options.sinceId,
|
||||
archivedCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
|
||||
const archiveEntries = entries.flatMap((entry) => {
|
||||
try {
|
||||
return [parseEvalArchiveEntry(entry, options.nowMs)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
|
||||
let archivedCount = 0;
|
||||
if (archiveEntries.length > 0) {
|
||||
const result = await collection.bulkWrite(
|
||||
archiveEntries.map((entry) => ({
|
||||
updateOne: {
|
||||
filter: { streamId: entry.streamId },
|
||||
update: { $setOnInsert: entry },
|
||||
upsert: true,
|
||||
},
|
||||
})),
|
||||
{ ordered: false },
|
||||
);
|
||||
archivedCount = result.upsertedCount ?? 0;
|
||||
}
|
||||
|
||||
await garnetClient.setString(EVAL_ARCHIVE_CURSOR_KEY, lastArchivedId);
|
||||
|
||||
return {
|
||||
lastArchivedId,
|
||||
archivedCount,
|
||||
};
|
||||
}
|
||||
|
||||
export function archiveEntryToSliceReportRows(
|
||||
entries: EvalArchiveEntry[],
|
||||
): SliceReportRow[] {
|
||||
const grouped = new Map<
|
||||
string,
|
||||
{
|
||||
count: number;
|
||||
latencySamples: number[];
|
||||
llmOverallCount: number;
|
||||
llmOverallSum: number;
|
||||
slice: SliceKey;
|
||||
validatorRejectCount: number;
|
||||
validatorSoftWarnCount: number;
|
||||
updatedAt: Date;
|
||||
}
|
||||
>();
|
||||
|
||||
for (const entry of entries) {
|
||||
const slice = getSlice(entry);
|
||||
const key = sliceKeyToMapKey(slice);
|
||||
const current =
|
||||
grouped.get(key) ??
|
||||
{
|
||||
count: 0,
|
||||
latencySamples: [],
|
||||
llmOverallCount: 0,
|
||||
llmOverallSum: 0,
|
||||
slice,
|
||||
validatorRejectCount: 0,
|
||||
validatorSoftWarnCount: 0,
|
||||
updatedAt: new Date(0),
|
||||
};
|
||||
|
||||
current.count += 1;
|
||||
if (entry.latencyMs !== null) {
|
||||
current.latencySamples.push(entry.latencyMs);
|
||||
}
|
||||
const llmOverall = getLlmOverall(entry);
|
||||
if (llmOverall !== null) {
|
||||
current.llmOverallCount += 1;
|
||||
current.llmOverallSum += llmOverall;
|
||||
}
|
||||
if (entry.validation.status === "reject") {
|
||||
current.validatorRejectCount += 1;
|
||||
}
|
||||
if (entry.validation.status === "soft-warn") {
|
||||
current.validatorSoftWarnCount += 1;
|
||||
}
|
||||
if (entry.evaluatedAt > current.updatedAt) {
|
||||
current.updatedAt = entry.evaluatedAt;
|
||||
}
|
||||
|
||||
grouped.set(key, current);
|
||||
}
|
||||
|
||||
return [...grouped.values()].map((group) => ({
|
||||
slice: group.slice,
|
||||
count: group.count,
|
||||
llmOverallAvg:
|
||||
group.llmOverallCount === 0
|
||||
? null
|
||||
: round(group.llmOverallSum / group.llmOverallCount),
|
||||
processJudgeAvg: null,
|
||||
validatorRejectRate:
|
||||
group.count === 0 ? 0 : round(group.validatorRejectCount / group.count),
|
||||
validatorSoftWarnRate:
|
||||
group.count === 0
|
||||
? 0
|
||||
: round(group.validatorSoftWarnCount / group.count),
|
||||
p50LatencyMs: calculatePercentile(group.latencySamples, 0.5),
|
||||
p95LatencyMs: calculatePercentile(group.latencySamples, 0.95),
|
||||
updatedAt: group.updatedAt.toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function queryArchiveSliceReportRows(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
limit?: number | undefined;
|
||||
sinceMs?: number | undefined;
|
||||
sort?: "count-desc" | "score-asc" | "score-desc" | undefined;
|
||||
},
|
||||
): Promise<SliceReportRow[]> {
|
||||
const sinceMs = options?.sinceMs ?? 7 * 24 * 60 * 60 * 1000;
|
||||
const since = new Date(Date.now() - sinceMs);
|
||||
const archiveEntries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find({ evaluatedAt: { $gte: since } })
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(10_000)
|
||||
.toArray();
|
||||
const rows = archiveEntryToSliceReportRows(archiveEntries);
|
||||
const sort = options?.sort ?? "count-desc";
|
||||
|
||||
rows.sort((left, right) => {
|
||||
if (sort === "score-asc") {
|
||||
return (left.llmOverallAvg ?? Number.POSITIVE_INFINITY) -
|
||||
(right.llmOverallAvg ?? Number.POSITIVE_INFINITY);
|
||||
}
|
||||
if (sort === "score-desc") {
|
||||
return (right.llmOverallAvg ?? Number.NEGATIVE_INFINITY) -
|
||||
(left.llmOverallAvg ?? Number.NEGATIVE_INFINITY);
|
||||
}
|
||||
return right.count - left.count;
|
||||
});
|
||||
|
||||
return rows.slice(0, options?.limit ?? 20);
|
||||
}
|
||||
Reference in new issue
Block a user