Files
lolympus-agent/src/eval/archive.ts
T

405 lines
11 KiB
TypeScript

import type { AgentRequest, AgentResponse } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ValidationResult } from "@skills/onboarding/runtime/validator";
import {
getEvalArchiveBatchSize,
getEvalArchiveTtlDays,
} from "../config.js";
import type { LlmJudgeResult } from "./types.js";
import type { ProcessJudgeResult } from "./types.js";
import {
EVAL_RESULTS_STREAM_KEY,
type SliceKey,
type SliceReportRow,
} from "./aggregator.js";
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
export interface EvalArchiveEntry {
streamId: string;
evaluatedAt: Date;
request: AgentRequest;
response: AgentResponse;
derived: DerivedState;
classified: ClassifiedIntent;
validation: ValidationResult;
llmJudge: LlmJudgeResult;
processJudge?: ProcessJudgeResult | undefined;
process?: unknown;
latencyMs: number | null;
tier: 0 | 1 | 2;
}
interface BulkWriteResultLike {
upsertedCount?: number | undefined;
}
interface FindCursorLike<T> {
limit(value: number): FindCursorLike<T>;
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
toArray(): Promise<T[]>;
}
interface EvalArchiveCollectionLike {
bulkWrite(
operations: unknown[],
options?: { ordered?: boolean | undefined },
): Promise<BulkWriteResultLike>;
createIndex(
keys: Record<string, unknown>,
options?: Record<string, unknown>,
): Promise<unknown>;
find(filter: Record<string, unknown>): FindCursorLike<EvalArchiveEntry>;
}
export interface MongoConnectionLike {
collection(name: string): unknown;
}
type ArchiveGarnetClient = Pick<
GarnetClient,
"readStreamRange" | "setString"
>;
function parseJsonField<T>(value: string | undefined, field: string): T {
if (!value?.trim()) {
throw new Error(`Missing ${field} field`);
}
return JSON.parse(value) as T;
}
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
if (!value?.trim()) {
return undefined;
}
return JSON.parse(value) as T;
}
function parseFiniteNumber(value: unknown): number | null {
if (typeof value === "number" && Number.isFinite(value)) {
return value;
}
if (typeof value !== "string" || !value.trim()) {
return null;
}
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function parseTier(value: unknown): 0 | 1 | 2 | null {
const parsed = parseFiniteNumber(value);
if (parsed === 0 || parsed === 1 || parsed === 2) {
return parsed;
}
return null;
}
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
if (!value?.trim()) {
return new Date(nowMs);
}
const parsed = new Date(value);
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
}
function streamRangeStart(sinceId: string): string {
const trimmed = sinceId.trim();
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
return "-";
}
return `(${trimmed}`;
}
function round(value: number): number {
return Number(value.toFixed(3));
}
function calculatePercentile(values: number[], percentile: number): number | null {
if (values.length === 0) {
return null;
}
const sorted = [...values].sort((left, right) => left - right);
const index = Math.min(
sorted.length - 1,
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
);
const selected = sorted[index];
return selected === undefined ? null : round(selected);
}
function getSlice(entry: EvalArchiveEntry): SliceKey {
return {
tier: entry.tier,
intent: entry.derived.intent.kind,
phase: entry.derived.phase.kind,
};
}
function getLlmOverall(entry: EvalArchiveEntry): number | null {
if (entry.llmJudge.skipped) {
return null;
}
return entry.llmJudge.scores.overall;
}
function sliceKeyToMapKey(slice: SliceKey): string {
return `${slice.tier}:${slice.intent}:${slice.phase}`;
}
function parseEvalArchiveEntry(
entry: StreamEntry,
nowMs: number,
): EvalArchiveEntry {
const input = parseJsonField<{ request: AgentRequest }>(
entry.message["input"],
"input",
);
const derived = parseJsonField<DerivedState>(
entry.message["derived"],
"derived",
);
const processJudge = parseOptionalJsonField<ProcessJudgeResult>(
entry.message["processJudge"],
);
const process = parseOptionalJsonField(entry.message["process"]);
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived.tier);
if (tier === null) {
throw new Error("Missing valid tier");
}
const archiveEntry: EvalArchiveEntry = {
streamId: entry.id,
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
request: input.request,
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
derived,
classified: parseJsonField<ClassifiedIntent>(
entry.message["classified"],
"classified",
),
validation: parseJsonField<ValidationResult>(
entry.message["validation"],
"validation",
),
llmJudge: parseJsonField<LlmJudgeResult>(
entry.message["llmJudge"],
"llmJudge",
),
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
tier,
};
if (processJudge !== undefined) {
archiveEntry.processJudge = processJudge;
}
if (process !== undefined) {
archiveEntry.process = process;
}
return archiveEntry;
}
async function ensureEvalArchiveIndexes(
collection: EvalArchiveCollectionLike,
): Promise<void> {
await Promise.all([
collection.createIndex({ streamId: 1 }, { unique: true }),
collection.createIndex(
{ evaluatedAt: 1 },
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
),
collection.createIndex({
tier: 1,
"derived.intent.kind": 1,
"derived.phase.kind": 1,
evaluatedAt: -1,
}),
]);
}
export function getEvalArchiveCollection(
mongoConnection: MongoConnectionLike,
): EvalArchiveCollectionLike {
return mongoConnection.collection(EVAL_ARCHIVE_COLLECTION) as EvalArchiveCollectionLike;
}
export async function archiveEvalResults(
garnetClient: ArchiveGarnetClient,
mongoConnection: MongoConnectionLike,
options: {
batchSize?: number | undefined;
nowMs: number;
sinceId: string;
},
): Promise<{ lastArchivedId: string; archivedCount: number }> {
const collection = getEvalArchiveCollection(mongoConnection);
await ensureEvalArchiveIndexes(collection);
const entries = await garnetClient.readStreamRange(
EVAL_RESULTS_STREAM_KEY,
streamRangeStart(options.sinceId),
"+",
{
count: options.batchSize ?? getEvalArchiveBatchSize(),
},
);
if (entries.length === 0) {
return {
lastArchivedId: options.sinceId,
archivedCount: 0,
};
}
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
const archiveEntries = entries.flatMap((entry) => {
try {
return [parseEvalArchiveEntry(entry, options.nowMs)];
} catch {
return [];
}
});
let archivedCount = 0;
if (archiveEntries.length > 0) {
const result = await collection.bulkWrite(
archiveEntries.map((entry) => ({
updateOne: {
filter: { streamId: entry.streamId },
update: { $setOnInsert: entry },
upsert: true,
},
})),
{ ordered: false },
);
archivedCount = result.upsertedCount ?? 0;
}
await garnetClient.setString(EVAL_ARCHIVE_CURSOR_KEY, lastArchivedId);
return {
lastArchivedId,
archivedCount,
};
}
export function archiveEntryToSliceReportRows(
entries: EvalArchiveEntry[],
): SliceReportRow[] {
const grouped = new Map<
string,
{
count: number;
latencySamples: number[];
llmOverallCount: number;
llmOverallSum: number;
slice: SliceKey;
validatorRejectCount: number;
validatorSoftWarnCount: number;
updatedAt: Date;
}
>();
for (const entry of entries) {
const slice = getSlice(entry);
const key = sliceKeyToMapKey(slice);
const current =
grouped.get(key) ??
{
count: 0,
latencySamples: [],
llmOverallCount: 0,
llmOverallSum: 0,
slice,
validatorRejectCount: 0,
validatorSoftWarnCount: 0,
updatedAt: new Date(0),
};
current.count += 1;
if (entry.latencyMs !== null) {
current.latencySamples.push(entry.latencyMs);
}
const llmOverall = getLlmOverall(entry);
if (llmOverall !== null) {
current.llmOverallCount += 1;
current.llmOverallSum += llmOverall;
}
if (entry.validation.status === "reject") {
current.validatorRejectCount += 1;
}
if (entry.validation.status === "soft-warn") {
current.validatorSoftWarnCount += 1;
}
if (entry.evaluatedAt > current.updatedAt) {
current.updatedAt = entry.evaluatedAt;
}
grouped.set(key, current);
}
return [...grouped.values()].map((group) => ({
slice: group.slice,
count: group.count,
llmOverallAvg:
group.llmOverallCount === 0
? null
: round(group.llmOverallSum / group.llmOverallCount),
processJudgeAvg: null,
validatorRejectRate:
group.count === 0 ? 0 : round(group.validatorRejectCount / group.count),
validatorSoftWarnRate:
group.count === 0
? 0
: round(group.validatorSoftWarnCount / group.count),
p50LatencyMs: calculatePercentile(group.latencySamples, 0.5),
p95LatencyMs: calculatePercentile(group.latencySamples, 0.95),
updatedAt: group.updatedAt.toISOString(),
}));
}
export async function queryArchiveSliceReportRows(
mongoConnection: MongoConnectionLike,
options?: {
limit?: number | undefined;
sinceMs?: number | undefined;
sort?: "count-desc" | "score-asc" | "score-desc" | undefined;
},
): Promise<SliceReportRow[]> {
const sinceMs = options?.sinceMs ?? 7 * 24 * 60 * 60 * 1000;
const since = new Date(Date.now() - sinceMs);
const archiveEntries = await getEvalArchiveCollection(mongoConnection)
.find({ evaluatedAt: { $gte: since } })
.sort({ evaluatedAt: -1 })
.limit(10_000)
.toArray();
const rows = archiveEntryToSliceReportRows(archiveEntries);
const sort = options?.sort ?? "count-desc";
rows.sort((left, right) => {
if (sort === "score-asc") {
return (left.llmOverallAvg ?? Number.POSITIVE_INFINITY) -
(right.llmOverallAvg ?? Number.POSITIVE_INFINITY);
}
if (sort === "score-desc") {
return (right.llmOverallAvg ?? Number.NEGATIVE_INFINITY) -
(left.llmOverallAvg ?? Number.NEGATIVE_INFINITY);
}
return right.count - left.count;
});
return rows.slice(0, options?.limit ?? 20);
}