import type { AgentRequest, AgentResponse } from "@agent/types"; import type { GarnetClient, StreamEntry } from "@common/garnet"; import type { ClassifiedIntent, DerivedState, } from "@skills/onboarding/state/onboarding-state"; import type { ValidationResult } from "@skills/onboarding/runtime/validator"; import { getEvalArchiveBatchSize, getEvalArchiveTtlDays, } from "../config.js"; import type { LlmJudgeResult } from "./types.js"; import type { ProcessJudgeResult } from "./types.js"; import { EVAL_RESULTS_STREAM_KEY, type SliceKey, type SliceReportRow, } from "./aggregator.js"; export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor"; export const EVAL_ARCHIVE_COLLECTION = "eval_archive"; export interface EvalArchiveEntry { streamId: string; evaluatedAt: Date; request: AgentRequest; response: AgentResponse; derived: DerivedState; classified: ClassifiedIntent; validation: ValidationResult; llmJudge: LlmJudgeResult; processJudge?: ProcessJudgeResult | undefined; process?: unknown; latencyMs: number | null; tier: 0 | 1 | 2; } interface BulkWriteResultLike { upsertedCount?: number | undefined; } interface FindCursorLike { limit(value: number): FindCursorLike; sort(value: Record): FindCursorLike; toArray(): Promise; } interface EvalArchiveCollectionLike { bulkWrite( operations: unknown[], options?: { ordered?: boolean | undefined }, ): Promise; createIndex( keys: Record, options?: Record, ): Promise; find(filter: Record): FindCursorLike; } export interface MongoConnectionLike { collection(name: string): unknown; } type ArchiveGarnetClient = Pick< GarnetClient, "readStreamRange" | "setString" >; function parseJsonField(value: string | undefined, field: string): T { if (!value?.trim()) { throw new Error(`Missing ${field} field`); } return JSON.parse(value) as T; } function parseOptionalJsonField(value: string | undefined): T | undefined { if (!value?.trim()) { return undefined; } return JSON.parse(value) as T; } function parseFiniteNumber(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) { return value; } if (typeof value !== "string" || !value.trim()) { return null; } const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function parseTier(value: unknown): 0 | 1 | 2 | null { const parsed = parseFiniteNumber(value); if (parsed === 0 || parsed === 1 || parsed === 2) { return parsed; } return null; } function parseEvaluatedAt(value: string | undefined, nowMs: number): Date { if (!value?.trim()) { return new Date(nowMs); } const parsed = new Date(value); return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs); } function streamRangeStart(sinceId: string): string { const trimmed = sinceId.trim(); if (!trimmed || trimmed === "0-0" || trimmed === "-") { return "-"; } return `(${trimmed}`; } function round(value: number): number { return Number(value.toFixed(3)); } function calculatePercentile(values: number[], percentile: number): number | null { if (values.length === 0) { return null; } const sorted = [...values].sort((left, right) => left - right); const index = Math.min( sorted.length - 1, Math.max(0, Math.ceil(sorted.length * percentile) - 1), ); const selected = sorted[index]; return selected === undefined ? null : round(selected); } function getSlice(entry: EvalArchiveEntry): SliceKey { return { tier: entry.tier, intent: entry.derived.intent.kind, phase: entry.derived.phase.kind, }; } function getLlmOverall(entry: EvalArchiveEntry): number | null { if (entry.llmJudge.skipped) { return null; } return entry.llmJudge.scores.overall; } function sliceKeyToMapKey(slice: SliceKey): string { return `${slice.tier}:${slice.intent}:${slice.phase}`; } function parseEvalArchiveEntry( entry: StreamEntry, nowMs: number, ): EvalArchiveEntry { const input = parseJsonField<{ request: AgentRequest }>( entry.message["input"], "input", ); const derived = parseJsonField( entry.message["derived"], "derived", ); const processJudge = parseOptionalJsonField( entry.message["processJudge"], ); const process = parseOptionalJsonField(entry.message["process"]); const tier = parseTier(entry.message["tier"]) ?? parseTier(derived.tier); if (tier === null) { throw new Error("Missing valid tier"); } const archiveEntry: EvalArchiveEntry = { streamId: entry.id, evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs), request: input.request, response: parseJsonField(entry.message["output"], "output"), derived, classified: parseJsonField( entry.message["classified"], "classified", ), validation: parseJsonField( entry.message["validation"], "validation", ), llmJudge: parseJsonField( entry.message["llmJudge"], "llmJudge", ), latencyMs: parseFiniteNumber(entry.message["latencyMs"]), tier, }; if (processJudge !== undefined) { archiveEntry.processJudge = processJudge; } if (process !== undefined) { archiveEntry.process = process; } return archiveEntry; } async function ensureEvalArchiveIndexes( collection: EvalArchiveCollectionLike, ): Promise { await Promise.all([ collection.createIndex({ streamId: 1 }, { unique: true }), collection.createIndex( { evaluatedAt: 1 }, { expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 }, ), collection.createIndex({ tier: 1, "derived.intent.kind": 1, "derived.phase.kind": 1, evaluatedAt: -1, }), ]); } export function getEvalArchiveCollection( mongoConnection: MongoConnectionLike, ): EvalArchiveCollectionLike { return mongoConnection.collection(EVAL_ARCHIVE_COLLECTION) as EvalArchiveCollectionLike; } export async function archiveEvalResults( garnetClient: ArchiveGarnetClient, mongoConnection: MongoConnectionLike, options: { batchSize?: number | undefined; nowMs: number; sinceId: string; }, ): Promise<{ lastArchivedId: string; archivedCount: number }> { const collection = getEvalArchiveCollection(mongoConnection); await ensureEvalArchiveIndexes(collection); const entries = await garnetClient.readStreamRange( EVAL_RESULTS_STREAM_KEY, streamRangeStart(options.sinceId), "+", { count: options.batchSize ?? getEvalArchiveBatchSize(), }, ); if (entries.length === 0) { return { lastArchivedId: options.sinceId, archivedCount: 0, }; } const lastArchivedId = entries.at(-1)?.id ?? options.sinceId; const archiveEntries = entries.flatMap((entry) => { try { return [parseEvalArchiveEntry(entry, options.nowMs)]; } catch { return []; } }); let archivedCount = 0; if (archiveEntries.length > 0) { const result = await collection.bulkWrite( archiveEntries.map((entry) => ({ updateOne: { filter: { streamId: entry.streamId }, update: { $setOnInsert: entry }, upsert: true, }, })), { ordered: false }, ); archivedCount = result.upsertedCount ?? 0; } await garnetClient.setString(EVAL_ARCHIVE_CURSOR_KEY, lastArchivedId); return { lastArchivedId, archivedCount, }; } export function archiveEntryToSliceReportRows( entries: EvalArchiveEntry[], ): SliceReportRow[] { const grouped = new Map< string, { count: number; latencySamples: number[]; llmOverallCount: number; llmOverallSum: number; slice: SliceKey; validatorRejectCount: number; validatorSoftWarnCount: number; updatedAt: Date; } >(); for (const entry of entries) { const slice = getSlice(entry); const key = sliceKeyToMapKey(slice); const current = grouped.get(key) ?? { count: 0, latencySamples: [], llmOverallCount: 0, llmOverallSum: 0, slice, validatorRejectCount: 0, validatorSoftWarnCount: 0, updatedAt: new Date(0), }; current.count += 1; if (entry.latencyMs !== null) { current.latencySamples.push(entry.latencyMs); } const llmOverall = getLlmOverall(entry); if (llmOverall !== null) { current.llmOverallCount += 1; current.llmOverallSum += llmOverall; } if (entry.validation.status === "reject") { current.validatorRejectCount += 1; } if (entry.validation.status === "soft-warn") { current.validatorSoftWarnCount += 1; } if (entry.evaluatedAt > current.updatedAt) { current.updatedAt = entry.evaluatedAt; } grouped.set(key, current); } return [...grouped.values()].map((group) => ({ slice: group.slice, count: group.count, llmOverallAvg: group.llmOverallCount === 0 ? null : round(group.llmOverallSum / group.llmOverallCount), processJudgeAvg: null, validatorRejectRate: group.count === 0 ? 0 : round(group.validatorRejectCount / group.count), validatorSoftWarnRate: group.count === 0 ? 0 : round(group.validatorSoftWarnCount / group.count), p50LatencyMs: calculatePercentile(group.latencySamples, 0.5), p95LatencyMs: calculatePercentile(group.latencySamples, 0.95), updatedAt: group.updatedAt.toISOString(), })); } export async function queryArchiveSliceReportRows( mongoConnection: MongoConnectionLike, options?: { limit?: number | undefined; sinceMs?: number | undefined; sort?: "count-desc" | "score-asc" | "score-desc" | undefined; }, ): Promise { const sinceMs = options?.sinceMs ?? 7 * 24 * 60 * 60 * 1000; const since = new Date(Date.now() - sinceMs); const archiveEntries = await getEvalArchiveCollection(mongoConnection) .find({ evaluatedAt: { $gte: since } }) .sort({ evaluatedAt: -1 }) .limit(10_000) .toArray(); const rows = archiveEntryToSliceReportRows(archiveEntries); const sort = options?.sort ?? "count-desc"; rows.sort((left, right) => { if (sort === "score-asc") { return (left.llmOverallAvg ?? Number.POSITIVE_INFINITY) - (right.llmOverallAvg ?? Number.POSITIVE_INFINITY); } if (sort === "score-desc") { return (right.llmOverallAvg ?? Number.NEGATIVE_INFINITY) - (left.llmOverallAvg ?? Number.NEGATIVE_INFINITY); } return right.count - left.count; }); return rows.slice(0, options?.limit ?? 20); }