405 lines
11 KiB
TypeScript
405 lines
11 KiB
TypeScript
import type { AgentRequest, AgentResponse } from "@agent/types";
|
|
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
|
import type {
|
|
ClassifiedIntent,
|
|
DerivedState,
|
|
} from "@skills/onboarding/state/onboarding-state";
|
|
import type { ValidationResult } from "@skills/onboarding/runtime/validator";
|
|
import {
|
|
getEvalArchiveBatchSize,
|
|
getEvalArchiveTtlDays,
|
|
} from "../config.js";
|
|
import type { LlmJudgeResult } from "./types.js";
|
|
import type { ProcessJudgeResult } from "./types.js";
|
|
import {
|
|
EVAL_RESULTS_STREAM_KEY,
|
|
type SliceKey,
|
|
type SliceReportRow,
|
|
} from "./aggregator.js";
|
|
|
|
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
|
|
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
|
|
|
|
export interface EvalArchiveEntry {
|
|
streamId: string;
|
|
evaluatedAt: Date;
|
|
request: AgentRequest;
|
|
response: AgentResponse;
|
|
derived: DerivedState;
|
|
classified: ClassifiedIntent;
|
|
validation: ValidationResult;
|
|
llmJudge: LlmJudgeResult;
|
|
processJudge?: ProcessJudgeResult | undefined;
|
|
process?: unknown;
|
|
latencyMs: number | null;
|
|
tier: 0 | 1 | 2;
|
|
}
|
|
|
|
interface BulkWriteResultLike {
|
|
upsertedCount?: number | undefined;
|
|
}
|
|
|
|
interface FindCursorLike<T> {
|
|
limit(value: number): FindCursorLike<T>;
|
|
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
|
|
toArray(): Promise<T[]>;
|
|
}
|
|
|
|
interface EvalArchiveCollectionLike {
|
|
bulkWrite(
|
|
operations: unknown[],
|
|
options?: { ordered?: boolean | undefined },
|
|
): Promise<BulkWriteResultLike>;
|
|
createIndex(
|
|
keys: Record<string, unknown>,
|
|
options?: Record<string, unknown>,
|
|
): Promise<unknown>;
|
|
find(filter: Record<string, unknown>): FindCursorLike<EvalArchiveEntry>;
|
|
}
|
|
|
|
export interface MongoConnectionLike {
|
|
collection(name: string): unknown;
|
|
}
|
|
|
|
type ArchiveGarnetClient = Pick<
|
|
GarnetClient,
|
|
"readStreamRange" | "setString"
|
|
>;
|
|
|
|
function parseJsonField<T>(value: string | undefined, field: string): T {
|
|
if (!value?.trim()) {
|
|
throw new Error(`Missing ${field} field`);
|
|
}
|
|
|
|
return JSON.parse(value) as T;
|
|
}
|
|
|
|
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
|
if (!value?.trim()) {
|
|
return undefined;
|
|
}
|
|
|
|
return JSON.parse(value) as T;
|
|
}
|
|
|
|
function parseFiniteNumber(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) {
|
|
return value;
|
|
}
|
|
if (typeof value !== "string" || !value.trim()) {
|
|
return null;
|
|
}
|
|
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
|
const parsed = parseFiniteNumber(value);
|
|
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
|
return parsed;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
|
|
if (!value?.trim()) {
|
|
return new Date(nowMs);
|
|
}
|
|
|
|
const parsed = new Date(value);
|
|
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
|
|
}
|
|
|
|
function streamRangeStart(sinceId: string): string {
|
|
const trimmed = sinceId.trim();
|
|
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
|
return "-";
|
|
}
|
|
return `(${trimmed}`;
|
|
}
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(3));
|
|
}
|
|
|
|
function calculatePercentile(values: number[], percentile: number): number | null {
|
|
if (values.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
const sorted = [...values].sort((left, right) => left - right);
|
|
const index = Math.min(
|
|
sorted.length - 1,
|
|
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
|
);
|
|
const selected = sorted[index];
|
|
return selected === undefined ? null : round(selected);
|
|
}
|
|
|
|
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
|
return {
|
|
tier: entry.tier,
|
|
intent: entry.derived.intent.kind,
|
|
phase: entry.derived.phase.kind,
|
|
};
|
|
}
|
|
|
|
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
|
if (entry.llmJudge.skipped) {
|
|
return null;
|
|
}
|
|
|
|
return entry.llmJudge.scores.overall;
|
|
}
|
|
|
|
function sliceKeyToMapKey(slice: SliceKey): string {
|
|
return `${slice.tier}:${slice.intent}:${slice.phase}`;
|
|
}
|
|
|
|
function parseEvalArchiveEntry(
|
|
entry: StreamEntry,
|
|
nowMs: number,
|
|
): EvalArchiveEntry {
|
|
const input = parseJsonField<{ request: AgentRequest }>(
|
|
entry.message["input"],
|
|
"input",
|
|
);
|
|
const derived = parseJsonField<DerivedState>(
|
|
entry.message["derived"],
|
|
"derived",
|
|
);
|
|
const processJudge = parseOptionalJsonField<ProcessJudgeResult>(
|
|
entry.message["processJudge"],
|
|
);
|
|
const process = parseOptionalJsonField(entry.message["process"]);
|
|
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived.tier);
|
|
|
|
if (tier === null) {
|
|
throw new Error("Missing valid tier");
|
|
}
|
|
|
|
const archiveEntry: EvalArchiveEntry = {
|
|
streamId: entry.id,
|
|
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
|
|
request: input.request,
|
|
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
|
|
derived,
|
|
classified: parseJsonField<ClassifiedIntent>(
|
|
entry.message["classified"],
|
|
"classified",
|
|
),
|
|
validation: parseJsonField<ValidationResult>(
|
|
entry.message["validation"],
|
|
"validation",
|
|
),
|
|
llmJudge: parseJsonField<LlmJudgeResult>(
|
|
entry.message["llmJudge"],
|
|
"llmJudge",
|
|
),
|
|
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
|
|
tier,
|
|
};
|
|
|
|
if (processJudge !== undefined) {
|
|
archiveEntry.processJudge = processJudge;
|
|
}
|
|
if (process !== undefined) {
|
|
archiveEntry.process = process;
|
|
}
|
|
|
|
return archiveEntry;
|
|
}
|
|
|
|
async function ensureEvalArchiveIndexes(
|
|
collection: EvalArchiveCollectionLike,
|
|
): Promise<void> {
|
|
await Promise.all([
|
|
collection.createIndex({ streamId: 1 }, { unique: true }),
|
|
collection.createIndex(
|
|
{ evaluatedAt: 1 },
|
|
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
|
|
),
|
|
collection.createIndex({
|
|
tier: 1,
|
|
"derived.intent.kind": 1,
|
|
"derived.phase.kind": 1,
|
|
evaluatedAt: -1,
|
|
}),
|
|
]);
|
|
}
|
|
|
|
export function getEvalArchiveCollection(
|
|
mongoConnection: MongoConnectionLike,
|
|
): EvalArchiveCollectionLike {
|
|
return mongoConnection.collection(EVAL_ARCHIVE_COLLECTION) as EvalArchiveCollectionLike;
|
|
}
|
|
|
|
export async function archiveEvalResults(
|
|
garnetClient: ArchiveGarnetClient,
|
|
mongoConnection: MongoConnectionLike,
|
|
options: {
|
|
batchSize?: number | undefined;
|
|
nowMs: number;
|
|
sinceId: string;
|
|
},
|
|
): Promise<{ lastArchivedId: string; archivedCount: number }> {
|
|
const collection = getEvalArchiveCollection(mongoConnection);
|
|
await ensureEvalArchiveIndexes(collection);
|
|
|
|
const entries = await garnetClient.readStreamRange(
|
|
EVAL_RESULTS_STREAM_KEY,
|
|
streamRangeStart(options.sinceId),
|
|
"+",
|
|
{
|
|
count: options.batchSize ?? getEvalArchiveBatchSize(),
|
|
},
|
|
);
|
|
|
|
if (entries.length === 0) {
|
|
return {
|
|
lastArchivedId: options.sinceId,
|
|
archivedCount: 0,
|
|
};
|
|
}
|
|
|
|
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
|
|
const archiveEntries = entries.flatMap((entry) => {
|
|
try {
|
|
return [parseEvalArchiveEntry(entry, options.nowMs)];
|
|
} catch {
|
|
return [];
|
|
}
|
|
});
|
|
|
|
let archivedCount = 0;
|
|
if (archiveEntries.length > 0) {
|
|
const result = await collection.bulkWrite(
|
|
archiveEntries.map((entry) => ({
|
|
updateOne: {
|
|
filter: { streamId: entry.streamId },
|
|
update: { $setOnInsert: entry },
|
|
upsert: true,
|
|
},
|
|
})),
|
|
{ ordered: false },
|
|
);
|
|
archivedCount = result.upsertedCount ?? 0;
|
|
}
|
|
|
|
await garnetClient.setString(EVAL_ARCHIVE_CURSOR_KEY, lastArchivedId);
|
|
|
|
return {
|
|
lastArchivedId,
|
|
archivedCount,
|
|
};
|
|
}
|
|
|
|
export function archiveEntryToSliceReportRows(
|
|
entries: EvalArchiveEntry[],
|
|
): SliceReportRow[] {
|
|
const grouped = new Map<
|
|
string,
|
|
{
|
|
count: number;
|
|
latencySamples: number[];
|
|
llmOverallCount: number;
|
|
llmOverallSum: number;
|
|
slice: SliceKey;
|
|
validatorRejectCount: number;
|
|
validatorSoftWarnCount: number;
|
|
updatedAt: Date;
|
|
}
|
|
>();
|
|
|
|
for (const entry of entries) {
|
|
const slice = getSlice(entry);
|
|
const key = sliceKeyToMapKey(slice);
|
|
const current =
|
|
grouped.get(key) ??
|
|
{
|
|
count: 0,
|
|
latencySamples: [],
|
|
llmOverallCount: 0,
|
|
llmOverallSum: 0,
|
|
slice,
|
|
validatorRejectCount: 0,
|
|
validatorSoftWarnCount: 0,
|
|
updatedAt: new Date(0),
|
|
};
|
|
|
|
current.count += 1;
|
|
if (entry.latencyMs !== null) {
|
|
current.latencySamples.push(entry.latencyMs);
|
|
}
|
|
const llmOverall = getLlmOverall(entry);
|
|
if (llmOverall !== null) {
|
|
current.llmOverallCount += 1;
|
|
current.llmOverallSum += llmOverall;
|
|
}
|
|
if (entry.validation.status === "reject") {
|
|
current.validatorRejectCount += 1;
|
|
}
|
|
if (entry.validation.status === "soft-warn") {
|
|
current.validatorSoftWarnCount += 1;
|
|
}
|
|
if (entry.evaluatedAt > current.updatedAt) {
|
|
current.updatedAt = entry.evaluatedAt;
|
|
}
|
|
|
|
grouped.set(key, current);
|
|
}
|
|
|
|
return [...grouped.values()].map((group) => ({
|
|
slice: group.slice,
|
|
count: group.count,
|
|
llmOverallAvg:
|
|
group.llmOverallCount === 0
|
|
? null
|
|
: round(group.llmOverallSum / group.llmOverallCount),
|
|
processJudgeAvg: null,
|
|
validatorRejectRate:
|
|
group.count === 0 ? 0 : round(group.validatorRejectCount / group.count),
|
|
validatorSoftWarnRate:
|
|
group.count === 0
|
|
? 0
|
|
: round(group.validatorSoftWarnCount / group.count),
|
|
p50LatencyMs: calculatePercentile(group.latencySamples, 0.5),
|
|
p95LatencyMs: calculatePercentile(group.latencySamples, 0.95),
|
|
updatedAt: group.updatedAt.toISOString(),
|
|
}));
|
|
}
|
|
|
|
export async function queryArchiveSliceReportRows(
|
|
mongoConnection: MongoConnectionLike,
|
|
options?: {
|
|
limit?: number | undefined;
|
|
sinceMs?: number | undefined;
|
|
sort?: "count-desc" | "score-asc" | "score-desc" | undefined;
|
|
},
|
|
): Promise<SliceReportRow[]> {
|
|
const sinceMs = options?.sinceMs ?? 7 * 24 * 60 * 60 * 1000;
|
|
const since = new Date(Date.now() - sinceMs);
|
|
const archiveEntries = await getEvalArchiveCollection(mongoConnection)
|
|
.find({ evaluatedAt: { $gte: since } })
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(10_000)
|
|
.toArray();
|
|
const rows = archiveEntryToSliceReportRows(archiveEntries);
|
|
const sort = options?.sort ?? "count-desc";
|
|
|
|
rows.sort((left, right) => {
|
|
if (sort === "score-asc") {
|
|
return (left.llmOverallAvg ?? Number.POSITIVE_INFINITY) -
|
|
(right.llmOverallAvg ?? Number.POSITIVE_INFINITY);
|
|
}
|
|
if (sort === "score-desc") {
|
|
return (right.llmOverallAvg ?? Number.NEGATIVE_INFINITY) -
|
|
(left.llmOverallAvg ?? Number.NEGATIVE_INFINITY);
|
|
}
|
|
return right.count - left.count;
|
|
});
|
|
|
|
return rows.slice(0, options?.limit ?? 20);
|
|
}
|