Files
lolympus-agent/src/eval/review/cli.ts
T

270 lines
7.5 KiB
TypeScript

import { execFileSync } from "node:child_process";
import { mkdir, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { createInterface } from "node:readline/promises";
import { stdin as input, stdout as output } from "node:process";
import type { GarnetClient } from "@common/garnet";
import {
acknowledgeReviewItem,
enqueueReviewCandidate,
fetchNextReviewItem,
recordReviewLabel,
resetPendingReviewItems,
type ReviewLabel,
type ReviewQueueEntry,
type ReviewReason,
type ReviewConsumerGarnetClient,
} from "./queue.js";
export interface ReviewCliOptions {
consumerName?: string | undefined;
exportLabeled?: string | undefined;
limit?: number | undefined;
reason?: ReviewReason | undefined;
resetPending?: boolean | undefined;
}
function getReviewerId(): string {
const configured = process.env["REVIEWER_ID"]?.trim();
if (configured) {
return configured;
}
try {
return execFileSync("git", ["config", "user.email"], {
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
}).trim() || "unknown";
} catch {
return "unknown";
}
}
function formatLlmJudge(entry: ReviewQueueEntry): string {
if (entry.llmJudge.skipped) {
return `skipped=${entry.llmJudge.reason}`;
}
return [
`overall=${entry.llmJudge.scores.overall}`,
`friendliness=${entry.llmJudge.scores.friendliness}`,
`accuracy=${entry.llmJudge.scores.accuracy}`,
`concision=${entry.llmJudge.scores.concision}`,
`scope=${entry.llmJudge.scores.scope}`,
`rationale=${entry.llmJudge.rationale}`,
].join(" ");
}
function formatProcess(entry: ReviewQueueEntry): string {
const steps = entry.process?.steps;
if (!steps) {
return "n/a";
}
return [
steps.classify ? "classify" : null,
steps.derive ? "derive" : null,
steps.plan ? "plan" : null,
...(steps.toolCalls?.map((call) => `tool:${call.output.name}`) ?? []),
steps.respond ? "respond" : null,
steps.critic ? "critic" : null,
steps.validate ? "validate" : null,
]
.filter((value): value is string => Boolean(value))
.join(" -> ");
}
function printReviewItem(
entry: ReviewQueueEntry,
index: number,
limit: number,
): void {
console.info("");
console.info(`[${index}/${limit}] reason: ${entry.reason}`);
console.info(
`chatroom: ${entry.request.chatroomId} / user: ${entry.request.userId} / trigger: ${entry.request.trigger}`,
);
console.info(
`intent: ${entry.classified.intent.kind} (confidence ${entry.classified.confidence}, ${entry.classified.source})`,
);
console.info(
`tier: ${entry.derived.tier} / phase: ${entry.derived.phase.kind} / responseMode: ${entry.derived.responseMode}`,
);
console.info("");
console.info("[user message]");
console.info(entry.request.userMessage ?? "(no user message)");
console.info("");
console.info("[assistant response]");
console.info(entry.response.text);
console.info("");
console.info(`[llm judge] ${formatLlmJudge(entry)}`);
console.info(`[validator] status=${entry.validation.status} reasons=${entry.validation.reasons.join(",") || "-"}`);
console.info(`[process] ${formatProcess(entry)}`);
console.info("");
}
async function askVerdict(
rl: ReturnType<typeof createInterface>,
): Promise<"fail" | "pass" | "quit" | "skip"> {
while (true) {
const answer = (await rl.question("verdict (p=pass / f=fail / s=skip / q=quit) > "))
.trim()
.toLowerCase();
if (answer === "p") return "pass";
if (answer === "f") return "fail";
if (answer === "s") return "skip";
if (answer === "q") return "quit";
}
}
async function askHumanScore(
rl: ReturnType<typeof createInterface>,
): Promise<1 | 2 | 3 | 4 | 5> {
while (true) {
const answer = Number((await rl.question("humanScore (1-5) > ")).trim());
if (answer === 1 || answer === 2 || answer === 3 || answer === 4 || answer === 5) {
return answer;
}
}
}
async function askNonEmpty(
rl: ReturnType<typeof createInterface>,
prompt: string,
): Promise<string> {
while (true) {
const answer = (await rl.question(prompt)).trim();
if (answer) {
return answer;
}
}
}
async function exportLabels(
labels: ReviewLabel[],
outputPath: string,
): Promise<string> {
const resolved = resolve(process.cwd(), outputPath);
await mkdir(dirname(resolved), { recursive: true });
await writeFile(
resolved,
labels.map((label) => JSON.stringify(label)).join("\n") + "\n",
"utf8",
);
return resolved;
}
function isReviewReason(value: string | undefined): value is ReviewReason {
return (
value === "low-score" ||
value === "rare-slice" ||
value === "rule-llm-disagree" ||
value === "validator-soft-warn"
);
}
export function parseReviewReason(value: string | undefined): ReviewReason | undefined {
return isReviewReason(value) ? value : undefined;
}
export async function runReviewCli(
garnetClient: Pick<
GarnetClient,
| "acknowledgeStream"
| "appendStream"
| "claimPendingStream"
| "createConsumerGroup"
| "getPendingStreamRange"
| "readGroup"
>,
options?: ReviewCliOptions,
): Promise<{ labeled: number; skipped: number }> {
const rl = createInterface({ input, output });
const limit = options?.limit ?? 20;
const consumerName = options?.consumerName ?? `review-cli-${process.pid}`;
const reviewedBy = getReviewerId();
const labels: ReviewLabel[] = [];
let skipped = 0;
let mismatchAttempts = 0;
try {
if (options?.resetPending) {
const result = await resetPendingReviewItems(garnetClient, {
consumerName,
nowMs: Date.now(),
});
console.info(`[eval] resetPending=${result.resetCount}`);
}
for (let index = 1; index <= limit; index++) {
const item = await fetchNextReviewItem(garnetClient as ReviewConsumerGarnetClient, {
consumerName,
blockMs: 5_000,
});
if (!item) {
break;
}
if (options?.reason && item.entry.reason !== options.reason) {
await enqueueReviewCandidate(garnetClient, {
...item.entry,
enqueuedAt: new Date().toISOString(),
});
await acknowledgeReviewItem(garnetClient, item.streamId);
mismatchAttempts += 1;
if (mismatchAttempts >= limit * 5) {
break;
}
index -= 1;
continue;
}
printReviewItem(item.entry, index, limit);
const verdict = await askVerdict(rl);
if (verdict === "quit") {
break;
}
if (verdict === "skip") {
await acknowledgeReviewItem(garnetClient, item.streamId);
skipped += 1;
continue;
}
const humanScore = await askHumanScore(rl);
const rationale = await askNonEmpty(rl, "rationale > ");
const betterResponse = (await rl.question("better (optional) > ")).trim();
const label: ReviewLabel = {
evalStreamId: item.entry.evalStreamId,
verdict,
humanScore,
rationale,
reviewedBy,
reviewedAt: new Date().toISOString(),
};
if (betterResponse) {
label.betterResponse = betterResponse;
}
await recordReviewLabel(garnetClient, {
consumerName,
label,
streamId: item.streamId,
});
labels.push(label);
}
} finally {
rl.close();
}
if (options?.exportLabeled && labels.length > 0) {
const outputPath = await exportLabels(labels, options.exportLabeled);
console.info(`[eval] exported labeled=${labels.length} output=${outputPath}`);
}
return {
labeled: labels.length,
skipped,
};
}