Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,269 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import {
|
||||
acknowledgeReviewItem,
|
||||
enqueueReviewCandidate,
|
||||
fetchNextReviewItem,
|
||||
recordReviewLabel,
|
||||
resetPendingReviewItems,
|
||||
type ReviewLabel,
|
||||
type ReviewQueueEntry,
|
||||
type ReviewReason,
|
||||
type ReviewConsumerGarnetClient,
|
||||
} from "./queue.js";
|
||||
|
||||
export interface ReviewCliOptions {
|
||||
consumerName?: string | undefined;
|
||||
exportLabeled?: string | undefined;
|
||||
limit?: number | undefined;
|
||||
reason?: ReviewReason | undefined;
|
||||
resetPending?: boolean | undefined;
|
||||
}
|
||||
|
||||
function getReviewerId(): string {
|
||||
const configured = process.env["REVIEWER_ID"]?.trim();
|
||||
if (configured) {
|
||||
return configured;
|
||||
}
|
||||
|
||||
try {
|
||||
return execFileSync("git", ["config", "user.email"], {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
}).trim() || "unknown";
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
function formatLlmJudge(entry: ReviewQueueEntry): string {
|
||||
if (entry.llmJudge.skipped) {
|
||||
return `skipped=${entry.llmJudge.reason}`;
|
||||
}
|
||||
|
||||
return [
|
||||
`overall=${entry.llmJudge.scores.overall}`,
|
||||
`friendliness=${entry.llmJudge.scores.friendliness}`,
|
||||
`accuracy=${entry.llmJudge.scores.accuracy}`,
|
||||
`concision=${entry.llmJudge.scores.concision}`,
|
||||
`scope=${entry.llmJudge.scores.scope}`,
|
||||
`rationale=${entry.llmJudge.rationale}`,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function formatProcess(entry: ReviewQueueEntry): string {
|
||||
const steps = entry.process?.steps;
|
||||
if (!steps) {
|
||||
return "n/a";
|
||||
}
|
||||
|
||||
return [
|
||||
steps.classify ? "classify" : null,
|
||||
steps.derive ? "derive" : null,
|
||||
steps.plan ? "plan" : null,
|
||||
...(steps.toolCalls?.map((call) => `tool:${call.output.name}`) ?? []),
|
||||
steps.respond ? "respond" : null,
|
||||
steps.critic ? "critic" : null,
|
||||
steps.validate ? "validate" : null,
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join(" -> ");
|
||||
}
|
||||
|
||||
function printReviewItem(
|
||||
entry: ReviewQueueEntry,
|
||||
index: number,
|
||||
limit: number,
|
||||
): void {
|
||||
console.info("");
|
||||
console.info(`[${index}/${limit}] reason: ${entry.reason}`);
|
||||
console.info(
|
||||
`chatroom: ${entry.request.chatroomId} / user: ${entry.request.userId} / trigger: ${entry.request.trigger}`,
|
||||
);
|
||||
console.info(
|
||||
`intent: ${entry.classified.intent.kind} (confidence ${entry.classified.confidence}, ${entry.classified.source})`,
|
||||
);
|
||||
console.info(
|
||||
`tier: ${entry.derived.tier} / phase: ${entry.derived.phase.kind} / responseMode: ${entry.derived.responseMode}`,
|
||||
);
|
||||
console.info("");
|
||||
console.info("[user message]");
|
||||
console.info(entry.request.userMessage ?? "(no user message)");
|
||||
console.info("");
|
||||
console.info("[assistant response]");
|
||||
console.info(entry.response.text);
|
||||
console.info("");
|
||||
console.info(`[llm judge] ${formatLlmJudge(entry)}`);
|
||||
console.info(`[validator] status=${entry.validation.status} reasons=${entry.validation.reasons.join(",") || "-"}`);
|
||||
console.info(`[process] ${formatProcess(entry)}`);
|
||||
console.info("");
|
||||
}
|
||||
|
||||
async function askVerdict(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
): Promise<"fail" | "pass" | "quit" | "skip"> {
|
||||
while (true) {
|
||||
const answer = (await rl.question("verdict (p=pass / f=fail / s=skip / q=quit) > "))
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (answer === "p") return "pass";
|
||||
if (answer === "f") return "fail";
|
||||
if (answer === "s") return "skip";
|
||||
if (answer === "q") return "quit";
|
||||
}
|
||||
}
|
||||
|
||||
async function askHumanScore(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
): Promise<1 | 2 | 3 | 4 | 5> {
|
||||
while (true) {
|
||||
const answer = Number((await rl.question("humanScore (1-5) > ")).trim());
|
||||
if (answer === 1 || answer === 2 || answer === 3 || answer === 4 || answer === 5) {
|
||||
return answer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function askNonEmpty(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
prompt: string,
|
||||
): Promise<string> {
|
||||
while (true) {
|
||||
const answer = (await rl.question(prompt)).trim();
|
||||
if (answer) {
|
||||
return answer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function exportLabels(
|
||||
labels: ReviewLabel[],
|
||||
outputPath: string,
|
||||
): Promise<string> {
|
||||
const resolved = resolve(process.cwd(), outputPath);
|
||||
await mkdir(dirname(resolved), { recursive: true });
|
||||
await writeFile(
|
||||
resolved,
|
||||
labels.map((label) => JSON.stringify(label)).join("\n") + "\n",
|
||||
"utf8",
|
||||
);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function isReviewReason(value: string | undefined): value is ReviewReason {
|
||||
return (
|
||||
value === "low-score" ||
|
||||
value === "rare-slice" ||
|
||||
value === "rule-llm-disagree" ||
|
||||
value === "validator-soft-warn"
|
||||
);
|
||||
}
|
||||
|
||||
export function parseReviewReason(value: string | undefined): ReviewReason | undefined {
|
||||
return isReviewReason(value) ? value : undefined;
|
||||
}
|
||||
|
||||
export async function runReviewCli(
|
||||
garnetClient: Pick<
|
||||
GarnetClient,
|
||||
| "acknowledgeStream"
|
||||
| "appendStream"
|
||||
| "claimPendingStream"
|
||||
| "createConsumerGroup"
|
||||
| "getPendingStreamRange"
|
||||
| "readGroup"
|
||||
>,
|
||||
options?: ReviewCliOptions,
|
||||
): Promise<{ labeled: number; skipped: number }> {
|
||||
const rl = createInterface({ input, output });
|
||||
const limit = options?.limit ?? 20;
|
||||
const consumerName = options?.consumerName ?? `review-cli-${process.pid}`;
|
||||
const reviewedBy = getReviewerId();
|
||||
const labels: ReviewLabel[] = [];
|
||||
let skipped = 0;
|
||||
let mismatchAttempts = 0;
|
||||
|
||||
try {
|
||||
if (options?.resetPending) {
|
||||
const result = await resetPendingReviewItems(garnetClient, {
|
||||
consumerName,
|
||||
nowMs: Date.now(),
|
||||
});
|
||||
console.info(`[eval] resetPending=${result.resetCount}`);
|
||||
}
|
||||
|
||||
for (let index = 1; index <= limit; index++) {
|
||||
const item = await fetchNextReviewItem(garnetClient as ReviewConsumerGarnetClient, {
|
||||
consumerName,
|
||||
blockMs: 5_000,
|
||||
});
|
||||
if (!item) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (options?.reason && item.entry.reason !== options.reason) {
|
||||
await enqueueReviewCandidate(garnetClient, {
|
||||
...item.entry,
|
||||
enqueuedAt: new Date().toISOString(),
|
||||
});
|
||||
await acknowledgeReviewItem(garnetClient, item.streamId);
|
||||
mismatchAttempts += 1;
|
||||
if (mismatchAttempts >= limit * 5) {
|
||||
break;
|
||||
}
|
||||
index -= 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
printReviewItem(item.entry, index, limit);
|
||||
const verdict = await askVerdict(rl);
|
||||
if (verdict === "quit") {
|
||||
break;
|
||||
}
|
||||
if (verdict === "skip") {
|
||||
await acknowledgeReviewItem(garnetClient, item.streamId);
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const humanScore = await askHumanScore(rl);
|
||||
const rationale = await askNonEmpty(rl, "rationale > ");
|
||||
const betterResponse = (await rl.question("better (optional) > ")).trim();
|
||||
const label: ReviewLabel = {
|
||||
evalStreamId: item.entry.evalStreamId,
|
||||
verdict,
|
||||
humanScore,
|
||||
rationale,
|
||||
reviewedBy,
|
||||
reviewedAt: new Date().toISOString(),
|
||||
};
|
||||
if (betterResponse) {
|
||||
label.betterResponse = betterResponse;
|
||||
}
|
||||
|
||||
await recordReviewLabel(garnetClient, {
|
||||
consumerName,
|
||||
label,
|
||||
streamId: item.streamId,
|
||||
});
|
||||
labels.push(label);
|
||||
}
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
|
||||
if (options?.exportLabeled && labels.length > 0) {
|
||||
const outputPath = await exportLabels(labels, options.exportLabeled);
|
||||
console.info(`[eval] exported labeled=${labels.length} output=${outputPath}`);
|
||||
}
|
||||
|
||||
return {
|
||||
labeled: labels.length,
|
||||
skipped,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user