270 lines
7.5 KiB
TypeScript
270 lines
7.5 KiB
TypeScript
import { execFileSync } from "node:child_process";
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import { dirname, resolve } from "node:path";
|
|
import { createInterface } from "node:readline/promises";
|
|
import { stdin as input, stdout as output } from "node:process";
|
|
|
|
import type { GarnetClient } from "@common/garnet";
|
|
import {
|
|
acknowledgeReviewItem,
|
|
enqueueReviewCandidate,
|
|
fetchNextReviewItem,
|
|
recordReviewLabel,
|
|
resetPendingReviewItems,
|
|
type ReviewLabel,
|
|
type ReviewQueueEntry,
|
|
type ReviewReason,
|
|
type ReviewConsumerGarnetClient,
|
|
} from "./queue.js";
|
|
|
|
export interface ReviewCliOptions {
|
|
consumerName?: string | undefined;
|
|
exportLabeled?: string | undefined;
|
|
limit?: number | undefined;
|
|
reason?: ReviewReason | undefined;
|
|
resetPending?: boolean | undefined;
|
|
}
|
|
|
|
function getReviewerId(): string {
|
|
const configured = process.env["REVIEWER_ID"]?.trim();
|
|
if (configured) {
|
|
return configured;
|
|
}
|
|
|
|
try {
|
|
return execFileSync("git", ["config", "user.email"], {
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "ignore"],
|
|
}).trim() || "unknown";
|
|
} catch {
|
|
return "unknown";
|
|
}
|
|
}
|
|
|
|
function formatLlmJudge(entry: ReviewQueueEntry): string {
|
|
if (entry.llmJudge.skipped) {
|
|
return `skipped=${entry.llmJudge.reason}`;
|
|
}
|
|
|
|
return [
|
|
`overall=${entry.llmJudge.scores.overall}`,
|
|
`friendliness=${entry.llmJudge.scores.friendliness}`,
|
|
`accuracy=${entry.llmJudge.scores.accuracy}`,
|
|
`concision=${entry.llmJudge.scores.concision}`,
|
|
`scope=${entry.llmJudge.scores.scope}`,
|
|
`rationale=${entry.llmJudge.rationale}`,
|
|
].join(" ");
|
|
}
|
|
|
|
function formatProcess(entry: ReviewQueueEntry): string {
|
|
const steps = entry.process?.steps;
|
|
if (!steps) {
|
|
return "n/a";
|
|
}
|
|
|
|
return [
|
|
steps.classify ? "classify" : null,
|
|
steps.derive ? "derive" : null,
|
|
steps.plan ? "plan" : null,
|
|
...(steps.toolCalls?.map((call) => `tool:${call.output.name}`) ?? []),
|
|
steps.respond ? "respond" : null,
|
|
steps.critic ? "critic" : null,
|
|
steps.validate ? "validate" : null,
|
|
]
|
|
.filter((value): value is string => Boolean(value))
|
|
.join(" -> ");
|
|
}
|
|
|
|
function printReviewItem(
|
|
entry: ReviewQueueEntry,
|
|
index: number,
|
|
limit: number,
|
|
): void {
|
|
console.info("");
|
|
console.info(`[${index}/${limit}] reason: ${entry.reason}`);
|
|
console.info(
|
|
`chatroom: ${entry.request.chatroomId} / user: ${entry.request.userId} / trigger: ${entry.request.trigger}`,
|
|
);
|
|
console.info(
|
|
`intent: ${entry.classified.intent.kind} (confidence ${entry.classified.confidence}, ${entry.classified.source})`,
|
|
);
|
|
console.info(
|
|
`tier: ${entry.derived.tier} / phase: ${entry.derived.phase.kind} / responseMode: ${entry.derived.responseMode}`,
|
|
);
|
|
console.info("");
|
|
console.info("[user message]");
|
|
console.info(entry.request.userMessage ?? "(no user message)");
|
|
console.info("");
|
|
console.info("[assistant response]");
|
|
console.info(entry.response.text);
|
|
console.info("");
|
|
console.info(`[llm judge] ${formatLlmJudge(entry)}`);
|
|
console.info(`[validator] status=${entry.validation.status} reasons=${entry.validation.reasons.join(",") || "-"}`);
|
|
console.info(`[process] ${formatProcess(entry)}`);
|
|
console.info("");
|
|
}
|
|
|
|
async function askVerdict(
|
|
rl: ReturnType<typeof createInterface>,
|
|
): Promise<"fail" | "pass" | "quit" | "skip"> {
|
|
while (true) {
|
|
const answer = (await rl.question("verdict (p=pass / f=fail / s=skip / q=quit) > "))
|
|
.trim()
|
|
.toLowerCase();
|
|
if (answer === "p") return "pass";
|
|
if (answer === "f") return "fail";
|
|
if (answer === "s") return "skip";
|
|
if (answer === "q") return "quit";
|
|
}
|
|
}
|
|
|
|
async function askHumanScore(
|
|
rl: ReturnType<typeof createInterface>,
|
|
): Promise<1 | 2 | 3 | 4 | 5> {
|
|
while (true) {
|
|
const answer = Number((await rl.question("humanScore (1-5) > ")).trim());
|
|
if (answer === 1 || answer === 2 || answer === 3 || answer === 4 || answer === 5) {
|
|
return answer;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function askNonEmpty(
|
|
rl: ReturnType<typeof createInterface>,
|
|
prompt: string,
|
|
): Promise<string> {
|
|
while (true) {
|
|
const answer = (await rl.question(prompt)).trim();
|
|
if (answer) {
|
|
return answer;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function exportLabels(
|
|
labels: ReviewLabel[],
|
|
outputPath: string,
|
|
): Promise<string> {
|
|
const resolved = resolve(process.cwd(), outputPath);
|
|
await mkdir(dirname(resolved), { recursive: true });
|
|
await writeFile(
|
|
resolved,
|
|
labels.map((label) => JSON.stringify(label)).join("\n") + "\n",
|
|
"utf8",
|
|
);
|
|
return resolved;
|
|
}
|
|
|
|
function isReviewReason(value: string | undefined): value is ReviewReason {
|
|
return (
|
|
value === "low-score" ||
|
|
value === "rare-slice" ||
|
|
value === "rule-llm-disagree" ||
|
|
value === "validator-soft-warn"
|
|
);
|
|
}
|
|
|
|
export function parseReviewReason(value: string | undefined): ReviewReason | undefined {
|
|
return isReviewReason(value) ? value : undefined;
|
|
}
|
|
|
|
export async function runReviewCli(
|
|
garnetClient: Pick<
|
|
GarnetClient,
|
|
| "acknowledgeStream"
|
|
| "appendStream"
|
|
| "claimPendingStream"
|
|
| "createConsumerGroup"
|
|
| "getPendingStreamRange"
|
|
| "readGroup"
|
|
>,
|
|
options?: ReviewCliOptions,
|
|
): Promise<{ labeled: number; skipped: number }> {
|
|
const rl = createInterface({ input, output });
|
|
const limit = options?.limit ?? 20;
|
|
const consumerName = options?.consumerName ?? `review-cli-${process.pid}`;
|
|
const reviewedBy = getReviewerId();
|
|
const labels: ReviewLabel[] = [];
|
|
let skipped = 0;
|
|
let mismatchAttempts = 0;
|
|
|
|
try {
|
|
if (options?.resetPending) {
|
|
const result = await resetPendingReviewItems(garnetClient, {
|
|
consumerName,
|
|
nowMs: Date.now(),
|
|
});
|
|
console.info(`[eval] resetPending=${result.resetCount}`);
|
|
}
|
|
|
|
for (let index = 1; index <= limit; index++) {
|
|
const item = await fetchNextReviewItem(garnetClient as ReviewConsumerGarnetClient, {
|
|
consumerName,
|
|
blockMs: 5_000,
|
|
});
|
|
if (!item) {
|
|
break;
|
|
}
|
|
|
|
if (options?.reason && item.entry.reason !== options.reason) {
|
|
await enqueueReviewCandidate(garnetClient, {
|
|
...item.entry,
|
|
enqueuedAt: new Date().toISOString(),
|
|
});
|
|
await acknowledgeReviewItem(garnetClient, item.streamId);
|
|
mismatchAttempts += 1;
|
|
if (mismatchAttempts >= limit * 5) {
|
|
break;
|
|
}
|
|
index -= 1;
|
|
continue;
|
|
}
|
|
|
|
printReviewItem(item.entry, index, limit);
|
|
const verdict = await askVerdict(rl);
|
|
if (verdict === "quit") {
|
|
break;
|
|
}
|
|
if (verdict === "skip") {
|
|
await acknowledgeReviewItem(garnetClient, item.streamId);
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
const humanScore = await askHumanScore(rl);
|
|
const rationale = await askNonEmpty(rl, "rationale > ");
|
|
const betterResponse = (await rl.question("better (optional) > ")).trim();
|
|
const label: ReviewLabel = {
|
|
evalStreamId: item.entry.evalStreamId,
|
|
verdict,
|
|
humanScore,
|
|
rationale,
|
|
reviewedBy,
|
|
reviewedAt: new Date().toISOString(),
|
|
};
|
|
if (betterResponse) {
|
|
label.betterResponse = betterResponse;
|
|
}
|
|
|
|
await recordReviewLabel(garnetClient, {
|
|
consumerName,
|
|
label,
|
|
streamId: item.streamId,
|
|
});
|
|
labels.push(label);
|
|
}
|
|
} finally {
|
|
rl.close();
|
|
}
|
|
|
|
if (options?.exportLabeled && labels.length > 0) {
|
|
const outputPath = await exportLabels(labels, options.exportLabeled);
|
|
console.info(`[eval] exported labeled=${labels.length} output=${outputPath}`);
|
|
}
|
|
|
|
return {
|
|
labeled: labels.length,
|
|
skipped,
|
|
};
|
|
}
|