import { execFileSync } from "node:child_process"; import { mkdir, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { createInterface } from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; import type { GarnetClient } from "@common/garnet"; import { acknowledgeReviewItem, enqueueReviewCandidate, fetchNextReviewItem, recordReviewLabel, resetPendingReviewItems, type ReviewLabel, type ReviewQueueEntry, type ReviewReason, type ReviewConsumerGarnetClient, } from "./queue.js"; export interface ReviewCliOptions { consumerName?: string | undefined; exportLabeled?: string | undefined; limit?: number | undefined; reason?: ReviewReason | undefined; resetPending?: boolean | undefined; } function getReviewerId(): string { const configured = process.env["REVIEWER_ID"]?.trim(); if (configured) { return configured; } try { return execFileSync("git", ["config", "user.email"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], }).trim() || "unknown"; } catch { return "unknown"; } } function formatLlmJudge(entry: ReviewQueueEntry): string { if (entry.llmJudge.skipped) { return `skipped=${entry.llmJudge.reason}`; } return [ `overall=${entry.llmJudge.scores.overall}`, `friendliness=${entry.llmJudge.scores.friendliness}`, `accuracy=${entry.llmJudge.scores.accuracy}`, `concision=${entry.llmJudge.scores.concision}`, `scope=${entry.llmJudge.scores.scope}`, `rationale=${entry.llmJudge.rationale}`, ].join(" "); } function formatProcess(entry: ReviewQueueEntry): string { const steps = entry.process?.steps; if (!steps) { return "n/a"; } return [ steps.classify ? "classify" : null, steps.derive ? "derive" : null, steps.plan ? "plan" : null, ...(steps.toolCalls?.map((call) => `tool:${call.output.name}`) ?? []), steps.respond ? "respond" : null, steps.critic ? "critic" : null, steps.validate ? "validate" : null, ] .filter((value): value is string => Boolean(value)) .join(" -> "); } function printReviewItem( entry: ReviewQueueEntry, index: number, limit: number, ): void { console.info(""); console.info(`[${index}/${limit}] reason: ${entry.reason}`); console.info( `chatroom: ${entry.request.chatroomId} / user: ${entry.request.userId} / trigger: ${entry.request.trigger}`, ); console.info( `intent: ${entry.classified.intent.kind} (confidence ${entry.classified.confidence}, ${entry.classified.source})`, ); console.info( `tier: ${entry.derived.tier} / phase: ${entry.derived.phase.kind} / responseMode: ${entry.derived.responseMode}`, ); console.info(""); console.info("[user message]"); console.info(entry.request.userMessage ?? "(no user message)"); console.info(""); console.info("[assistant response]"); console.info(entry.response.text); console.info(""); console.info(`[llm judge] ${formatLlmJudge(entry)}`); console.info(`[validator] status=${entry.validation.status} reasons=${entry.validation.reasons.join(",") || "-"}`); console.info(`[process] ${formatProcess(entry)}`); console.info(""); } async function askVerdict( rl: ReturnType, ): Promise<"fail" | "pass" | "quit" | "skip"> { while (true) { const answer = (await rl.question("verdict (p=pass / f=fail / s=skip / q=quit) > ")) .trim() .toLowerCase(); if (answer === "p") return "pass"; if (answer === "f") return "fail"; if (answer === "s") return "skip"; if (answer === "q") return "quit"; } } async function askHumanScore( rl: ReturnType, ): Promise<1 | 2 | 3 | 4 | 5> { while (true) { const answer = Number((await rl.question("humanScore (1-5) > ")).trim()); if (answer === 1 || answer === 2 || answer === 3 || answer === 4 || answer === 5) { return answer; } } } async function askNonEmpty( rl: ReturnType, prompt: string, ): Promise { while (true) { const answer = (await rl.question(prompt)).trim(); if (answer) { return answer; } } } async function exportLabels( labels: ReviewLabel[], outputPath: string, ): Promise { const resolved = resolve(process.cwd(), outputPath); await mkdir(dirname(resolved), { recursive: true }); await writeFile( resolved, labels.map((label) => JSON.stringify(label)).join("\n") + "\n", "utf8", ); return resolved; } function isReviewReason(value: string | undefined): value is ReviewReason { return ( value === "low-score" || value === "rare-slice" || value === "rule-llm-disagree" || value === "validator-soft-warn" ); } export function parseReviewReason(value: string | undefined): ReviewReason | undefined { return isReviewReason(value) ? value : undefined; } export async function runReviewCli( garnetClient: Pick< GarnetClient, | "acknowledgeStream" | "appendStream" | "claimPendingStream" | "createConsumerGroup" | "getPendingStreamRange" | "readGroup" >, options?: ReviewCliOptions, ): Promise<{ labeled: number; skipped: number }> { const rl = createInterface({ input, output }); const limit = options?.limit ?? 20; const consumerName = options?.consumerName ?? `review-cli-${process.pid}`; const reviewedBy = getReviewerId(); const labels: ReviewLabel[] = []; let skipped = 0; let mismatchAttempts = 0; try { if (options?.resetPending) { const result = await resetPendingReviewItems(garnetClient, { consumerName, nowMs: Date.now(), }); console.info(`[eval] resetPending=${result.resetCount}`); } for (let index = 1; index <= limit; index++) { const item = await fetchNextReviewItem(garnetClient as ReviewConsumerGarnetClient, { consumerName, blockMs: 5_000, }); if (!item) { break; } if (options?.reason && item.entry.reason !== options.reason) { await enqueueReviewCandidate(garnetClient, { ...item.entry, enqueuedAt: new Date().toISOString(), }); await acknowledgeReviewItem(garnetClient, item.streamId); mismatchAttempts += 1; if (mismatchAttempts >= limit * 5) { break; } index -= 1; continue; } printReviewItem(item.entry, index, limit); const verdict = await askVerdict(rl); if (verdict === "quit") { break; } if (verdict === "skip") { await acknowledgeReviewItem(garnetClient, item.streamId); skipped += 1; continue; } const humanScore = await askHumanScore(rl); const rationale = await askNonEmpty(rl, "rationale > "); const betterResponse = (await rl.question("better (optional) > ")).trim(); const label: ReviewLabel = { evalStreamId: item.entry.evalStreamId, verdict, humanScore, rationale, reviewedBy, reviewedAt: new Date().toISOString(), }; if (betterResponse) { label.betterResponse = betterResponse; } await recordReviewLabel(garnetClient, { consumerName, label, streamId: item.streamId, }); labels.push(label); } } finally { rl.close(); } if (options?.exportLabeled && labels.length > 0) { const outputPath = await exportLabels(labels, options.exportLabeled); console.info(`[eval] exported labeled=${labels.length} output=${outputPath}`); } return { labeled: labels.length, skipped, }; }