Make eval pipeline skill-aware and validate SKILL manifests

This commit is contained in:
dahyeon committed 2026-05-18 16:26:31 -07:00
1 parent 12e731a428
commit 54bc295ead
35 files changed
+1280 -499

No files matched your search

+100 -41
View File
@@ -12,12 +12,6 @@ import { summarizeConversation } from "@conv/summarizer";
import type { LlmClient } from "@llm/llm-client";
import { getLogger } from "@obs/logger";
import type { AlertSender } from "@obs/alert";
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ConversationStore } from "@conv/conversation-store";
import {
getAlertThresholdDriftPct,
@@ -49,13 +43,14 @@ import {
type MongoConnectionLike,
} from "./archive.js";
import { computeDriftReports } from "./drift.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
import { writeProcessAttributions } from "./judges/attribution.js";
import { loadPrmModel } from "./judges/prm.js";
import {
judgeProcessRule,
judgeProcessWithLlm,
} from "./judges/process-judge.js";
import { getSkillJudgeConfig } from "./judges/skill-config.js";
import {
appendCandidatesToJsonl,
selectPromotionCandidates,
@@ -72,6 +67,7 @@ import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalDerivedSnapshot,
EvalProcessTraceSnapshot,
ProcessJudgeResult,
} from "./types.js";
@@ -100,13 +96,15 @@ const STOP_TIMEOUT_MS = 10_000;
type BackgroundJobType = "shadow-eval" | "summarize";
interface ShadowEvalJob {
classified: ClassifiedIntent;
classified: unknown;
createdAt: string;
derived: DerivedState;
derived: unknown;
latencyMs?: number | undefined;
process?: ProcessTraceSnapshot | undefined;
process?: EvalProcessTraceSnapshot | undefined;
request: AgentRequest;
response: AgentResponse;
skill: string;
version: number;
}
interface EvalObservation {
@@ -168,26 +166,75 @@ function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
return JSON.parse(value) as T;
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function getStringField(
record: Record<string, unknown>,
key: string,
): string | null {
const value = record[key];
return typeof value === "string" && value.trim() ? value : null;
}
function getNestedString(
record: Record<string, unknown>,
first: string,
second: string,
): string | null {
return getStringField(asRecord(record[first]), second);
}
function parseTier(value: unknown): 0 | 1 | 2 {
const parsed = typeof value === "number" ? value : Number(value);
return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0;
}
function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] {
return value === "skip" ||
value === "template" ||
value === "llm" ||
value === "llm-with-tools"
? value
: "llm";
}
function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot {
const record = asRecord(derived);
const intent = getStringField(record, "intent") ??
getNestedString(record, "intent", "kind") ??
"unknown";
const phase = getStringField(record, "phase") ??
getNestedString(record, "phase", "kind") ??
"unknown";
const abstentionScore = record["abstentionScore"];
const suggestedMaxTokens = record["suggestedMaxTokens"];
return {
...(derived.abstentionScore !== undefined
? { abstentionScore: derived.abstentionScore }
...(typeof abstentionScore === "number"
? { abstentionScore }
: {}),
allowShortResponse: derived.allowShortResponse,
intent: derived.intent.kind,
phase: derived.phase.kind,
responseMode: derived.responseMode,
...(derived.routingDecision
? { routingDecision: derived.routingDecision }
allowShortResponse: record["allowShortResponse"] === true,
intent,
phase,
responseMode: parseResponseMode(record["responseMode"]),
...(record["routingDecision"]
? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] }
: {}),
shouldRespond: derived.shouldRespond,
suggestedMaxTokens: derived.suggestedMaxTokens,
tier: derived.tier,
shouldRespond: record["shouldRespond"] !== false,
suggestedMaxTokens:
typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens)
? suggestedMaxTokens
: 192,
tier: parseTier(record["tier"]),
};
}
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined):
| NonNullable<EvalProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
| undefined {
return process?.steps.classify?.output.abstention;
}
@@ -208,17 +255,17 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
message.message["input"],
"input",
);
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
const process = parseOptionalJsonField<EvalProcessTraceSnapshot>(
message.message["process"],
);
const job: ShadowEvalJob = {
classified: parseJsonField<ClassifiedIntent>(
classified: parseJsonField<unknown>(
message.message["classified"],
"classified",
),
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
derived: parseJsonField<DerivedState>(
derived: parseJsonField<unknown>(
message.message["derived"],
"derived",
),
@@ -228,6 +275,8 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
message.message["output"],
"output",
),
skill: message.message["skill"]?.trim() || input.request.skill || "onboarding",
version: Number(message.message["version"] ?? 1),
};
if (process) {
@@ -500,6 +549,8 @@ export class ShadowWorker {
private async processShadowEval(message: StreamEntry): Promise<void> {
const job = parseShadowEvalJob(message);
const judgeConfig = getSkillJudgeConfig(job.skill);
const derivedSnapshot = toDerivedSnapshot(job.derived);
const abstention = getTraceAbstention(job.process);
const responseTextForValidation =
job.response.text.trim()
@@ -507,18 +558,19 @@ export class ShadowWorker {
: abstention?.action.kind === "no-response"
? "[NO_RESPONSE]"
: job.response.text;
const validation = validateOnboardingResponse(
responseTextForValidation,
job.derived,
{
allowShortResponse: job.derived.allowShortResponse,
},
);
const llmJudge = await judgeOnboardingWithLlm(
const validation = judgeConfig.validator
? judgeConfig.validator(responseTextForValidation, job.derived)
: {
reasons: [],
status: "pass" as const,
text: responseTextForValidation,
};
const llmJudge = await judgeSkillWithLlm(
this.llmClient,
job.skill,
toEvalCase(job, message.id),
job.response.text,
toDerivedSnapshot(job.derived),
derivedSnapshot,
);
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
const llmJudgeErrored =
@@ -541,10 +593,12 @@ export class ShadowWorker {
requestId: job.request.requestId,
chatroomId: job.request.chatroomId,
userId: job.request.userId,
tier: String(job.derived.tier),
intent: job.derived.intent.kind,
phase: job.derived.phase.kind,
responseMode: job.derived.responseMode,
skill: job.skill,
version: String(Math.max(2, job.version)),
tier: String(derivedSnapshot.tier),
intent: derivedSnapshot.intent,
phase: derivedSnapshot.phase,
responseMode: derivedSnapshot.responseMode,
validatorStatus: validation.status,
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
llmOverall: llmOverall === null ? "" : String(llmOverall),
@@ -589,7 +643,12 @@ export class ShadowWorker {
throw new Error("Missing process trace");
}
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
const judgeConfig = getSkillJudgeConfig(job.skill);
const ruleResult = judgeProcessRule(
toEvalCase(job, messageId).expected,
job.process,
judgeConfig.toolRegistryFactory(),
);
if (
!getEvalProcessLlmJudgeEnabled() ||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()