Make eval pipeline skill-aware and validate SKILL manifests
This commit is contained in:
1 parent
12e731a428
commit
54bc295ead
35 files changed
+1280
-499
No files matched your search
+100
-41
@@ -12,12 +12,6 @@ import { summarizeConversation } from "@conv/summarizer";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { AlertSender } from "@obs/alert";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import {
|
||||
getAlertThresholdDriftPct,
|
||||
@@ -49,13 +43,14 @@ import {
|
||||
type MongoConnectionLike,
|
||||
} from "./archive.js";
|
||||
import { computeDriftReports } from "./drift.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||
import { writeProcessAttributions } from "./judges/attribution.js";
|
||||
import { loadPrmModel } from "./judges/prm.js";
|
||||
import {
|
||||
judgeProcessRule,
|
||||
judgeProcessWithLlm,
|
||||
} from "./judges/process-judge.js";
|
||||
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||
import {
|
||||
appendCandidatesToJsonl,
|
||||
selectPromotionCandidates,
|
||||
@@ -72,6 +67,7 @@ import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
EvalProcessTraceSnapshot,
|
||||
ProcessJudgeResult,
|
||||
} from "./types.js";
|
||||
|
||||
@@ -100,13 +96,15 @@ const STOP_TIMEOUT_MS = 10_000;
|
||||
type BackgroundJobType = "shadow-eval" | "summarize";
|
||||
|
||||
interface ShadowEvalJob {
|
||||
classified: ClassifiedIntent;
|
||||
classified: unknown;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
derived: unknown;
|
||||
latencyMs?: number | undefined;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
process?: EvalProcessTraceSnapshot | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: string;
|
||||
version: number;
|
||||
}
|
||||
|
||||
interface EvalObservation {
|
||||
@@ -168,26 +166,75 @@ function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function getStringField(
|
||||
record: Record<string, unknown>,
|
||||
key: string,
|
||||
): string | null {
|
||||
const value = record[key];
|
||||
return typeof value === "string" && value.trim() ? value : null;
|
||||
}
|
||||
|
||||
function getNestedString(
|
||||
record: Record<string, unknown>,
|
||||
first: string,
|
||||
second: string,
|
||||
): string | null {
|
||||
return getStringField(asRecord(record[first]), second);
|
||||
}
|
||||
|
||||
function parseTier(value: unknown): 0 | 1 | 2 {
|
||||
const parsed = typeof value === "number" ? value : Number(value);
|
||||
return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0;
|
||||
}
|
||||
|
||||
function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] {
|
||||
return value === "skip" ||
|
||||
value === "template" ||
|
||||
value === "llm" ||
|
||||
value === "llm-with-tools"
|
||||
? value
|
||||
: "llm";
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot {
|
||||
const record = asRecord(derived);
|
||||
const intent = getStringField(record, "intent") ??
|
||||
getNestedString(record, "intent", "kind") ??
|
||||
"unknown";
|
||||
const phase = getStringField(record, "phase") ??
|
||||
getNestedString(record, "phase", "kind") ??
|
||||
"unknown";
|
||||
const abstentionScore = record["abstentionScore"];
|
||||
const suggestedMaxTokens = record["suggestedMaxTokens"];
|
||||
|
||||
return {
|
||||
...(derived.abstentionScore !== undefined
|
||||
? { abstentionScore: derived.abstentionScore }
|
||||
...(typeof abstentionScore === "number"
|
||||
? { abstentionScore }
|
||||
: {}),
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
...(derived.routingDecision
|
||||
? { routingDecision: derived.routingDecision }
|
||||
allowShortResponse: record["allowShortResponse"] === true,
|
||||
intent,
|
||||
phase,
|
||||
responseMode: parseResponseMode(record["responseMode"]),
|
||||
...(record["routingDecision"]
|
||||
? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] }
|
||||
: {}),
|
||||
shouldRespond: derived.shouldRespond,
|
||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
||||
tier: derived.tier,
|
||||
shouldRespond: record["shouldRespond"] !== false,
|
||||
suggestedMaxTokens:
|
||||
typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens)
|
||||
? suggestedMaxTokens
|
||||
: 192,
|
||||
tier: parseTier(record["tier"]),
|
||||
};
|
||||
}
|
||||
|
||||
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
|
||||
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
||||
function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined):
|
||||
| NonNullable<EvalProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
||||
| undefined {
|
||||
return process?.steps.classify?.output.abstention;
|
||||
}
|
||||
@@ -208,17 +255,17 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
message.message["input"],
|
||||
"input",
|
||||
);
|
||||
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
|
||||
const process = parseOptionalJsonField<EvalProcessTraceSnapshot>(
|
||||
message.message["process"],
|
||||
);
|
||||
|
||||
const job: ShadowEvalJob = {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
classified: parseJsonField<unknown>(
|
||||
message.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
||||
derived: parseJsonField<DerivedState>(
|
||||
derived: parseJsonField<unknown>(
|
||||
message.message["derived"],
|
||||
"derived",
|
||||
),
|
||||
@@ -228,6 +275,8 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
message.message["output"],
|
||||
"output",
|
||||
),
|
||||
skill: message.message["skill"]?.trim() || input.request.skill || "onboarding",
|
||||
version: Number(message.message["version"] ?? 1),
|
||||
};
|
||||
|
||||
if (process) {
|
||||
@@ -500,6 +549,8 @@ export class ShadowWorker {
|
||||
|
||||
private async processShadowEval(message: StreamEntry): Promise<void> {
|
||||
const job = parseShadowEvalJob(message);
|
||||
const judgeConfig = getSkillJudgeConfig(job.skill);
|
||||
const derivedSnapshot = toDerivedSnapshot(job.derived);
|
||||
const abstention = getTraceAbstention(job.process);
|
||||
const responseTextForValidation =
|
||||
job.response.text.trim()
|
||||
@@ -507,18 +558,19 @@ export class ShadowWorker {
|
||||
: abstention?.action.kind === "no-response"
|
||||
? "[NO_RESPONSE]"
|
||||
: job.response.text;
|
||||
const validation = validateOnboardingResponse(
|
||||
responseTextForValidation,
|
||||
job.derived,
|
||||
{
|
||||
allowShortResponse: job.derived.allowShortResponse,
|
||||
},
|
||||
);
|
||||
const llmJudge = await judgeOnboardingWithLlm(
|
||||
const validation = judgeConfig.validator
|
||||
? judgeConfig.validator(responseTextForValidation, job.derived)
|
||||
: {
|
||||
reasons: [],
|
||||
status: "pass" as const,
|
||||
text: responseTextForValidation,
|
||||
};
|
||||
const llmJudge = await judgeSkillWithLlm(
|
||||
this.llmClient,
|
||||
job.skill,
|
||||
toEvalCase(job, message.id),
|
||||
job.response.text,
|
||||
toDerivedSnapshot(job.derived),
|
||||
derivedSnapshot,
|
||||
);
|
||||
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
||||
const llmJudgeErrored =
|
||||
@@ -541,10 +593,12 @@ export class ShadowWorker {
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
phase: job.derived.phase.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
skill: job.skill,
|
||||
version: String(Math.max(2, job.version)),
|
||||
tier: String(derivedSnapshot.tier),
|
||||
intent: derivedSnapshot.intent,
|
||||
phase: derivedSnapshot.phase,
|
||||
responseMode: derivedSnapshot.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall: llmOverall === null ? "" : String(llmOverall),
|
||||
@@ -589,7 +643,12 @@ export class ShadowWorker {
|
||||
throw new Error("Missing process trace");
|
||||
}
|
||||
|
||||
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
|
||||
const judgeConfig = getSkillJudgeConfig(job.skill);
|
||||
const ruleResult = judgeProcessRule(
|
||||
toEvalCase(job, messageId).expected,
|
||||
job.process,
|
||||
judgeConfig.toolRegistryFactory(),
|
||||
);
|
||||
if (
|
||||
!getEvalProcessLlmJudgeEnabled() ||
|
||||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
|
||||
|
||||
Reference in new issue
Block a user