Make eval pipeline skill-aware and validate SKILL manifests

This commit is contained in:
dahyeon committed 2026-05-18 16:26:31 -07:00
1 parent 12e731a428
commit 54bc295ead
35 files changed
+1280 -499

No files matched your search

+36 -6
View File
@@ -9,6 +9,7 @@ export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor";
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
export interface SliceKey {
skill?: string | undefined;
tier: 0 | 1 | 2;
intent: string;
phase: string;
@@ -111,6 +112,14 @@ function getNestedString(
return typeof value === "string" && value.trim() ? value : null;
}
function getStringField(
record: Record<string, unknown>,
key: string,
): string | null {
const value = record[key];
return typeof value === "string" && value.trim() ? value : null;
}
function extractLlmOverall(message: Record<string, string>): number | null {
const direct = parseFiniteNumber(message["llmOverall"]);
if (direct !== null) {
@@ -177,9 +186,13 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
const intent =
entry.message["intent"]?.trim() ||
getStringField(derived, "intent") ||
getNestedString(derived, "intent", "kind");
const phase =
entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind");
entry.message["phase"]?.trim() ||
getStringField(derived, "phase") ||
getNestedString(derived, "phase", "kind");
const skill = entry.message["skill"]?.trim();
if (tier === null || !intent || !phase) {
return null;
@@ -194,6 +207,7 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
llmOverall: extractLlmOverall(entry.message),
processJudgeScore: extractProcessJudgeScore(entry.message),
slice: {
...(skill ? { skill } : {}),
tier,
intent,
phase,
@@ -341,17 +355,28 @@ function average(sum: number, count: number): number | null {
function sliceKeyFromString(value: string): SliceKey | null {
const parts = value.split(":");
const [tierRaw, intent, phase] = parts;
let skill: string | undefined;
let offset = 0;
let tierRaw = parts[0];
if (parseTier(tierRaw) === null && parts.length >= 4) {
skill = parts[0];
offset = 1;
tierRaw = parts[offset];
}
const intent = parts[offset + 1];
const phase = parts[offset + 2];
const tier = parseTier(tierRaw);
if (tier === null || !intent || !phase) {
return null;
}
const routingDecision = parts.length >= 6 ? parts[3] : undefined;
const abstentionAction = parts.length >= 6 ? parts[4] : parts[3];
const abstentionReason = parts.length >= 6 ? parts[5] : parts[4];
const tail = parts.slice(offset + 3);
const routingDecision = tail.length >= 3 ? tail[0] : undefined;
const abstentionAction = tail.length >= 3 ? tail[1] : tail[0];
const abstentionReason = tail.length >= 3 ? tail[2] : tail[1];
return {
...(skill ? { skill } : {}),
tier,
intent,
phase,
@@ -376,7 +401,12 @@ function sliceKeyFromRedisKey(key: string): SliceKey | null {
}
export function sliceKeyToString(key: SliceKey): string {
const base = `${key.tier}:${key.intent}:${key.phase}`;
const base = [
...(key.skill ? [key.skill] : []),
String(key.tier),
key.intent,
key.phase,
].join(":");
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
return base;
}
+53 -9
View File
@@ -17,17 +17,19 @@ import {
type SliceKey,
type SliceReportRow,
} from "./aggregator.js";
import type { EvalClassifiedSnapshot, EvalDerivedSnapshot } from "./types.js";
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
export interface EvalArchiveEntry {
streamId: string;
skill?: string | undefined;
evaluatedAt: Date;
request: AgentRequest;
response: AgentResponse;
derived: DerivedState;
classified: ClassifiedIntent;
derived: DerivedState | EvalDerivedSnapshot;
classified: ClassifiedIntent | EvalClassifiedSnapshot;
validation: ValidationResult;
llmJudge: LlmJudgeResult;
processJudge?: ProcessJudgeResult | undefined;
@@ -127,6 +129,44 @@ function getAbstention(
};
}
function getStringField(
record: Record<string, unknown>,
key: string,
): string | null {
const value = record[key];
return typeof value === "string" && value.trim() ? value : null;
}
function getNestedString(
record: Record<string, unknown>,
first: string,
second: string,
): string | null {
const nested = asRecord(record[first]);
return getStringField(nested, second);
}
export function getArchiveDerivedIntent(derived: EvalArchiveEntry["derived"]): string {
const record = asRecord(derived);
return getStringField(record, "intent") ??
getNestedString(record, "intent", "kind") ??
"unknown";
}
export function getArchiveDerivedPhase(derived: EvalArchiveEntry["derived"]): string {
const record = asRecord(derived);
return getStringField(record, "phase") ??
getNestedString(record, "phase", "kind") ??
"unknown";
}
export function getArchiveRoutingDecision(
derived: EvalArchiveEntry["derived"],
): string | null {
const record = asRecord(derived);
return getNestedString(record, "routingDecision", "kind");
}
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
if (!value?.trim()) {
return new Date(nowMs);
@@ -164,13 +204,13 @@ function calculatePercentile(values: number[], percentile: number): number | nul
function getSlice(entry: EvalArchiveEntry): SliceKey {
const abstention = getAbstention(entry);
const routingDecision = getArchiveRoutingDecision(entry.derived);
return {
skill: entry.skill || entry.request.skill || "onboarding",
tier: entry.tier,
intent: entry.derived.intent.kind,
phase: entry.derived.phase.kind,
...(entry.derived.routingDecision
? { routingDecision: entry.derived.routingDecision.kind }
: {}),
intent: getArchiveDerivedIntent(entry.derived),
phase: getArchiveDerivedPhase(entry.derived),
...(routingDecision ? { routingDecision } : {}),
...(abstention.action ? { abstentionAction: abstention.action } : {}),
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
};
@@ -192,7 +232,7 @@ function parseEvalArchiveEntry(
entry.message["input"],
"input",
);
const derived = parseJsonField<DerivedState>(
const derived = parseJsonField<DerivedState | EvalDerivedSnapshot>(
entry.message["derived"],
"derived",
);
@@ -208,11 +248,12 @@ function parseEvalArchiveEntry(
const archiveEntry: EvalArchiveEntry = {
streamId: entry.id,
skill: entry.message["skill"]?.trim() || input.request.skill || "onboarding",
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
request: input.request,
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
derived,
classified: parseJsonField<ClassifiedIntent>(
classified: parseJsonField<ClassifiedIntent | EvalClassifiedSnapshot>(
entry.message["classified"],
"classified",
),
@@ -248,9 +289,12 @@ async function ensureEvalArchiveIndexes(
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
),
collection.createIndex({
skill: 1,
tier: 1,
"derived.intent.kind": 1,
"derived.intent": 1,
"derived.phase.kind": 1,
"derived.phase": 1,
evaluatedAt: -1,
}),
]);
+12 -1
View File
@@ -65,6 +65,7 @@ function parseSlice(value: string | undefined): SliceKey {
const tier = Number(fields["tier"]);
const intent = fields["intent"];
const phase = fields["phase"];
const skill = fields["skill"]?.trim();
if (
(tier !== 0 && tier !== 1 && tier !== 2) ||
@@ -77,6 +78,7 @@ function parseSlice(value: string | undefined): SliceKey {
}
return {
...(skill ? { skill } : {}),
tier,
intent,
phase,
@@ -620,9 +622,14 @@ async function main(): Promise<void> {
} = await import("./loop/propose.js");
const mongo = await connectMongoForEval();
try {
const slice = parseSlice(getFlagString(flags, "slice"));
const skill = getFlagString(flags, "skill");
const input = await buildProposalInputFromArchive(
mongo.connection,
parseSlice(getFlagString(flags, "slice")),
{
...slice,
...(skill ? { skill } : {}),
},
{
force: hasFlag(flags, "force"),
limit: parsePositiveInteger(getFlagString(flags, "limit")),
@@ -668,13 +675,16 @@ async function main(): Promise<void> {
const mongo = await connectMongoForEval();
try {
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
const skill = getFlagString(flags, "skill");
const cases = await loadTournamentCases(mongo.connection, {
caseIds: parseCsv(getFlagString(flags, "case-ids")),
limit: maxMatches,
skill,
});
const result = await runTournament(createEvalLlmClient(), {
cases,
maxMatches,
skill,
variantIds,
});
await saveTournamentResult(mongo.connection, result);
@@ -691,6 +701,7 @@ async function main(): Promise<void> {
try {
const results = await queryTournamentResults(mongo.connection, {
limit: parsePositiveInteger(getFlagString(flags, "limit")),
skill: getFlagString(flags, "skill"),
variantId: getFlagString(flags, "variant"),
});
for (const result of results) {
+12 -6
View File
@@ -3,7 +3,12 @@ import type {
EvalArchiveEntry,
MongoConnectionLike,
} from "./archive.js";
import { getEvalArchiveCollection } from "./archive.js";
import {
getArchiveDerivedIntent,
getArchiveDerivedPhase,
getArchiveRoutingDecision,
getEvalArchiveCollection,
} from "./archive.js";
import type { SliceKey } from "./aggregator.js";
const DAY_MS = 24 * 60 * 60 * 1000;
@@ -41,6 +46,7 @@ function average(values: number[]): number | null {
function sliceKeyToString(slice: SliceKey): string {
return [
slice.skill ?? "onboarding",
slice.tier,
slice.intent,
slice.phase,
@@ -51,13 +57,13 @@ function sliceKeyToString(slice: SliceKey): string {
}
function getSlice(entry: EvalArchiveEntry): SliceKey {
const routingDecision = getArchiveRoutingDecision(entry.derived);
return {
skill: entry.skill || entry.request.skill || "onboarding",
tier: entry.tier,
intent: entry.derived.intent.kind,
phase: entry.derived.phase.kind,
...(entry.derived.routingDecision
? { routingDecision: entry.derived.routingDecision.kind }
: {}),
intent: getArchiveDerivedIntent(entry.derived),
phase: getArchiveDerivedPhase(entry.derived),
...(routingDecision ? { routingDecision } : {}),
};
}
+26 -3
View File
@@ -1,8 +1,9 @@
import { describe, expect, it } from "vitest";
import type { PromptVariantManifest } from "@eval/prompt-variant";
import { SkillRegistry } from "@skills/skill-loader";
import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder";
import { createBuiltInToolRegistry } from "@tools/builtins/index";
import { createModerationToolRegistry } from "@skills/moderation/tools";
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
import { runG1Static, runG4Permission } from "./gates.js";
function manifest(
@@ -69,7 +70,7 @@ describe("governance gates", () => {
it("checks required tool and resource permissions in G4", () => {
const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`);
const toolRegistry = createBuiltInToolRegistry();
const toolRegistry = createOnboardingToolRegistry();
expect(
runG4Permission(
@@ -80,6 +81,7 @@ describe("governance gates", () => {
} as PromptVariantManifest,
toolRegistry,
skillRegistry,
"onboarding",
),
).toMatchObject({
passed: true,
@@ -94,10 +96,31 @@ describe("governance gates", () => {
} as PromptVariantManifest,
toolRegistry,
skillRegistry,
"onboarding",
),
).toMatchObject({
passed: false,
reason: "missing-required-capability",
});
});
it("checks moderation resources against the target skill", () => {
const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`);
expect(
runG4Permission(
{
...manifest(),
requiredResources: ["community-rules"],
requiredTools: ["lookup-rule"],
skill: "moderation",
} as PromptVariantManifest,
createModerationToolRegistry(),
skillRegistry,
"moderation",
),
).toMatchObject({
passed: true,
});
});
});
+10 -4
View File
@@ -1,5 +1,4 @@
import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder";
import type { RuntimeFlagManifest } from "@skills/onboarding/runtime/runtime-flag-manifest";
import type { PromptVariantManifest } from "@eval/prompt-variant";
import type { SkillRegistry } from "@skills/skill-loader";
import type { ToolRegistry } from "@tools/registry";
@@ -10,6 +9,12 @@ export interface GateResult {
reason?: string | undefined;
}
interface RuntimeFlagManifest {
description: string;
flags: Record<string, unknown>;
variantId: string;
}
type GovernedManifest = PromptVariantManifest | RuntimeFlagManifest;
const riskyPatterns: Array<{ pattern: RegExp; reason: string }> = [
@@ -94,14 +99,15 @@ export function runG4Permission(
manifest: GovernedManifest,
toolRegistry: ToolRegistry,
skillRegistry: SkillRegistry,
targetSkill: string,
): GateResult {
const record = asRecord(manifest);
const requiredTools = readStringArray(record, "requiredTools");
const requiredResources = readStringArray(record, "requiredResources");
const missingTools = requiredTools.filter((toolName) => !toolRegistry.has(toolName));
const onboarding = skillRegistry.get("onboarding");
const skillManifest = skillRegistry.get(targetSkill);
const missingResources = requiredResources.filter(
(resource) => !onboarding?.resources[resource],
(resource) => !skillManifest?.resources[resource],
);
if (missingTools.length > 0 || missingResources.length > 0) {
+12 -6
View File
@@ -2,7 +2,13 @@ import type {
ProcessTraceSnapshot,
ProcessTraceStepName,
} from "@skills/onboarding/runtime/trace";
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
import {
getArchiveDerivedIntent,
getArchiveDerivedPhase,
getArchiveRoutingDecision,
type EvalArchiveEntry,
type MongoConnectionLike,
} from "../archive.js";
import type { SliceKey } from "../aggregator.js";
import type { PrmModel, PrmFeatureVector } from "./prm.js";
import {
@@ -136,14 +142,14 @@ export function attributeTraceWithPrm(
function sliceFromEntry(entry: EvalArchiveEntry): SliceKey {
const abstention = (entry.process as ProcessTraceSnapshot | undefined)
?.steps.classify?.output.abstention;
const routingDecision = getArchiveRoutingDecision(entry.derived);
return {
skill: entry.skill || entry.request.skill || "onboarding",
tier: entry.tier,
intent: entry.derived.intent.kind,
phase: entry.derived.phase.kind,
...(entry.derived.routingDecision
? { routingDecision: entry.derived.routingDecision.kind }
: {}),
intent: getArchiveDerivedIntent(entry.derived),
phase: getArchiveDerivedPhase(entry.derived),
...(routingDecision ? { routingDecision } : {}),
...(abstention?.action.kind !== undefined &&
abstention.action.kind !== "proceed"
? { abstentionAction: abstention.action.kind }
+49 -26
View File
@@ -7,29 +7,36 @@ import type {
LlmJudgeResult,
LlmJudgeScores,
} from "../types.js";
const llmJudgeSchema = z.object({
friendliness: z.number().int().min(1).max(5),
accuracy: z.number().int().min(1).max(5),
concision: z.number().int().min(1).max(5),
scope: z.number().int().min(1).max(5),
rationale: z.string().trim().min(1).max(200),
});
import { getSkillJudgeConfig } from "./skill-config.js";
function round(value: number): number {
return Number(value.toFixed(3));
}
function createJudgeSchema(axes: readonly string[]): z.ZodObject<z.ZodRawShape> {
return z.object({
...Object.fromEntries(
axes.map((axis) => [axis, z.number().int().min(1).max(5)]),
),
rationale: z.string().trim().min(1).max(200),
});
}
function buildJudgePrompt(
skillName: string,
evalCase: EvalCaseDefinition,
responseText: string,
derived: EvalDerivedSnapshot,
): string {
const config = getSkillJudgeConfig(skillName);
const axes = config.axes.join(", ");
const scoreShape = Object.fromEntries(config.axes.map((axis) => [axis, 1]));
return [
"다음 온보딩 에이전트 응답을 평가해 주세요.",
"평가 축은 friendliness, accuracy, concision, scope 입니다.",
`다음 ${skillName} 에이전트 응답을 평가해 주세요.`,
`평가 축은 ${axes} 입니다.`,
"각 점수는 1~5의 정수만 사용하세요.",
"scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.",
config.promptTemplate ??
"accuracy는 기대 조건과 실제 답변의 사실성을, concision은 과도한 부가 설명 여부를 포함해 평가하세요.",
"",
"[request]",
JSON.stringify(evalCase.input.request),
@@ -48,12 +55,13 @@ function buildJudgePrompt(
"[assistant_response]",
responseText,
"",
'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}',
`JSON object only: ${JSON.stringify({ ...scoreShape, rationale: "..." })}`,
].join("\n");
}
export async function judgeOnboardingWithLlm(
export async function judgeSkillWithLlm(
llmClient: LlmClient,
skillName: string,
evalCase: EvalCaseDefinition,
responseText: string,
derived: EvalDerivedSnapshot,
@@ -77,11 +85,11 @@ export async function judgeOnboardingWithLlm(
{
role: "system",
content:
"당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
`당신은 ${skillName} 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.`,
},
{
role: "user",
content: buildJudgePrompt(evalCase, responseText, derived),
content: buildJudgePrompt(skillName, evalCase, responseText, derived),
},
],
{
@@ -100,18 +108,18 @@ export async function judgeOnboardingWithLlm(
}
try {
const parsed = llmJudgeSchema.parse(JSON.parse(response.content));
const config = getSkillJudgeConfig(skillName);
const parsed = createJudgeSchema(config.axes).parse(
JSON.parse(response.content),
);
const axisScores = Object.fromEntries(
config.axes.map((axis) => [axis, Number(parsed[axis])]),
);
const scores: LlmJudgeScores = {
friendliness: parsed.friendliness,
accuracy: parsed.accuracy,
concision: parsed.concision,
scope: parsed.scope,
...axisScores,
overall: round(
(parsed.friendliness +
parsed.accuracy +
parsed.concision +
parsed.scope) /
4,
config.axes.reduce((sum, axis) => sum + Number(parsed[axis]), 0) /
config.axes.length,
),
};
@@ -121,7 +129,7 @@ export async function judgeOnboardingWithLlm(
endpoint: response.endpoint,
provider: response.provider,
scores,
rationale: parsed.rationale,
rationale: String(parsed["rationale"]),
raw: response.content,
};
} catch {
@@ -132,3 +140,18 @@ export async function judgeOnboardingWithLlm(
};
}
}
export async function judgeOnboardingWithLlm(
llmClient: LlmClient,
evalCase: EvalCaseDefinition,
responseText: string,
derived: EvalDerivedSnapshot,
): Promise<LlmJudgeResult> {
return judgeSkillWithLlm(
llmClient,
"onboarding",
evalCase,
responseText,
derived,
);
}
+10 -1
View File
@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";
import { afterEach, describe, expect, it } from "vitest";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
import { judgeProcessRule } from "./process-judge.js";
import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js";
@@ -86,6 +87,8 @@ function makeTrace(
}
describe("judgeProcessRule", () => {
const toolRegistry = createOnboardingToolRegistry();
afterEach(() => {
delete process.env["EVAL_PRM_ENABLED"];
delete process.env["EVAL_PRM_WEIGHTS_PATH"];
@@ -102,6 +105,7 @@ describe("judgeProcessRule", () => {
},
},
makeTrace(),
toolRegistry,
);
expect(result.score).toBe(1);
@@ -122,6 +126,7 @@ describe("judgeProcessRule", () => {
},
},
makeTrace(),
toolRegistry,
);
expect(
@@ -155,6 +160,7 @@ describe("judgeProcessRule", () => {
},
],
}),
toolRegistry,
);
expect(
@@ -180,6 +186,7 @@ describe("judgeProcessRule", () => {
},
},
}),
toolRegistry,
);
expect(
@@ -214,6 +221,7 @@ describe("judgeProcessRule", () => {
},
},
}),
toolRegistry,
);
expect(
@@ -226,7 +234,7 @@ describe("judgeProcessRule", () => {
});
it("skips optional expectations when a golden case has not opted in", () => {
const result = judgeProcessRule({}, makeTrace());
const result = judgeProcessRule({}, makeTrace(), toolRegistry);
expect(
result.checks.find((check) => check.rule === "intent-match")?.skipped,
@@ -261,6 +269,7 @@ describe("judgeProcessRule", () => {
},
},
}),
toolRegistry,
);
expect(result.prmFeatures).toMatchObject({
+14 -11
View File
@@ -2,8 +2,7 @@ import { z } from "zod";
import type { AgentRequest } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import { createBuiltInToolRegistry } from "@tools/builtins/index";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import type { ToolRegistry } from "@tools/registry";
import {
getEvalProcessLlmJudgeMaxTokens,
getEvalPrmEnabled,
@@ -12,6 +11,7 @@ import {
} from "../../config.js";
import type {
EvalCaseExpected,
EvalProcessTraceSnapshot,
ProcessJudgeCheck,
ProcessJudgeResult,
} from "../types.js";
@@ -80,7 +80,7 @@ function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudge
function enrichWithPrm(
result: ProcessJudgeResult,
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
): ProcessJudgeResult {
if (!getEvalPrmEnabled()) {
return result;
@@ -125,7 +125,7 @@ function hasAllTools(
return subset && actual.size === expected.size;
}
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
function countRecordedSteps(trace: EvalProcessTraceSnapshot): number {
const scalarSteps = [
trace.steps.classify,
trace.steps.derive,
@@ -138,18 +138,20 @@ function countRecordedSteps(trace: ProcessTraceSnapshot): number {
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
}
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
function validateToolArgs(
trace: EvalProcessTraceSnapshot,
toolRegistry: ToolRegistry,
): ProcessJudgeCheck {
const toolCalls = trace.steps.toolCalls ?? [];
if (toolCalls.length === 0) {
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
}
const registry = createBuiltInToolRegistry();
const failures: string[] = [];
for (const call of toolCalls) {
try {
registry.get(call.output.name).argsSchema.parse(call.output.args);
toolRegistry.get(call.output.name).argsSchema.parse(call.output.args);
} catch {
failures.push(call.output.name);
}
@@ -167,7 +169,8 @@ function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
export function judgeProcessRule(
expected: EvalCaseExpected,
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
toolRegistry: ToolRegistry,
): ProcessJudgeResult {
const checks: ProcessJudgeCheck[] = [];
@@ -244,7 +247,7 @@ export function judgeProcessRule(
);
}
checks.push(validateToolArgs(trace));
checks.push(validateToolArgs(trace, toolRegistry));
const toolPlanStepCount = trace.steps.plan?.output.toolPlanStepCount;
if (toolPlanStepCount !== undefined) {
@@ -318,7 +321,7 @@ export function judgeProcessRule(
}
function buildProcessJudgePrompt(
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
options: { finalText: string; request: AgentRequest },
): string {
return [
@@ -341,7 +344,7 @@ function buildProcessJudgePrompt(
export async function judgeProcessWithLlm(
llmClient: LlmClient,
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
options: { finalText: string; request: AgentRequest },
): Promise<ProcessJudgeResult> {
if (!llmClient.hasEndpoints()) {
+31 -10
View File
@@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import { judgeOnboardingRuleCase } from "./rule-judge.js";
import { judgeRuleCase } from "./rule-judge.js";
import { getSkillJudgeConfig } from "./skill-config.js";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
function makeDerived(
@@ -18,9 +19,12 @@ function makeDerived(
};
}
describe("judgeOnboardingRuleCase", () => {
describe("judgeRuleCase", () => {
const validator = getSkillJudgeConfig("onboarding").validator;
it("passes when active checks match", () => {
const result = judgeOnboardingRuleCase(
const derived = makeDerived();
const result = judgeRuleCase(
{
intent: "status-question",
responseMode: "llm",
@@ -28,7 +32,12 @@ describe("judgeOnboardingRuleCase", () => {
mustNotInclude: ["디스코드 서버"],
},
"이제 방 하트 인증만 남았어요.",
makeDerived(),
{
derived,
intent: derived.intent.kind,
responseMode: derived.responseMode,
validator,
},
);
expect(result.passed).toBe(true);
@@ -37,13 +46,19 @@ describe("judgeOnboardingRuleCase", () => {
});
it("fails intent and mustNotInclude mismatches", () => {
const result = judgeOnboardingRuleCase(
const derived = makeDerived();
const result = judgeRuleCase(
{
intent: "help-how",
mustNotInclude: ["디스코드"],
},
"디스코드 서버에 먼저 들어오시면 돼요.",
makeDerived(),
{
derived,
intent: derived.intent.kind,
responseMode: derived.responseMode,
validator,
},
);
expect(result.passed).toBe(false);
@@ -56,12 +71,18 @@ describe("judgeOnboardingRuleCase", () => {
});
it("treats rejecting validator results as a failed rule check", () => {
const result = judgeOnboardingRuleCase(
const derived = makeDerived({
intent: { kind: "photo-question" },
});
const result = judgeRuleCase(
{},
"디스코드 서버에 들어가시면 돼요.",
makeDerived({
intent: { kind: "photo-question" },
}),
{
derived,
intent: derived.intent.kind,
responseMode: derived.responseMode,
validator,
},
);
expect(result.passed).toBe(false);
+26 -120
View File
@@ -1,6 +1,5 @@
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
import type { SkillJudgeConfig } from "./skill-config.js";
function normalizeText(value: string): string {
return value.trim().toLowerCase();
@@ -14,18 +13,35 @@ export function judgeRuleCase(
expected: EvalCaseExpected,
responseText: string,
actual: {
derived?: unknown;
intent: string;
responseMode?: string | undefined;
validator?: SkillJudgeConfig["validator"] | undefined;
},
): RuleJudgeResult {
const checks: RuleJudgeCheck[] = [];
const validation = actual.validator
? actual.validator(responseText, actual.derived)
: null;
checks.push({
name: "validator",
passed: true,
skipped: true,
detail: "not configured for this skill",
});
if (validation) {
checks.push({
name: "validator",
passed: validation.status !== "reject",
skipped: false,
detail:
validation.reasons.length > 0
? validation.reasons.join(", ")
: validation.status,
});
} else {
checks.push({
name: "validator",
passed: true,
skipped: true,
detail: "not configured for this skill",
});
}
const expectedIntent = expected.intent?.trim();
if (expectedIntent) {
@@ -112,117 +128,7 @@ export function judgeRuleCase(
passed: activeChecks.every((check) => check.passed),
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
checks,
validatorStatus: "pass",
validatorReasons: [],
};
}
export function judgeOnboardingRuleCase(
expected: EvalCaseExpected,
responseText: string,
derived: DerivedState,
): RuleJudgeResult {
const validation = validateOnboardingResponse(responseText, derived, {
allowShortResponse: derived.allowShortResponse,
});
const checks: RuleJudgeCheck[] = [];
checks.push({
name: "validator",
passed: validation.status !== "reject",
skipped: false,
detail:
validation.reasons.length > 0
? validation.reasons.join(", ")
: validation.status,
});
const expectedIntent = expected.intent?.trim();
if (expectedIntent) {
checks.push({
name: "intent",
passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind),
skipped: false,
detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`,
});
} else {
checks.push({
name: "intent",
passed: true,
skipped: true,
});
}
const expectedResponseMode = expected.responseMode?.trim();
if (expectedResponseMode) {
checks.push({
name: "responseMode",
passed:
normalizeText(expectedResponseMode) ===
normalizeText(derived.responseMode),
skipped: false,
detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`,
});
} else {
checks.push({
name: "responseMode",
passed: true,
skipped: true,
});
}
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
if (mustIncludeAny.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustIncludeAny.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustIncludeAny",
passed: Boolean(matched),
skipped: false,
detail: matched
? `matched=${matched}`
: `expected one of: ${mustIncludeAny.join(", ")}`,
});
} else {
checks.push({
name: "mustIncludeAny",
passed: true,
skipped: true,
});
}
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
if (mustNotInclude.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustNotInclude.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustNotInclude",
passed: !matched,
skipped: false,
detail: matched ? `forbidden=${matched}` : undefined,
});
} else {
checks.push({
name: "mustNotInclude",
passed: true,
skipped: true,
});
}
const activeChecks = checks.filter((check) => !check.skipped);
const passedChecks = activeChecks.filter((check) => check.passed).length;
return {
passed: activeChecks.every((check) => check.passed),
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
checks,
validatorStatus: validation.status,
validatorReasons: validation.reasons,
validatorStatus: validation?.status ?? "pass",
validatorReasons: validation?.reasons ?? [],
};
}
+46
View File
@@ -0,0 +1,46 @@
import type { ToolRegistry } from "@tools/registry";
import { createModerationToolRegistry } from "@skills/moderation/tools";
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
import {
validateOnboardingResponse,
type ValidationResult,
} from "@skills/onboarding/runtime/validator";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
export type SkillValidationResult = ValidationResult;
export interface SkillJudgeConfig {
axes: readonly string[];
promptTemplate?: string | undefined;
toolRegistryFactory: () => ToolRegistry;
tournamentInstruction: string;
validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined;
weights?: Record<string, number> | undefined;
}
export const SKILL_JUDGE_CONFIGS: Record<string, SkillJudgeConfig> = {
moderation: {
axes: ["appropriateness", "accuracy", "helpfulness"],
toolRegistryFactory: createModerationToolRegistry,
tournamentInstruction:
"커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.",
},
onboarding: {
axes: ["friendliness", "accuracy", "concision", "scope"],
toolRegistryFactory: createOnboardingToolRegistry,
tournamentInstruction:
"온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.",
validator: (text, derived) =>
validateOnboardingResponse(text, derived as DerivedState, {
allowShortResponse: (derived as DerivedState).allowShortResponse,
}),
},
};
export function getSkillJudgeConfig(skillName: string): SkillJudgeConfig {
const config = SKILL_JUDGE_CONFIGS[skillName] ?? SKILL_JUDGE_CONFIGS["onboarding"];
if (!config) {
throw new Error("Missing onboarding judge config");
}
return config;
}
+41 -45
View File
@@ -5,15 +5,11 @@ import { dirname, resolve } from "node:path";
import { z } from "zod";
import type { LlmClient } from "@llm/llm-client";
import { SkillRegistry } from "@skills/skill-loader";
import { createBuiltInToolRegistry } from "@tools/builtins/index";
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
import {
PromptBuilder,
type PromptLayerDiff,
type PromptLayerName,
type PromptVariantManifest,
} from "@skills/onboarding/prompts/builder";
} from "@eval/prompt-variant";
import {
getEvalTournamentMinCases,
getLlmProposeMaxTokens,
@@ -26,6 +22,8 @@ import { queryArchivedReviewLabels } from "../review/label-archive.js";
import type { ReviewLabel } from "../review/queue.js";
import { runG1Static, runG4Permission } from "../governance/gates.js";
import { promoteTrustTier } from "../governance/trust.js";
import { getSkillJudgeConfig } from "../judges/skill-config.js";
import { getSkillProposeAdapter } from "./skill-adapters.js";
export interface ProposalInput {
slice: SliceKey;
@@ -129,44 +127,12 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null {
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
}
function emptyBlackboard(): Blackboard {
return {
flaggedIssues: [],
openQuestions: [],
recentNicknameAttempts: [],
toneMemory: {},
};
}
function derivedFromSlice(slice: SliceKey): DerivedState {
return {
allowShortResponse: false,
intent: { kind: slice.intent } as Intent,
phase: { kind: slice.phase } as OnboardingPhase,
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 384,
tier: slice.tier,
};
}
function defaultPromptContext(): PromptContext {
return {
blackboard: emptyBlackboard(),
conversationHistory: [],
userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
};
}
export function buildCurrentPromptSnapshot(
slice: SliceKey,
archiveEntry?: EvalArchiveEntry,
): string {
return PromptBuilder.forDerived(
archiveEntry?.derived ?? derivedFromSlice(slice),
null,
defaultPromptContext(),
).buildSystemPrompt();
return getSkillProposeAdapter(slice.skill ?? archiveEntry?.skill ?? "onboarding")
.buildSnapshot(slice, archiveEntry);
}
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
@@ -233,8 +199,9 @@ export function summarizeRootCauses(
}
function buildProposalPrompt(input: ProposalInput): string {
const skill = input.slice.skill ?? "onboarding";
return [
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
`${skill} 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.`,
"반드시 JSON object만 출력하세요.",
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
@@ -271,7 +238,7 @@ export async function proposeVariants(
{
role: "system",
content:
"당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.",
`당신은 ${input.slice.skill ?? "onboarding"} 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.`,
},
{
role: "user",
@@ -296,10 +263,12 @@ export async function proposeVariants(
return {
...proposal,
diff,
skill: input.slice.skill ?? "onboarding",
variantId: stableHash({
description: proposal.description,
diff,
rationale: proposal.rationale,
skill: input.slice.skill ?? "onboarding",
}),
};
});
@@ -325,11 +294,34 @@ export async function buildProposalInputFromArchive(
},
): Promise<ProposalInput> {
const limit = options?.limit ?? 50;
const skill = slice.skill ?? "onboarding";
const skillFilter =
skill === "onboarding"
? {
$or: [
{ skill },
{ skill: { $exists: false } },
],
}
: { skill };
const entries = await getEvalArchiveCollection(mongoConnection)
.find({
tier: slice.tier,
"derived.intent.kind": slice.intent,
"derived.phase.kind": slice.phase,
$and: [
skillFilter,
{ tier: slice.tier },
{
$or: [
{
"derived.intent.kind": slice.intent,
"derived.phase.kind": slice.phase,
},
{
"derived.intent": slice.intent,
"derived.phase": slice.phase,
},
],
},
],
})
.sort({ evaluatedAt: -1 })
.limit(limit)
@@ -373,13 +365,17 @@ export async function writePromptVariantProposal(
await mkdir(dirname(outputPath), { recursive: true });
const manifest: PromptVariantProposal = {
...proposal,
skill: proposal.skill ?? "onboarding",
trustTier: proposal.trustTier ?? "T1",
};
const g1 = runG1Static(manifest);
const skill = manifest.skill ?? "onboarding";
const judgeConfig = getSkillJudgeConfig(skill);
const g4 = runG4Permission(
manifest,
createBuiltInToolRegistry(),
judgeConfig.toolRegistryFactory(),
SkillRegistry.load(resolve(process.cwd(), "src", "skills")),
skill,
);
const gatedManifest: PromptVariantProposal = {
...manifest,
+16
View File
@@ -0,0 +1,16 @@
import { moderationProposeAdapter } from "@skills/moderation/eval/propose-adapter";
import { onboardingProposeAdapter } from "@skills/onboarding/eval/propose-adapter";
import type { SkillProposeAdapter } from "./skill-propose-adapter.js";
const SKILL_PROPOSE_ADAPTERS: Record<string, SkillProposeAdapter> = {
[moderationProposeAdapter.skill]: moderationProposeAdapter,
[onboardingProposeAdapter.skill]: onboardingProposeAdapter,
};
export function getSkillProposeAdapter(skill: string): SkillProposeAdapter {
const adapter = SKILL_PROPOSE_ADAPTERS[skill] ?? SKILL_PROPOSE_ADAPTERS["onboarding"];
if (!adapter) {
throw new Error("Missing onboarding propose adapter");
}
return adapter;
}
+38
View File
@@ -0,0 +1,38 @@
import type { AgentRequest } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { PromptVariantManifest } from "@eval/prompt-variant";
import type { EvalArchiveEntry } from "../archive.js";
import type { SliceKey } from "../aggregator.js";
export interface SkillProposeAdapter {
readonly skill: string;
buildSnapshot(slice: SliceKey, archiveEntry?: EvalArchiveEntry): string;
derivedFromSlice(slice: SliceKey): unknown;
generateResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string>;
}
export function getVariantManifest(
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): PromptVariantManifest | string | null {
if (variantId === "control") {
return null;
}
return manifests?.[variantId] ?? variantId;
}
export function basicPromptSnapshot(messages: ChatMessage[]): string {
return messages
.map((message) => `[${message.role}]\n${message.content}`)
.join("\n\n");
}
export function getRequestText(request: AgentRequest): string {
return request.userMessage?.trim() || "";
}
+60 -62
View File
@@ -2,11 +2,12 @@ import { createHash } from "node:crypto";
import { z } from "zod";
import type { LlmClient } from "@llm/llm-client";
import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder";
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
import type { PromptVariantManifest } from "@eval/prompt-variant";
import { getEvalTournamentMaxMatches } from "../../config.js";
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
import { getEvalArchiveCollection } from "../archive.js";
import { getSkillJudgeConfig } from "../judges/skill-config.js";
import { getSkillProposeAdapter } from "./skill-adapters.js";
import {
createEloRating,
scoreForVerdict,
@@ -30,6 +31,7 @@ export interface TournamentMatch {
export interface TournamentResult {
generatedAt: string;
matches: TournamentMatch[];
skill?: string | undefined;
variants: EloRating[];
}
@@ -58,6 +60,10 @@ const sideBySideJudgeSchema = z.object({
verdict: z.enum(["A", "B", "tie"]),
});
function entrySkill(entry: EvalArchiveEntry): string {
return entry.skill || entry.request.skill || "onboarding";
}
function stableHash(value: unknown): string {
return createHash("sha256")
.update(JSON.stringify(value))
@@ -65,56 +71,18 @@ function stableHash(value: unknown): string {
.slice(0, 12);
}
function emptyBlackboard(): Blackboard {
return {
flaggedIssues: [],
openQuestions: [],
recentNicknameAttempts: [],
toneMemory: {},
};
}
function promptContextForCase(entry: EvalArchiveEntry): PromptContext {
return {
blackboard: emptyBlackboard(),
conversationHistory: [],
userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.",
};
}
function getVariantManifest(
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): PromptVariantManifest | string | null {
if (variantId === "control") {
return null;
}
return manifests?.[variantId] ?? variantId;
}
async function generateTournamentResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string> {
let builder = PromptBuilder.forDerived(
entry.derived,
null,
promptContextForCase(entry),
return getSkillProposeAdapter(entrySkill(entry)).generateResponse(
llmClient,
entry,
variantId,
manifests,
);
const manifest = getVariantManifest(variantId, manifests);
if (manifest) {
builder = builder.applyVariant(manifest);
}
const response = await llmClient.chatCompletion(builder.build(), {
maxTokens: entry.derived.suggestedMaxTokens,
temperature: 0,
tier: entry.derived.tier,
});
return response?.content.trim() ?? "";
}
function buildJudgePrompt(
@@ -122,9 +90,12 @@ function buildJudgePrompt(
responseA: string,
responseB: string,
): string {
const skill = entrySkill(entry);
const config = getSkillJudgeConfig(skill);
return [
"아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.",
"평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.",
`아래 두 ${skill} 응답 중 더 나은 쪽을 고르세요.`,
config.tournamentInstruction,
`평가 축: ${config.axes.join(", ")}`,
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
"",
"[request]",
@@ -157,7 +128,7 @@ async function judgeTournamentMatch(
{
role: "system",
content:
"당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.",
`당신은 ${entrySkill(entry)} 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.`,
},
{
role: "user",
@@ -201,6 +172,7 @@ async function ensureTournamentIndexes(
): Promise<void> {
await Promise.all([
collection.createIndex({ generatedAt: -1 }),
collection.createIndex({ skill: 1, generatedAt: -1 }),
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
]);
}
@@ -225,6 +197,7 @@ export async function runTournament(
cases: EvalArchiveEntry[];
maxMatches?: number | undefined;
nowMs?: number | undefined;
skill?: string | undefined;
variantIds: string[];
variantManifests?: Record<string, PromptVariantManifest> | undefined;
},
@@ -303,9 +276,11 @@ export async function runTournament(
}
}
const skill = options.skill ?? options.cases[0]?.skill;
return {
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
matches,
...(skill ? { skill } : {}),
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
};
}
@@ -315,22 +290,35 @@ export async function loadTournamentCases(
options: {
caseIds?: string[] | undefined;
limit?: number | undefined;
skill?: string | undefined;
},
): Promise<EvalArchiveEntry[]> {
const filter =
options.caseIds && options.caseIds.length > 0
? {
streamId: {
$in: options.caseIds,
},
}
: {};
const filter: Record<string, unknown> = {};
if (options.caseIds && options.caseIds.length > 0) {
filter.streamId = {
$in: options.caseIds,
};
}
if (options.skill) {
if (options.skill === "onboarding") {
filter.$or = [
{ skill: options.skill },
{ skill: { $exists: false } },
];
} else {
filter.skill = options.skill;
}
}
return getEvalArchiveCollection(mongoConnection)
const entries = await getEvalArchiveCollection(mongoConnection)
.find(filter)
.sort({ evaluatedAt: -1 })
.limit(options.limit ?? getEvalTournamentMaxMatches())
.toArray();
return entries.map((entry) => ({
...entry,
skill: entry.skill || entry.request.skill || "onboarding",
}));
}
export async function saveTournamentResult(
@@ -346,14 +334,24 @@ export async function queryTournamentResults(
mongoConnection: MongoConnectionLike,
options?: {
limit?: number | undefined;
skill?: string | undefined;
variantId?: string | undefined;
},
): Promise<TournamentResult[]> {
const filter = options?.variantId
? {
"variants.id": options.variantId,
}
: {};
const filter: Record<string, unknown> = {};
if (options?.variantId) {
filter["variants.id"] = options.variantId;
}
if (options?.skill) {
if (options.skill === "onboarding") {
filter.$or = [
{ skill: options.skill },
{ skill: { $exists: false } },
];
} else {
filter.skill = options.skill;
}
}
return getTournamentCollection(mongoConnection)
.find(filter)
+8 -4
View File
@@ -16,7 +16,11 @@ import {
} from "./anonymize.js";
import type { EvalCaseDefinition } from "./types.js";
import type { EvalArchiveEntry, MongoConnectionLike } from "./archive.js";
import { getEvalArchiveCollection } from "./archive.js";
import {
getArchiveDerivedIntent,
getArchiveDerivedPhase,
getEvalArchiveCollection,
} from "./archive.js";
import { queryArchivedReviewLabels } from "./review/label-archive.js";
import type { ReviewLabel } from "./review/queue.js";
@@ -59,7 +63,7 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null {
}
function sliceKey(entry: EvalArchiveEntry): string {
return `${entry.tier}:${entry.derived.intent.kind}:${entry.derived.phase.kind}`;
return `${entry.skill || entry.request.skill || "onboarding"}:${entry.tier}:${getArchiveDerivedIntent(entry.derived)}:${getArchiveDerivedPhase(entry.derived)}`;
}
function sortByLowestScore(
@@ -286,8 +290,8 @@ function candidateToEvalCase(candidate: PromotionCandidate): EvalCaseDefinition
candidate.labelRationale === undefined
? null
: `humanRationale=${candidate.labelRationale}`,
`derivedIntent=${candidate.derived.intent.kind}`,
`derivedPhase=${candidate.derived.phase.kind}`,
`derivedIntent=${getArchiveDerivedIntent(candidate.derived)}`,
`derivedPhase=${getArchiveDerivedPhase(candidate.derived)}`,
`assistantResponse=${candidate.response.text}`,
candidate.betterResponse === undefined
? null
+70
View File
@@ -0,0 +1,70 @@
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
export type PromptLayerName =
| "context"
| "examples"
| "persona"
| "policy"
| "state"
| "tool-policy";
export type PromptLayerOperation = "append" | "remove" | "replace";
export interface PromptLayerDiff {
layer: PromptLayerName;
op: PromptLayerOperation;
reason?: string | undefined;
text?: string | undefined;
}
export interface PromptVariantManifest {
description?: string | undefined;
diff: PromptLayerDiff[];
expectedImprovement?: string | undefined;
rationale?: string | undefined;
requiredResources?: string[] | undefined;
requiredTools?: string[] | undefined;
skill?: string | undefined;
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
variantId: string;
}
const variantCache = new Map<string, PromptVariantManifest | null>();
export function loadPromptVariantManifest(
variantId: string,
): PromptVariantManifest | null {
const normalized = variantId.trim();
if (!normalized) {
return null;
}
if (variantCache.has(normalized)) {
return variantCache.get(normalized) ?? null;
}
const path = resolve(
process.cwd(),
"src",
"eval",
"proposals",
`${normalized}.json`,
);
if (!existsSync(path)) {
variantCache.set(normalized, null);
return null;
}
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
variantCache.set(normalized, null);
return null;
}
variantCache.set(normalized, parsed);
return parsed;
} catch {
variantCache.set(normalized, null);
return null;
}
}
+5 -4
View File
@@ -81,11 +81,12 @@ function toCalibrationPairs(
pairs.push({
evalStreamId: label.evalStreamId,
humanScore: label.humanScore,
llmAccuracy: archive.llmJudge.scores.accuracy,
llmConcision: archive.llmJudge.scores.concision,
llmFriendliness: archive.llmJudge.scores.friendliness,
llmAccuracy: archive.llmJudge.scores.accuracy ?? archive.llmJudge.scores.overall,
llmConcision: archive.llmJudge.scores.concision ?? archive.llmJudge.scores.overall,
llmFriendliness:
archive.llmJudge.scores.friendliness ?? archive.llmJudge.scores.overall,
llmOverall: archive.llmJudge.scores.overall,
llmScope: archive.llmJudge.scores.scope,
llmScope: archive.llmJudge.scores.scope ?? archive.llmJudge.scores.overall,
});
}
+25 -2
View File
@@ -12,6 +12,26 @@ import {
} from "@skills/onboarding/runtime/router";
import type { EvalArchiveEntry } from "./archive.js";
function isOnboardingArchiveEntry(
entry: EvalArchiveEntry,
): entry is EvalArchiveEntry & {
classified: ClassifiedIntent;
derived: DerivedState;
} {
return (
(entry.skill === undefined ||
entry.skill === "onboarding" ||
entry.request.skill === "onboarding") &&
typeof entry.derived === "object" &&
entry.derived !== null &&
typeof (entry.derived as { intent?: unknown }).intent === "object" &&
typeof (entry.derived as { phase?: unknown }).phase === "object" &&
typeof entry.classified === "object" &&
entry.classified !== null &&
typeof (entry.classified as { intent?: unknown }).intent === "object"
);
}
function getLlmOverall(entry: EvalArchiveEntry): number | null {
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
}
@@ -21,6 +41,9 @@ function requestFromEntry(entry: EvalArchiveEntry): AgentRequest {
}
function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null {
if (!isOnboardingArchiveEntry(entry)) {
return null;
}
const staticDecision = routerDecisionKind(
decidePreGenerationRouting({
classified: entry.classified,
@@ -57,11 +80,11 @@ function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null {
}
function toClassified(entry: EvalArchiveEntry): ClassifiedIntent {
return entry.classified;
return entry.classified as ClassifiedIntent;
}
function toDerived(entry: EvalArchiveEntry): DerivedState {
return entry.derived;
return entry.derived as DerivedState;
}
export function buildRouterTrainingSamples(
+59 -19
View File
@@ -30,12 +30,10 @@ import type {
} from "@skills/onboarding/state/onboarding-state";
import { noopShadowEvalPublisher } from "./shadow-bus.js";
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
import { judgeProcessRule } from "./judges/process-judge.js";
import {
judgeOnboardingRuleCase,
judgeRuleCase,
} from "./judges/rule-judge.js";
import { judgeRuleCase } from "./judges/rule-judge.js";
import { getSkillJudgeConfig } from "./judges/skill-config.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
@@ -161,11 +159,18 @@ function round(value: number): number {
return Number(value.toFixed(3));
}
function average(values: number[]): number | null {
if (values.length === 0) {
function average(values: Array<number | undefined>): number | null {
const finiteValues = values.filter(
(value): value is number =>
typeof value === "number" && Number.isFinite(value),
);
if (finiteValues.length === 0) {
return null;
}
return round(values.reduce((sum, value) => sum + value, 0) / values.length);
return round(
finiteValues.reduce((sum, value) => sum + value, 0) /
finiteValues.length,
);
}
function averageOrZero(values: number[]): number {
@@ -741,8 +746,16 @@ async function evaluateCase(
skill,
llmClient,
enableLlmJudge,
effectiveSkill,
)
: evaluateGenericSkillCase(evalCase, input, skill),
: evaluateGenericSkillCase(
evalCase,
input,
skill,
llmClient,
enableLlmJudge,
effectiveSkill,
),
);
}
@@ -752,9 +765,11 @@ async function evaluateOnboardingCase(
skill: Skill,
llmClient: LlmClient,
enableLlmJudge: boolean,
skillName: string,
): Promise<EvalCaseResult> {
const classifier = new OnboardingIntentClassifier(llmClient);
const processTrace = new ProcessTraceCollector();
const judgeConfig = getSkillJudgeConfig(skillName);
return runWithProcessTrace(processTrace, async () => {
const classified = await classifier.classify(
@@ -775,14 +790,16 @@ async function evaluateOnboardingCase(
);
const latencyMs = Date.now() - startedAt;
const derivedSnapshot = toDerivedSnapshot(derived);
const ruleJudge = judgeOnboardingRuleCase(
evalCase.expected,
response.text,
const ruleJudge = judgeRuleCase(evalCase.expected, response.text, {
derived,
);
intent: derived.intent.kind,
responseMode: derived.responseMode,
validator: judgeConfig.validator,
});
const llmJudge = enableLlmJudge
? await judgeOnboardingWithLlm(
? await judgeSkillWithLlm(
llmClient,
skillName,
evalCase,
response.text,
derivedSnapshot,
@@ -795,6 +812,7 @@ async function evaluateOnboardingCase(
const processJudge = judgeProcessRule(
evalCase.expected,
processTraceSnapshot,
judgeConfig.toolRegistryFactory(),
);
return {
@@ -834,7 +852,12 @@ async function evaluateOnboardingCase(
error: errorMessage,
},
},
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
ruleJudge: judgeRuleCase(evalCase.expected, "", {
derived,
intent: derived.intent.kind,
responseMode: derived.responseMode,
validator: judgeConfig.validator,
}),
llmJudge: {
skipped: true,
reason: "skill-error",
@@ -842,6 +865,7 @@ async function evaluateOnboardingCase(
processJudge: judgeProcessRule(
evalCase.expected,
processTraceSnapshot,
judgeConfig.toolRegistryFactory(),
),
processTrace: processTraceSnapshot,
error: errorMessage,
@@ -854,8 +878,12 @@ async function evaluateGenericSkillCase(
evalCase: EvalCaseDefinition,
input: EvalCaseInput,
skill: Skill,
llmClient: LlmClient,
enableLlmJudge: boolean,
skillName: string,
): Promise<EvalCaseResult> {
const classified = classifyModerationIntent(input.request.userMessage ?? "");
const judgeConfig = getSkillJudgeConfig(skillName);
const derived: EvalDerivedSnapshot = {
allowShortResponse: true,
intent: classified.intent.kind,
@@ -887,13 +915,23 @@ async function evaluateGenericSkillCase(
request: input.request,
response,
ruleJudge: judgeRuleCase(evalCase.expected, response.text, {
derived,
intent: classified.intent.kind,
responseMode: derived.responseMode,
validator: judgeConfig.validator,
}),
llmJudge: {
skipped: true,
reason: "unsupported-skill",
},
llmJudge: enableLlmJudge
? await judgeSkillWithLlm(
llmClient,
skillName,
evalCase,
response.text,
derived,
)
: {
skipped: true,
reason: "disabled",
},
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
@@ -920,8 +958,10 @@ async function evaluateGenericSkillCase(
},
},
ruleJudge: judgeRuleCase(evalCase.expected, "", {
derived,
intent: classified.intent.kind,
responseMode: derived.responseMode,
validator: judgeConfig.validator,
}),
llmJudge: {
skipped: true,
+4
View File
@@ -61,6 +61,7 @@ describe("ShadowEvalBus", () => {
derived,
request,
response,
skill: "onboarding",
});
expect(appendStream).toHaveBeenCalledTimes(1);
@@ -68,7 +69,9 @@ describe("ShadowEvalBus", () => {
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
type: "shadow-eval",
requestId: "req-1",
skill: "onboarding",
tier: "0",
version: "2",
});
randomSpy.mockRestore();
@@ -86,6 +89,7 @@ describe("ShadowEvalBus", () => {
...response,
text: "",
},
skill: "onboarding",
});
expect(appendStream).not.toHaveBeenCalled();
+55 -18
View File
@@ -1,11 +1,6 @@
import type { AgentRequest, AgentResponse } from "@agent/types";
import type { GarnetClient } from "@common/garnet";
import { getRequestLogger } from "@obs/context";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import {
getBackgroundStreamMaxLen,
getShadowEvalSamplingRate,
@@ -14,13 +9,15 @@ import {
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
export interface ShadowEvalEnvelope {
classified: ClassifiedIntent;
classified: unknown;
createdAt: string;
derived: DerivedState;
derived: unknown;
latencyMs?: number | undefined;
process?: ProcessTraceSnapshot | undefined;
process?: unknown;
request: AgentRequest;
response: AgentResponse;
skill: string;
version?: 2 | undefined;
}
export interface ShadowEvalPublisher {
@@ -38,10 +35,46 @@ export function shouldSampleShadowEval(
return randomValue < getShadowEvalSamplingRate(tier);
}
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
| undefined {
return process?.steps.classify?.output.abstention;
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function parseTier(value: unknown): 0 | 1 | 2 | null {
const parsed = typeof value === "number" ? value : Number(value);
if (parsed === 0 || parsed === 1 || parsed === 2) {
return parsed;
}
return null;
}
function getDerivedTier(derived: unknown): 0 | 1 | 2 | null {
return parseTier(asRecord(derived)["tier"]);
}
function getTraceAbstention(process: unknown): Record<string, unknown> | null {
const steps = asRecord(asRecord(process)["steps"]);
const classify = asRecord(steps["classify"]);
const output = asRecord(classify["output"]);
const abstention = asRecord(output["abstention"]);
return Object.keys(abstention).length > 0 ? abstention : null;
}
function getAbstentionActionKind(abstention: Record<string, unknown> | null): string | null {
const action = asRecord(abstention?.["action"]);
const kind = action["kind"];
return typeof kind === "string" && kind.trim() ? kind : null;
}
function getAbstentionReason(abstention: Record<string, unknown> | null): string | null {
const reason = abstention?.["reason"];
return typeof reason === "string" && reason.trim() ? reason : null;
}
function getAbstentionScore(abstention: Record<string, unknown> | null): number | null {
const score = abstention?.["score"];
return typeof score === "number" && Number.isFinite(score) ? score : null;
}
export class ShadowEvalBus implements ShadowEvalPublisher {
@@ -51,11 +84,13 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
publish(envelope: ShadowEvalEnvelope): void {
const abstention = getTraceAbstention(envelope.process);
if (!envelope.response.text.trim() && abstention?.action.kind !== "no-response") {
const actionKind = getAbstentionActionKind(abstention);
if (!envelope.response.text.trim() && actionKind !== "no-response") {
return;
}
if (!shouldSampleShadowEval(envelope.derived.tier)) {
const tier = getDerivedTier(envelope.derived);
if (tier === null || !shouldSampleShadowEval(tier)) {
return;
}
@@ -66,7 +101,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
requestId: envelope.request.requestId,
chatroomId: envelope.request.chatroomId,
userId: envelope.request.userId,
tier: String(envelope.derived.tier),
skill: envelope.skill,
version: String(envelope.version ?? 2),
tier: String(tier),
latencyMs:
envelope.latencyMs !== undefined ? String(envelope.latencyMs) : "",
input: JSON.stringify({
@@ -79,9 +116,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
};
if (abstention) {
fields.abstentionAction = abstention.action.kind;
fields.abstentionReason = abstention.reason ?? "";
fields.abstentionScore = String(abstention.score);
fields.abstentionAction = actionKind ?? "";
fields.abstentionReason = getAbstentionReason(abstention) ?? "";
fields.abstentionScore = String(getAbstentionScore(abstention) ?? "");
}
if (envelope.process) {
+100 -41
View File
@@ -12,12 +12,6 @@ import { summarizeConversation } from "@conv/summarizer";
import type { LlmClient } from "@llm/llm-client";
import { getLogger } from "@obs/logger";
import type { AlertSender } from "@obs/alert";
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ConversationStore } from "@conv/conversation-store";
import {
getAlertThresholdDriftPct,
@@ -49,13 +43,14 @@ import {
type MongoConnectionLike,
} from "./archive.js";
import { computeDriftReports } from "./drift.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
import { writeProcessAttributions } from "./judges/attribution.js";
import { loadPrmModel } from "./judges/prm.js";
import {
judgeProcessRule,
judgeProcessWithLlm,
} from "./judges/process-judge.js";
import { getSkillJudgeConfig } from "./judges/skill-config.js";
import {
appendCandidatesToJsonl,
selectPromotionCandidates,
@@ -72,6 +67,7 @@ import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalDerivedSnapshot,
EvalProcessTraceSnapshot,
ProcessJudgeResult,
} from "./types.js";
@@ -100,13 +96,15 @@ const STOP_TIMEOUT_MS = 10_000;
type BackgroundJobType = "shadow-eval" | "summarize";
interface ShadowEvalJob {
classified: ClassifiedIntent;
classified: unknown;
createdAt: string;
derived: DerivedState;
derived: unknown;
latencyMs?: number | undefined;
process?: ProcessTraceSnapshot | undefined;
process?: EvalProcessTraceSnapshot | undefined;
request: AgentRequest;
response: AgentResponse;
skill: string;
version: number;
}
interface EvalObservation {
@@ -168,26 +166,75 @@ function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
return JSON.parse(value) as T;
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function getStringField(
record: Record<string, unknown>,
key: string,
): string | null {
const value = record[key];
return typeof value === "string" && value.trim() ? value : null;
}
function getNestedString(
record: Record<string, unknown>,
first: string,
second: string,
): string | null {
return getStringField(asRecord(record[first]), second);
}
function parseTier(value: unknown): 0 | 1 | 2 {
const parsed = typeof value === "number" ? value : Number(value);
return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0;
}
function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] {
return value === "skip" ||
value === "template" ||
value === "llm" ||
value === "llm-with-tools"
? value
: "llm";
}
function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot {
const record = asRecord(derived);
const intent = getStringField(record, "intent") ??
getNestedString(record, "intent", "kind") ??
"unknown";
const phase = getStringField(record, "phase") ??
getNestedString(record, "phase", "kind") ??
"unknown";
const abstentionScore = record["abstentionScore"];
const suggestedMaxTokens = record["suggestedMaxTokens"];
return {
...(derived.abstentionScore !== undefined
? { abstentionScore: derived.abstentionScore }
...(typeof abstentionScore === "number"
? { abstentionScore }
: {}),
allowShortResponse: derived.allowShortResponse,
intent: derived.intent.kind,
phase: derived.phase.kind,
responseMode: derived.responseMode,
...(derived.routingDecision
? { routingDecision: derived.routingDecision }
allowShortResponse: record["allowShortResponse"] === true,
intent,
phase,
responseMode: parseResponseMode(record["responseMode"]),
...(record["routingDecision"]
? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] }
: {}),
shouldRespond: derived.shouldRespond,
suggestedMaxTokens: derived.suggestedMaxTokens,
tier: derived.tier,
shouldRespond: record["shouldRespond"] !== false,
suggestedMaxTokens:
typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens)
? suggestedMaxTokens
: 192,
tier: parseTier(record["tier"]),
};
}
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined):
| NonNullable<EvalProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
| undefined {
return process?.steps.classify?.output.abstention;
}
@@ -208,17 +255,17 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
message.message["input"],
"input",
);
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
const process = parseOptionalJsonField<EvalProcessTraceSnapshot>(
message.message["process"],
);
const job: ShadowEvalJob = {
classified: parseJsonField<ClassifiedIntent>(
classified: parseJsonField<unknown>(
message.message["classified"],
"classified",
),
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
derived: parseJsonField<DerivedState>(
derived: parseJsonField<unknown>(
message.message["derived"],
"derived",
),
@@ -228,6 +275,8 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
message.message["output"],
"output",
),
skill: message.message["skill"]?.trim() || input.request.skill || "onboarding",
version: Number(message.message["version"] ?? 1),
};
if (process) {
@@ -500,6 +549,8 @@ export class ShadowWorker {
private async processShadowEval(message: StreamEntry): Promise<void> {
const job = parseShadowEvalJob(message);
const judgeConfig = getSkillJudgeConfig(job.skill);
const derivedSnapshot = toDerivedSnapshot(job.derived);
const abstention = getTraceAbstention(job.process);
const responseTextForValidation =
job.response.text.trim()
@@ -507,18 +558,19 @@ export class ShadowWorker {
: abstention?.action.kind === "no-response"
? "[NO_RESPONSE]"
: job.response.text;
const validation = validateOnboardingResponse(
responseTextForValidation,
job.derived,
{
allowShortResponse: job.derived.allowShortResponse,
},
);
const llmJudge = await judgeOnboardingWithLlm(
const validation = judgeConfig.validator
? judgeConfig.validator(responseTextForValidation, job.derived)
: {
reasons: [],
status: "pass" as const,
text: responseTextForValidation,
};
const llmJudge = await judgeSkillWithLlm(
this.llmClient,
job.skill,
toEvalCase(job, message.id),
job.response.text,
toDerivedSnapshot(job.derived),
derivedSnapshot,
);
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
const llmJudgeErrored =
@@ -541,10 +593,12 @@ export class ShadowWorker {
requestId: job.request.requestId,
chatroomId: job.request.chatroomId,
userId: job.request.userId,
tier: String(job.derived.tier),
intent: job.derived.intent.kind,
phase: job.derived.phase.kind,
responseMode: job.derived.responseMode,
skill: job.skill,
version: String(Math.max(2, job.version)),
tier: String(derivedSnapshot.tier),
intent: derivedSnapshot.intent,
phase: derivedSnapshot.phase,
responseMode: derivedSnapshot.responseMode,
validatorStatus: validation.status,
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
llmOverall: llmOverall === null ? "" : String(llmOverall),
@@ -589,7 +643,12 @@ export class ShadowWorker {
throw new Error("Missing process trace");
}
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
const judgeConfig = getSkillJudgeConfig(job.skill);
const ruleResult = judgeProcessRule(
toEvalCase(job, messageId).expected,
job.process,
judgeConfig.toolRegistryFactory(),
);
if (
!getEvalProcessLlmJudgeEnabled() ||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
+3 -4
View File
@@ -11,6 +11,8 @@ import type {
} from "@skills/onboarding/runtime/trace";
import type { PrmFeatureVector } from "./judges/prm.js";
export type EvalProcessTraceSnapshot = ProcessTraceSnapshot;
export interface EvalRoomContextMessage {
senderId: string;
nickname: string;
@@ -70,10 +72,7 @@ export interface RuleJudgeResult {
}
export interface LlmJudgeScores {
friendliness: number;
accuracy: number;
concision: number;
scope: number;
[axis: string]: number;
overall: number;
}
@@ -0,0 +1,211 @@
import type { ConversationMessage } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import {
loadPromptVariantManifest,
type PromptLayerDiff,
type PromptVariantManifest,
} from "@eval/prompt-variant";
import type { EvalArchiveEntry } from "@eval/archive";
import type { SliceKey } from "@eval/aggregator";
import {
basicPromptSnapshot,
getRequestText,
getVariantManifest,
type SkillProposeAdapter,
} from "../../../eval/loop/skill-propose-adapter.js";
import {
classifyModerationIntent,
type ModerationIntent,
type ModerationIntentKind,
} from "../intents.js";
import { MODERATION_PROMPT_DATA } from "../prompt-data.js";
interface ModerationDerivedSnapshot {
allowShortResponse: boolean;
intent: string;
phase: string;
responseMode: "skip" | "template" | "llm" | "llm-with-tools";
shouldRespond: boolean;
suggestedMaxTokens: number;
tier: 0 | 1 | 2;
}
function formatHistory(history: ConversationMessage[]): string {
if (history.length === 0) {
return "(none)";
}
return history
.slice(-5)
.map((message) => `${message.role}: ${message.content}`)
.join("\n");
}
function formatExamples(intent: ModerationIntent): string {
return MODERATION_PROMPT_DATA.examples[intent.kind]
.slice(0, 2)
.map((example) => `user: ${example.user}\nassistant: ${example.assistant}`)
.join("\n\n");
}
function applyDiff(
layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>,
diff: PromptLayerDiff,
): void {
const target = diff.layer === "tool-policy" ? "policy" : diff.layer;
if (target !== "context" &&
target !== "examples" &&
target !== "persona" &&
target !== "policy" &&
target !== "state") {
return;
}
if (diff.op === "remove") {
layers[target] = "";
return;
}
const text = diff.text?.trim();
if (!text) {
return;
}
if (diff.op === "replace") {
layers[target] = text;
return;
}
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
}
function getManifest(
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): PromptVariantManifest | null {
const manifest = getVariantManifest(variantId, manifests);
if (!manifest) {
return null;
}
return typeof manifest === "string"
? loadPromptVariantManifest(manifest)
: manifest;
}
function buildMessages(
entry: Pick<EvalArchiveEntry, "request">,
variant?: PromptVariantManifest | null,
): ChatMessage[] {
const classified = classifyModerationIntent(getRequestText(entry.request));
const intent = classified.intent;
const layers = {
persona: MODERATION_PROMPT_DATA.persona.join("\n"),
policy: [
...MODERATION_PROMPT_DATA.policy.base,
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
].join("\n"),
state: ["[intent]", intent.kind].join("\n"),
examples: formatExamples(intent),
context: "",
};
for (const diff of variant?.diff ?? []) {
applyDiff(layers, diff);
}
return [
{
role: "system",
content: [
layers.persona,
"",
"[policy]",
layers.policy,
"",
layers.state,
"",
"[examples]",
layers.examples,
...(layers.context ? ["", "[context]", layers.context] : []),
].join("\n"),
},
{
role: "user",
content: [
"[history]",
formatHistory([]),
"",
"[message]",
getRequestText(entry.request),
].join("\n"),
},
];
}
function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot {
return {
allowShortResponse: true,
intent: slice.intent,
phase: slice.phase,
responseMode: slice.intent === "off-topic" ? "template" : "llm",
shouldRespond: slice.intent !== "off-topic",
suggestedMaxTokens: 192,
tier: slice.tier,
};
}
function getSuggestedMaxTokens(entry: EvalArchiveEntry): number {
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
return typeof derived.suggestedMaxTokens === "number"
? derived.suggestedMaxTokens
: 192;
}
function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 {
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
return derived.tier === 0 || derived.tier === 1 || derived.tier === 2
? derived.tier
: entry.tier;
}
export const moderationProposeAdapter: SkillProposeAdapter = {
skill: "moderation",
buildSnapshot(slice, archiveEntry) {
const request = archiveEntry?.request ?? {
requestId: "proposal-snapshot",
skill: "moderation",
chatroomId: "proposal-room",
userId: "proposal-user",
nickname: "문의자",
trigger: "user-message" as const,
userMessage: `slice=${slice.intent}/${slice.phase}`,
};
return basicPromptSnapshot(buildMessages({ request }));
},
derivedFromSlice,
async generateResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string> {
const response = await llmClient.chatCompletion(
buildMessages(entry, getManifest(variantId, manifests)),
{
maxTokens: getSuggestedMaxTokens(entry),
temperature: 0,
tier: getTier(entry),
},
);
return response?.content.trim() ?? "";
},
};
export function normalizeModerationIntentKind(value: string): ModerationIntentKind {
return value in MODERATION_PROMPT_DATA.examples
? (value as ModerationIntentKind)
: "off-topic";
}
+12
View File
@@ -0,0 +1,12 @@
import { MODERATION_INTENT_TABLE } from "./intents.js";
import { createModerationToolRegistry } from "./tools.js";
export function getIntentKinds(): string[] {
return Object.keys(MODERATION_INTENT_TABLE);
}
export function getToolNames(): string[] {
return createModerationToolRegistry()
.list()
.map((tool) => tool.name);
}
@@ -0,0 +1,99 @@
import type { LlmClient } from "@llm/llm-client";
import type { PromptVariantManifest } from "@eval/prompt-variant";
import type { EvalArchiveEntry } from "@eval/archive";
import type { SliceKey } from "@eval/aggregator";
import {
getVariantManifest,
type SkillProposeAdapter,
} from "../../../eval/loop/skill-propose-adapter.js";
import {
PromptBuilder,
} from "../prompts/builder.js";
import type { DerivedState, Intent, OnboardingPhase } from "../state/onboarding-state.js";
import type { Blackboard, PromptContext } from "../types.js";
function emptyBlackboard(): Blackboard {
return {
flaggedIssues: [],
openQuestions: [],
recentNicknameAttempts: [],
toneMemory: {},
};
}
function derivedFromSlice(slice: SliceKey): DerivedState {
return {
allowShortResponse: false,
intent: { kind: slice.intent } as Intent,
phase: { kind: slice.phase } as OnboardingPhase,
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 384,
tier: slice.tier,
};
}
function defaultPromptContext(userMessage?: string): PromptContext {
return {
blackboard: emptyBlackboard(),
conversationHistory: [],
userMessage:
userMessage?.trim() ||
"현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
};
}
function getDerived(entry: EvalArchiveEntry, slice?: SliceKey): DerivedState {
const derived = entry.derived as Partial<DerivedState>;
if (derived.intent && derived.phase) {
return derived as DerivedState;
}
return derivedFromSlice(
slice ?? {
skill: "onboarding",
tier: entry.tier,
intent: "unknown",
phase: "unknown",
},
);
}
export const onboardingProposeAdapter: SkillProposeAdapter = {
skill: "onboarding",
buildSnapshot(slice, archiveEntry) {
return PromptBuilder.forDerived(
archiveEntry ? getDerived(archiveEntry, slice) : derivedFromSlice(slice),
null,
defaultPromptContext(archiveEntry?.request.userMessage),
).buildSystemPrompt();
},
derivedFromSlice,
async generateResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string> {
let builder = PromptBuilder.forDerived(
getDerived(entry),
null,
defaultPromptContext(entry.request.userMessage),
);
const manifest = getVariantManifest(variantId, manifests);
if (manifest) {
builder = builder.applyVariant(manifest);
}
const derived = getDerived(entry);
const response = await llmClient.chatCompletion(builder.build(), {
maxTokens: derived.suggestedMaxTokens,
temperature: 0,
tier: derived.tier,
});
return response?.content.trim() ?? "";
},
};
+12
View File
@@ -0,0 +1,12 @@
import { ONBOARDING_INTENT_TABLE } from "./intents.js";
import { createOnboardingToolRegistry } from "./tools.js";
export function getIntentKinds(): string[] {
return Object.keys(ONBOARDING_INTENT_TABLE);
}
export function getToolNames(): string[] {
return createOnboardingToolRegistry()
.list()
.map((tool) => tool.name);
}
+12 -67
View File
@@ -1,6 +1,11 @@
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import {
loadPromptVariantManifest,
type PromptLayerDiff,
type PromptLayerName,
type PromptVariantManifest,
} from "@eval/prompt-variant";
import { SkillRegistry } from "@skills/skill-loader";
import type { ChatMessage } from "@llm/types";
import {
@@ -19,35 +24,12 @@ import {
import { buildToolPolicy } from "./tool-policy.js";
import { defaultExampleRetriever } from "./example-store.js";
export type PromptLayerName =
| "context"
| "examples"
| "persona"
| "policy"
| "state"
| "tool-policy";
export type PromptLayerOperation = "append" | "remove" | "replace";
export interface PromptLayerDiff {
layer: PromptLayerName;
op: PromptLayerOperation;
reason?: string | undefined;
text?: string | undefined;
}
export interface PromptVariantManifest {
description?: string | undefined;
diff: PromptLayerDiff[];
expectedImprovement?: string | undefined;
rationale?: string | undefined;
requiredResources?: string[] | undefined;
requiredTools?: string[] | undefined;
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
variantId: string;
}
const variantCache = new Map<string, PromptVariantManifest | null>();
export {
loadPromptVariantManifest,
type PromptLayerDiff,
type PromptLayerName,
type PromptVariantManifest,
} from "@eval/prompt-variant";
function getSkillRootDir(): string {
return resolve(process.cwd(), "src", "skills");
@@ -85,43 +67,6 @@ function applyLayerDiff(
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
}
export function loadPromptVariantManifest(
variantId: string,
): PromptVariantManifest | null {
const normalized = variantId.trim();
if (!normalized) {
return null;
}
if (variantCache.has(normalized)) {
return variantCache.get(normalized) ?? null;
}
const path = resolve(
process.cwd(),
"src",
"eval",
"proposals",
`${normalized}.json`,
);
if (!existsSync(path)) {
variantCache.set(normalized, null);
return null;
}
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
variantCache.set(normalized, null);
return null;
}
variantCache.set(normalized, parsed);
return parsed;
} catch {
variantCache.set(normalized, null);
return null;
}
}
function skillIndexLayer(registry: SkillRegistry): string {
const lines = registry.list().map(
(manifest) =>
+3
View File
@@ -189,6 +189,7 @@ export class OnboardingSkill implements Skill {
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
return response;
}
@@ -222,6 +223,7 @@ export class OnboardingSkill implements Skill {
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
return response;
@@ -294,6 +296,7 @@ export class OnboardingSkill implements Skill {
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
}
-29
View File
@@ -1,6 +1,3 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { checkOnboardingProgressTool } from "@tools/onboarding/check-onboarding-progress";
import { getRoomContextTool } from "@tools/onboarding/get-room-context";
import { lookupCommunityRuleTool } from "@tools/onboarding/lookup-community-rule";
@@ -17,32 +14,6 @@ const ONBOARDING_TOOLS = [
verifyDiscordNicknameTool,
] as const satisfies readonly Tool<unknown, unknown>[];
function readSkillManifestToolNames(): string[] {
const skillPath = resolve(process.cwd(), "src", "skills", "onboarding", "SKILL.md");
const raw = readFileSync(skillPath, "utf8");
const match = raw.match(/^tools:\s*\[([^\]]*)\]/mu);
if (!match?.[1]) {
return [];
}
return match[1]
.split(",")
.map((item) => item.trim())
.filter(Boolean);
}
function assertToolManifestMatchesRegistry(): void {
const manifestNames = readSkillManifestToolNames().sort();
const registryNames = ONBOARDING_TOOLS.map((tool) => tool.name).sort();
if (manifestNames.join("\n") !== registryNames.join("\n")) {
throw new Error(
`Onboarding SKILL.md tools do not match registry: manifest=${manifestNames.join(",")} registry=${registryNames.join(",")}`,
);
}
}
assertToolManifestMatchesRegistry();
export function createOnboardingToolRegistry(): ToolRegistry {
const registry = new ToolRegistry();
for (const tool of ONBOARDING_TOOLS) {
+106
View File
@@ -0,0 +1,106 @@
import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
import * as moderationBinding from "./moderation/manifest-binding.js";
import * as onboardingBinding from "./onboarding/manifest-binding.js";
interface ManifestBinding {
getIntentKinds(): string[];
getToolNames(): string[];
}
const BINDINGS: Record<string, ManifestBinding> = {
moderation: moderationBinding,
onboarding: onboardingBinding,
};
function sortUnique(values: readonly string[]): string[] {
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
}
function format(values: readonly string[]): string {
return values.length === 0 ? "(none)" : values.join(", ");
}
function diff(left: readonly string[], right: readonly string[]): string[] {
const rightSet = new Set(right);
return left.filter((item) => !rightSet.has(item));
}
function validateManifest(
manifest: SkillManifest,
binding: ManifestBinding | undefined,
): string[] {
if (!binding) {
return [`${manifest.name}: missing manifest binding`];
}
const errors: string[] = [];
const manifestIntents = sortUnique(manifest.intents);
const codeIntents = sortUnique(binding.getIntentKinds());
const unknownIntents = diff(manifestIntents, codeIntents);
if (unknownIntents.length > 0) {
errors.push(
`${manifest.name}: SKILL.md intents not found in code: ${format(unknownIntents)}`,
);
}
const manifestTools = sortUnique(manifest.tools);
const codeTools = sortUnique(binding.getToolNames());
const missingTools = diff(manifestTools, codeTools);
const undocumentedTools = diff(codeTools, manifestTools);
if (missingTools.length > 0 || undocumentedTools.length > 0) {
errors.push(
`${manifest.name}: SKILL.md tools mismatch; missingInRegistry=${format(missingTools)} missingInManifest=${format(undocumentedTools)}`,
);
}
for (const [key, path] of Object.entries(manifest.resources)) {
if (!existsSync(path)) {
errors.push(`${manifest.name}: resource not found ${key} -> ${path}`);
}
}
return errors;
}
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
const byPriority = new Map<number, string[]>();
for (const manifest of manifests) {
const current = byPriority.get(manifest.priority) ?? [];
current.push(manifest.name);
byPriority.set(manifest.priority, current);
}
return [...byPriority.entries()].flatMap(([priority, names]) =>
names.length > 1
? [`priority=${priority} is shared by skills: ${format(names.sort())}`]
: [],
);
}
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
const registry = SkillRegistry.load(rootDir);
const manifests = registry.list();
return [
...manifests.flatMap((manifest) =>
validateManifest(manifest, BINDINGS[manifest.name]),
),
...validatePriorityConflicts(manifests),
];
}
const invokedPath = process.argv[1] ? resolve(process.argv[1]) : "";
if (invokedPath === resolve(fileURLToPath(import.meta.url))) {
const errors = validateSkillManifests();
if (errors.length > 0) {
for (const error of errors) {
console.error(`[skills] ${error}`);
}
process.exitCode = 1;
} else {
console.info("[skills] manifests ok");
}
}