Make eval pipeline skill-aware and validate SKILL manifests
This commit is contained in:
1 parent
12e731a428
commit
54bc295ead
35 files changed
+1280
-499
No files matched your search
+4
-1
@@ -9,7 +9,8 @@
|
|||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
"test:watch": "vitest",
|
"test:watch": "vitest",
|
||||||
"test:coverage": "vitest run --coverage",
|
"test:coverage": "vitest run --coverage",
|
||||||
"lint": "eslint .",
|
"lint": "eslint . && npm run lint:skills",
|
||||||
|
"lint:skills": "node --import tsx src/skills/validate-manifests.ts",
|
||||||
"lint:fix": "eslint . --fix",
|
"lint:fix": "eslint . --fix",
|
||||||
"format": "prettier --write .",
|
"format": "prettier --write .",
|
||||||
"eval": "node --import tsx src/eval/cli.ts run --set=onboarding/cases",
|
"eval": "node --import tsx src/eval/cli.ts run --set=onboarding/cases",
|
||||||
@@ -23,7 +24,9 @@
|
|||||||
"eval:train-prm": "node --import tsx src/eval/cli.ts train-prm",
|
"eval:train-prm": "node --import tsx src/eval/cli.ts train-prm",
|
||||||
"eval:train-router": "node --import tsx src/eval/cli.ts train-router",
|
"eval:train-router": "node --import tsx src/eval/cli.ts train-router",
|
||||||
"eval:propose": "node --import tsx src/eval/cli.ts propose",
|
"eval:propose": "node --import tsx src/eval/cli.ts propose",
|
||||||
|
"eval:propose:moderation": "node --import tsx src/eval/cli.ts propose --skill=moderation",
|
||||||
"eval:tournament": "node --import tsx src/eval/cli.ts tournament",
|
"eval:tournament": "node --import tsx src/eval/cli.ts tournament",
|
||||||
|
"eval:tournament:moderation": "node --import tsx src/eval/cli.ts tournament --skill=moderation",
|
||||||
"eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results",
|
"eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results",
|
||||||
"start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"",
|
"start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"",
|
||||||
"start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"",
|
"start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"",
|
||||||
|
|||||||
+36
-6
@@ -9,6 +9,7 @@ export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor";
|
|||||||
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
|
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
|
||||||
|
|
||||||
export interface SliceKey {
|
export interface SliceKey {
|
||||||
|
skill?: string | undefined;
|
||||||
tier: 0 | 1 | 2;
|
tier: 0 | 1 | 2;
|
||||||
intent: string;
|
intent: string;
|
||||||
phase: string;
|
phase: string;
|
||||||
@@ -111,6 +112,14 @@ function getNestedString(
|
|||||||
return typeof value === "string" && value.trim() ? value : null;
|
return typeof value === "string" && value.trim() ? value : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getStringField(
|
||||||
|
record: Record<string, unknown>,
|
||||||
|
key: string,
|
||||||
|
): string | null {
|
||||||
|
const value = record[key];
|
||||||
|
return typeof value === "string" && value.trim() ? value : null;
|
||||||
|
}
|
||||||
|
|
||||||
function extractLlmOverall(message: Record<string, string>): number | null {
|
function extractLlmOverall(message: Record<string, string>): number | null {
|
||||||
const direct = parseFiniteNumber(message["llmOverall"]);
|
const direct = parseFiniteNumber(message["llmOverall"]);
|
||||||
if (direct !== null) {
|
if (direct !== null) {
|
||||||
@@ -177,9 +186,13 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
|||||||
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
|
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
|
||||||
const intent =
|
const intent =
|
||||||
entry.message["intent"]?.trim() ||
|
entry.message["intent"]?.trim() ||
|
||||||
|
getStringField(derived, "intent") ||
|
||||||
getNestedString(derived, "intent", "kind");
|
getNestedString(derived, "intent", "kind");
|
||||||
const phase =
|
const phase =
|
||||||
entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind");
|
entry.message["phase"]?.trim() ||
|
||||||
|
getStringField(derived, "phase") ||
|
||||||
|
getNestedString(derived, "phase", "kind");
|
||||||
|
const skill = entry.message["skill"]?.trim();
|
||||||
|
|
||||||
if (tier === null || !intent || !phase) {
|
if (tier === null || !intent || !phase) {
|
||||||
return null;
|
return null;
|
||||||
@@ -194,6 +207,7 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
|||||||
llmOverall: extractLlmOverall(entry.message),
|
llmOverall: extractLlmOverall(entry.message),
|
||||||
processJudgeScore: extractProcessJudgeScore(entry.message),
|
processJudgeScore: extractProcessJudgeScore(entry.message),
|
||||||
slice: {
|
slice: {
|
||||||
|
...(skill ? { skill } : {}),
|
||||||
tier,
|
tier,
|
||||||
intent,
|
intent,
|
||||||
phase,
|
phase,
|
||||||
@@ -341,17 +355,28 @@ function average(sum: number, count: number): number | null {
|
|||||||
|
|
||||||
function sliceKeyFromString(value: string): SliceKey | null {
|
function sliceKeyFromString(value: string): SliceKey | null {
|
||||||
const parts = value.split(":");
|
const parts = value.split(":");
|
||||||
const [tierRaw, intent, phase] = parts;
|
let skill: string | undefined;
|
||||||
|
let offset = 0;
|
||||||
|
let tierRaw = parts[0];
|
||||||
|
if (parseTier(tierRaw) === null && parts.length >= 4) {
|
||||||
|
skill = parts[0];
|
||||||
|
offset = 1;
|
||||||
|
tierRaw = parts[offset];
|
||||||
|
}
|
||||||
|
const intent = parts[offset + 1];
|
||||||
|
const phase = parts[offset + 2];
|
||||||
const tier = parseTier(tierRaw);
|
const tier = parseTier(tierRaw);
|
||||||
if (tier === null || !intent || !phase) {
|
if (tier === null || !intent || !phase) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const routingDecision = parts.length >= 6 ? parts[3] : undefined;
|
const tail = parts.slice(offset + 3);
|
||||||
const abstentionAction = parts.length >= 6 ? parts[4] : parts[3];
|
const routingDecision = tail.length >= 3 ? tail[0] : undefined;
|
||||||
const abstentionReason = parts.length >= 6 ? parts[5] : parts[4];
|
const abstentionAction = tail.length >= 3 ? tail[1] : tail[0];
|
||||||
|
const abstentionReason = tail.length >= 3 ? tail[2] : tail[1];
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
...(skill ? { skill } : {}),
|
||||||
tier,
|
tier,
|
||||||
intent,
|
intent,
|
||||||
phase,
|
phase,
|
||||||
@@ -376,7 +401,12 @@ function sliceKeyFromRedisKey(key: string): SliceKey | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function sliceKeyToString(key: SliceKey): string {
|
export function sliceKeyToString(key: SliceKey): string {
|
||||||
const base = `${key.tier}:${key.intent}:${key.phase}`;
|
const base = [
|
||||||
|
...(key.skill ? [key.skill] : []),
|
||||||
|
String(key.tier),
|
||||||
|
key.intent,
|
||||||
|
key.phase,
|
||||||
|
].join(":");
|
||||||
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
|
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
|
||||||
return base;
|
return base;
|
||||||
}
|
}
|
||||||
|
|||||||
+53
-9
@@ -17,17 +17,19 @@ import {
|
|||||||
type SliceKey,
|
type SliceKey,
|
||||||
type SliceReportRow,
|
type SliceReportRow,
|
||||||
} from "./aggregator.js";
|
} from "./aggregator.js";
|
||||||
|
import type { EvalClassifiedSnapshot, EvalDerivedSnapshot } from "./types.js";
|
||||||
|
|
||||||
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
|
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
|
||||||
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
|
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
|
||||||
|
|
||||||
export interface EvalArchiveEntry {
|
export interface EvalArchiveEntry {
|
||||||
streamId: string;
|
streamId: string;
|
||||||
|
skill?: string | undefined;
|
||||||
evaluatedAt: Date;
|
evaluatedAt: Date;
|
||||||
request: AgentRequest;
|
request: AgentRequest;
|
||||||
response: AgentResponse;
|
response: AgentResponse;
|
||||||
derived: DerivedState;
|
derived: DerivedState | EvalDerivedSnapshot;
|
||||||
classified: ClassifiedIntent;
|
classified: ClassifiedIntent | EvalClassifiedSnapshot;
|
||||||
validation: ValidationResult;
|
validation: ValidationResult;
|
||||||
llmJudge: LlmJudgeResult;
|
llmJudge: LlmJudgeResult;
|
||||||
processJudge?: ProcessJudgeResult | undefined;
|
processJudge?: ProcessJudgeResult | undefined;
|
||||||
@@ -127,6 +129,44 @@ function getAbstention(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getStringField(
|
||||||
|
record: Record<string, unknown>,
|
||||||
|
key: string,
|
||||||
|
): string | null {
|
||||||
|
const value = record[key];
|
||||||
|
return typeof value === "string" && value.trim() ? value : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNestedString(
|
||||||
|
record: Record<string, unknown>,
|
||||||
|
first: string,
|
||||||
|
second: string,
|
||||||
|
): string | null {
|
||||||
|
const nested = asRecord(record[first]);
|
||||||
|
return getStringField(nested, second);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getArchiveDerivedIntent(derived: EvalArchiveEntry["derived"]): string {
|
||||||
|
const record = asRecord(derived);
|
||||||
|
return getStringField(record, "intent") ??
|
||||||
|
getNestedString(record, "intent", "kind") ??
|
||||||
|
"unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getArchiveDerivedPhase(derived: EvalArchiveEntry["derived"]): string {
|
||||||
|
const record = asRecord(derived);
|
||||||
|
return getStringField(record, "phase") ??
|
||||||
|
getNestedString(record, "phase", "kind") ??
|
||||||
|
"unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getArchiveRoutingDecision(
|
||||||
|
derived: EvalArchiveEntry["derived"],
|
||||||
|
): string | null {
|
||||||
|
const record = asRecord(derived);
|
||||||
|
return getNestedString(record, "routingDecision", "kind");
|
||||||
|
}
|
||||||
|
|
||||||
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
|
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
|
||||||
if (!value?.trim()) {
|
if (!value?.trim()) {
|
||||||
return new Date(nowMs);
|
return new Date(nowMs);
|
||||||
@@ -164,13 +204,13 @@ function calculatePercentile(values: number[], percentile: number): number | nul
|
|||||||
|
|
||||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||||
const abstention = getAbstention(entry);
|
const abstention = getAbstention(entry);
|
||||||
|
const routingDecision = getArchiveRoutingDecision(entry.derived);
|
||||||
return {
|
return {
|
||||||
|
skill: entry.skill || entry.request.skill || "onboarding",
|
||||||
tier: entry.tier,
|
tier: entry.tier,
|
||||||
intent: entry.derived.intent.kind,
|
intent: getArchiveDerivedIntent(entry.derived),
|
||||||
phase: entry.derived.phase.kind,
|
phase: getArchiveDerivedPhase(entry.derived),
|
||||||
...(entry.derived.routingDecision
|
...(routingDecision ? { routingDecision } : {}),
|
||||||
? { routingDecision: entry.derived.routingDecision.kind }
|
|
||||||
: {}),
|
|
||||||
...(abstention.action ? { abstentionAction: abstention.action } : {}),
|
...(abstention.action ? { abstentionAction: abstention.action } : {}),
|
||||||
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
|
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
|
||||||
};
|
};
|
||||||
@@ -192,7 +232,7 @@ function parseEvalArchiveEntry(
|
|||||||
entry.message["input"],
|
entry.message["input"],
|
||||||
"input",
|
"input",
|
||||||
);
|
);
|
||||||
const derived = parseJsonField<DerivedState>(
|
const derived = parseJsonField<DerivedState | EvalDerivedSnapshot>(
|
||||||
entry.message["derived"],
|
entry.message["derived"],
|
||||||
"derived",
|
"derived",
|
||||||
);
|
);
|
||||||
@@ -208,11 +248,12 @@ function parseEvalArchiveEntry(
|
|||||||
|
|
||||||
const archiveEntry: EvalArchiveEntry = {
|
const archiveEntry: EvalArchiveEntry = {
|
||||||
streamId: entry.id,
|
streamId: entry.id,
|
||||||
|
skill: entry.message["skill"]?.trim() || input.request.skill || "onboarding",
|
||||||
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
|
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
|
||||||
request: input.request,
|
request: input.request,
|
||||||
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
|
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
|
||||||
derived,
|
derived,
|
||||||
classified: parseJsonField<ClassifiedIntent>(
|
classified: parseJsonField<ClassifiedIntent | EvalClassifiedSnapshot>(
|
||||||
entry.message["classified"],
|
entry.message["classified"],
|
||||||
"classified",
|
"classified",
|
||||||
),
|
),
|
||||||
@@ -248,9 +289,12 @@ async function ensureEvalArchiveIndexes(
|
|||||||
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
|
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
|
||||||
),
|
),
|
||||||
collection.createIndex({
|
collection.createIndex({
|
||||||
|
skill: 1,
|
||||||
tier: 1,
|
tier: 1,
|
||||||
"derived.intent.kind": 1,
|
"derived.intent.kind": 1,
|
||||||
|
"derived.intent": 1,
|
||||||
"derived.phase.kind": 1,
|
"derived.phase.kind": 1,
|
||||||
|
"derived.phase": 1,
|
||||||
evaluatedAt: -1,
|
evaluatedAt: -1,
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
|
|||||||
+12
-1
@@ -65,6 +65,7 @@ function parseSlice(value: string | undefined): SliceKey {
|
|||||||
const tier = Number(fields["tier"]);
|
const tier = Number(fields["tier"]);
|
||||||
const intent = fields["intent"];
|
const intent = fields["intent"];
|
||||||
const phase = fields["phase"];
|
const phase = fields["phase"];
|
||||||
|
const skill = fields["skill"]?.trim();
|
||||||
|
|
||||||
if (
|
if (
|
||||||
(tier !== 0 && tier !== 1 && tier !== 2) ||
|
(tier !== 0 && tier !== 1 && tier !== 2) ||
|
||||||
@@ -77,6 +78,7 @@ function parseSlice(value: string | undefined): SliceKey {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
...(skill ? { skill } : {}),
|
||||||
tier,
|
tier,
|
||||||
intent,
|
intent,
|
||||||
phase,
|
phase,
|
||||||
@@ -620,9 +622,14 @@ async function main(): Promise<void> {
|
|||||||
} = await import("./loop/propose.js");
|
} = await import("./loop/propose.js");
|
||||||
const mongo = await connectMongoForEval();
|
const mongo = await connectMongoForEval();
|
||||||
try {
|
try {
|
||||||
|
const slice = parseSlice(getFlagString(flags, "slice"));
|
||||||
|
const skill = getFlagString(flags, "skill");
|
||||||
const input = await buildProposalInputFromArchive(
|
const input = await buildProposalInputFromArchive(
|
||||||
mongo.connection,
|
mongo.connection,
|
||||||
parseSlice(getFlagString(flags, "slice")),
|
{
|
||||||
|
...slice,
|
||||||
|
...(skill ? { skill } : {}),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
force: hasFlag(flags, "force"),
|
force: hasFlag(flags, "force"),
|
||||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||||
@@ -668,13 +675,16 @@ async function main(): Promise<void> {
|
|||||||
const mongo = await connectMongoForEval();
|
const mongo = await connectMongoForEval();
|
||||||
try {
|
try {
|
||||||
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
|
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
|
||||||
|
const skill = getFlagString(flags, "skill");
|
||||||
const cases = await loadTournamentCases(mongo.connection, {
|
const cases = await loadTournamentCases(mongo.connection, {
|
||||||
caseIds: parseCsv(getFlagString(flags, "case-ids")),
|
caseIds: parseCsv(getFlagString(flags, "case-ids")),
|
||||||
limit: maxMatches,
|
limit: maxMatches,
|
||||||
|
skill,
|
||||||
});
|
});
|
||||||
const result = await runTournament(createEvalLlmClient(), {
|
const result = await runTournament(createEvalLlmClient(), {
|
||||||
cases,
|
cases,
|
||||||
maxMatches,
|
maxMatches,
|
||||||
|
skill,
|
||||||
variantIds,
|
variantIds,
|
||||||
});
|
});
|
||||||
await saveTournamentResult(mongo.connection, result);
|
await saveTournamentResult(mongo.connection, result);
|
||||||
@@ -691,6 +701,7 @@ async function main(): Promise<void> {
|
|||||||
try {
|
try {
|
||||||
const results = await queryTournamentResults(mongo.connection, {
|
const results = await queryTournamentResults(mongo.connection, {
|
||||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||||
|
skill: getFlagString(flags, "skill"),
|
||||||
variantId: getFlagString(flags, "variant"),
|
variantId: getFlagString(flags, "variant"),
|
||||||
});
|
});
|
||||||
for (const result of results) {
|
for (const result of results) {
|
||||||
|
|||||||
+12
-6
@@ -3,7 +3,12 @@ import type {
|
|||||||
EvalArchiveEntry,
|
EvalArchiveEntry,
|
||||||
MongoConnectionLike,
|
MongoConnectionLike,
|
||||||
} from "./archive.js";
|
} from "./archive.js";
|
||||||
import { getEvalArchiveCollection } from "./archive.js";
|
import {
|
||||||
|
getArchiveDerivedIntent,
|
||||||
|
getArchiveDerivedPhase,
|
||||||
|
getArchiveRoutingDecision,
|
||||||
|
getEvalArchiveCollection,
|
||||||
|
} from "./archive.js";
|
||||||
import type { SliceKey } from "./aggregator.js";
|
import type { SliceKey } from "./aggregator.js";
|
||||||
|
|
||||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
@@ -41,6 +46,7 @@ function average(values: number[]): number | null {
|
|||||||
|
|
||||||
function sliceKeyToString(slice: SliceKey): string {
|
function sliceKeyToString(slice: SliceKey): string {
|
||||||
return [
|
return [
|
||||||
|
slice.skill ?? "onboarding",
|
||||||
slice.tier,
|
slice.tier,
|
||||||
slice.intent,
|
slice.intent,
|
||||||
slice.phase,
|
slice.phase,
|
||||||
@@ -51,13 +57,13 @@ function sliceKeyToString(slice: SliceKey): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||||
|
const routingDecision = getArchiveRoutingDecision(entry.derived);
|
||||||
return {
|
return {
|
||||||
|
skill: entry.skill || entry.request.skill || "onboarding",
|
||||||
tier: entry.tier,
|
tier: entry.tier,
|
||||||
intent: entry.derived.intent.kind,
|
intent: getArchiveDerivedIntent(entry.derived),
|
||||||
phase: entry.derived.phase.kind,
|
phase: getArchiveDerivedPhase(entry.derived),
|
||||||
...(entry.derived.routingDecision
|
...(routingDecision ? { routingDecision } : {}),
|
||||||
? { routingDecision: entry.derived.routingDecision.kind }
|
|
||||||
: {}),
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||||
import { SkillRegistry } from "@skills/skill-loader";
|
import { SkillRegistry } from "@skills/skill-loader";
|
||||||
import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
import { createModerationToolRegistry } from "@skills/moderation/tools";
|
||||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
|
||||||
import { runG1Static, runG4Permission } from "./gates.js";
|
import { runG1Static, runG4Permission } from "./gates.js";
|
||||||
|
|
||||||
function manifest(
|
function manifest(
|
||||||
@@ -69,7 +70,7 @@ describe("governance gates", () => {
|
|||||||
|
|
||||||
it("checks required tool and resource permissions in G4", () => {
|
it("checks required tool and resource permissions in G4", () => {
|
||||||
const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`);
|
const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`);
|
||||||
const toolRegistry = createBuiltInToolRegistry();
|
const toolRegistry = createOnboardingToolRegistry();
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
runG4Permission(
|
runG4Permission(
|
||||||
@@ -80,6 +81,7 @@ describe("governance gates", () => {
|
|||||||
} as PromptVariantManifest,
|
} as PromptVariantManifest,
|
||||||
toolRegistry,
|
toolRegistry,
|
||||||
skillRegistry,
|
skillRegistry,
|
||||||
|
"onboarding",
|
||||||
),
|
),
|
||||||
).toMatchObject({
|
).toMatchObject({
|
||||||
passed: true,
|
passed: true,
|
||||||
@@ -94,10 +96,31 @@ describe("governance gates", () => {
|
|||||||
} as PromptVariantManifest,
|
} as PromptVariantManifest,
|
||||||
toolRegistry,
|
toolRegistry,
|
||||||
skillRegistry,
|
skillRegistry,
|
||||||
|
"onboarding",
|
||||||
),
|
),
|
||||||
).toMatchObject({
|
).toMatchObject({
|
||||||
passed: false,
|
passed: false,
|
||||||
reason: "missing-required-capability",
|
reason: "missing-required-capability",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("checks moderation resources against the target skill", () => {
|
||||||
|
const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
runG4Permission(
|
||||||
|
{
|
||||||
|
...manifest(),
|
||||||
|
requiredResources: ["community-rules"],
|
||||||
|
requiredTools: ["lookup-rule"],
|
||||||
|
skill: "moderation",
|
||||||
|
} as PromptVariantManifest,
|
||||||
|
createModerationToolRegistry(),
|
||||||
|
skillRegistry,
|
||||||
|
"moderation",
|
||||||
|
),
|
||||||
|
).toMatchObject({
|
||||||
|
passed: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -1,5 +1,4 @@
|
|||||||
import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||||
import type { RuntimeFlagManifest } from "@skills/onboarding/runtime/runtime-flag-manifest";
|
|
||||||
import type { SkillRegistry } from "@skills/skill-loader";
|
import type { SkillRegistry } from "@skills/skill-loader";
|
||||||
import type { ToolRegistry } from "@tools/registry";
|
import type { ToolRegistry } from "@tools/registry";
|
||||||
|
|
||||||
@@ -10,6 +9,12 @@ export interface GateResult {
|
|||||||
reason?: string | undefined;
|
reason?: string | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface RuntimeFlagManifest {
|
||||||
|
description: string;
|
||||||
|
flags: Record<string, unknown>;
|
||||||
|
variantId: string;
|
||||||
|
}
|
||||||
|
|
||||||
type GovernedManifest = PromptVariantManifest | RuntimeFlagManifest;
|
type GovernedManifest = PromptVariantManifest | RuntimeFlagManifest;
|
||||||
|
|
||||||
const riskyPatterns: Array<{ pattern: RegExp; reason: string }> = [
|
const riskyPatterns: Array<{ pattern: RegExp; reason: string }> = [
|
||||||
@@ -94,14 +99,15 @@ export function runG4Permission(
|
|||||||
manifest: GovernedManifest,
|
manifest: GovernedManifest,
|
||||||
toolRegistry: ToolRegistry,
|
toolRegistry: ToolRegistry,
|
||||||
skillRegistry: SkillRegistry,
|
skillRegistry: SkillRegistry,
|
||||||
|
targetSkill: string,
|
||||||
): GateResult {
|
): GateResult {
|
||||||
const record = asRecord(manifest);
|
const record = asRecord(manifest);
|
||||||
const requiredTools = readStringArray(record, "requiredTools");
|
const requiredTools = readStringArray(record, "requiredTools");
|
||||||
const requiredResources = readStringArray(record, "requiredResources");
|
const requiredResources = readStringArray(record, "requiredResources");
|
||||||
const missingTools = requiredTools.filter((toolName) => !toolRegistry.has(toolName));
|
const missingTools = requiredTools.filter((toolName) => !toolRegistry.has(toolName));
|
||||||
const onboarding = skillRegistry.get("onboarding");
|
const skillManifest = skillRegistry.get(targetSkill);
|
||||||
const missingResources = requiredResources.filter(
|
const missingResources = requiredResources.filter(
|
||||||
(resource) => !onboarding?.resources[resource],
|
(resource) => !skillManifest?.resources[resource],
|
||||||
);
|
);
|
||||||
|
|
||||||
if (missingTools.length > 0 || missingResources.length > 0) {
|
if (missingTools.length > 0 || missingResources.length > 0) {
|
||||||
|
|||||||
@@ -2,7 +2,13 @@ import type {
|
|||||||
ProcessTraceSnapshot,
|
ProcessTraceSnapshot,
|
||||||
ProcessTraceStepName,
|
ProcessTraceStepName,
|
||||||
} from "@skills/onboarding/runtime/trace";
|
} from "@skills/onboarding/runtime/trace";
|
||||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
import {
|
||||||
|
getArchiveDerivedIntent,
|
||||||
|
getArchiveDerivedPhase,
|
||||||
|
getArchiveRoutingDecision,
|
||||||
|
type EvalArchiveEntry,
|
||||||
|
type MongoConnectionLike,
|
||||||
|
} from "../archive.js";
|
||||||
import type { SliceKey } from "../aggregator.js";
|
import type { SliceKey } from "../aggregator.js";
|
||||||
import type { PrmModel, PrmFeatureVector } from "./prm.js";
|
import type { PrmModel, PrmFeatureVector } from "./prm.js";
|
||||||
import {
|
import {
|
||||||
@@ -136,14 +142,14 @@ export function attributeTraceWithPrm(
|
|||||||
function sliceFromEntry(entry: EvalArchiveEntry): SliceKey {
|
function sliceFromEntry(entry: EvalArchiveEntry): SliceKey {
|
||||||
const abstention = (entry.process as ProcessTraceSnapshot | undefined)
|
const abstention = (entry.process as ProcessTraceSnapshot | undefined)
|
||||||
?.steps.classify?.output.abstention;
|
?.steps.classify?.output.abstention;
|
||||||
|
const routingDecision = getArchiveRoutingDecision(entry.derived);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
skill: entry.skill || entry.request.skill || "onboarding",
|
||||||
tier: entry.tier,
|
tier: entry.tier,
|
||||||
intent: entry.derived.intent.kind,
|
intent: getArchiveDerivedIntent(entry.derived),
|
||||||
phase: entry.derived.phase.kind,
|
phase: getArchiveDerivedPhase(entry.derived),
|
||||||
...(entry.derived.routingDecision
|
...(routingDecision ? { routingDecision } : {}),
|
||||||
? { routingDecision: entry.derived.routingDecision.kind }
|
|
||||||
: {}),
|
|
||||||
...(abstention?.action.kind !== undefined &&
|
...(abstention?.action.kind !== undefined &&
|
||||||
abstention.action.kind !== "proceed"
|
abstention.action.kind !== "proceed"
|
||||||
? { abstentionAction: abstention.action.kind }
|
? { abstentionAction: abstention.action.kind }
|
||||||
|
|||||||
@@ -7,29 +7,36 @@ import type {
|
|||||||
LlmJudgeResult,
|
LlmJudgeResult,
|
||||||
LlmJudgeScores,
|
LlmJudgeScores,
|
||||||
} from "../types.js";
|
} from "../types.js";
|
||||||
|
import { getSkillJudgeConfig } from "./skill-config.js";
|
||||||
const llmJudgeSchema = z.object({
|
|
||||||
friendliness: z.number().int().min(1).max(5),
|
|
||||||
accuracy: z.number().int().min(1).max(5),
|
|
||||||
concision: z.number().int().min(1).max(5),
|
|
||||||
scope: z.number().int().min(1).max(5),
|
|
||||||
rationale: z.string().trim().min(1).max(200),
|
|
||||||
});
|
|
||||||
|
|
||||||
function round(value: number): number {
|
function round(value: number): number {
|
||||||
return Number(value.toFixed(3));
|
return Number(value.toFixed(3));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createJudgeSchema(axes: readonly string[]): z.ZodObject<z.ZodRawShape> {
|
||||||
|
return z.object({
|
||||||
|
...Object.fromEntries(
|
||||||
|
axes.map((axis) => [axis, z.number().int().min(1).max(5)]),
|
||||||
|
),
|
||||||
|
rationale: z.string().trim().min(1).max(200),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function buildJudgePrompt(
|
function buildJudgePrompt(
|
||||||
|
skillName: string,
|
||||||
evalCase: EvalCaseDefinition,
|
evalCase: EvalCaseDefinition,
|
||||||
responseText: string,
|
responseText: string,
|
||||||
derived: EvalDerivedSnapshot,
|
derived: EvalDerivedSnapshot,
|
||||||
): string {
|
): string {
|
||||||
|
const config = getSkillJudgeConfig(skillName);
|
||||||
|
const axes = config.axes.join(", ");
|
||||||
|
const scoreShape = Object.fromEntries(config.axes.map((axis) => [axis, 1]));
|
||||||
return [
|
return [
|
||||||
"다음 온보딩 에이전트 응답을 평가해 주세요.",
|
`다음 ${skillName} 에이전트 응답을 평가해 주세요.`,
|
||||||
"평가 축은 friendliness, accuracy, concision, scope 입니다.",
|
`평가 축은 ${axes} 입니다.`,
|
||||||
"각 점수는 1~5의 정수만 사용하세요.",
|
"각 점수는 1~5의 정수만 사용하세요.",
|
||||||
"scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.",
|
config.promptTemplate ??
|
||||||
|
"accuracy는 기대 조건과 실제 답변의 사실성을, concision은 과도한 부가 설명 여부를 포함해 평가하세요.",
|
||||||
"",
|
"",
|
||||||
"[request]",
|
"[request]",
|
||||||
JSON.stringify(evalCase.input.request),
|
JSON.stringify(evalCase.input.request),
|
||||||
@@ -48,12 +55,13 @@ function buildJudgePrompt(
|
|||||||
"[assistant_response]",
|
"[assistant_response]",
|
||||||
responseText,
|
responseText,
|
||||||
"",
|
"",
|
||||||
'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}',
|
`JSON object only: ${JSON.stringify({ ...scoreShape, rationale: "..." })}`,
|
||||||
].join("\n");
|
].join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function judgeOnboardingWithLlm(
|
export async function judgeSkillWithLlm(
|
||||||
llmClient: LlmClient,
|
llmClient: LlmClient,
|
||||||
|
skillName: string,
|
||||||
evalCase: EvalCaseDefinition,
|
evalCase: EvalCaseDefinition,
|
||||||
responseText: string,
|
responseText: string,
|
||||||
derived: EvalDerivedSnapshot,
|
derived: EvalDerivedSnapshot,
|
||||||
@@ -77,11 +85,11 @@ export async function judgeOnboardingWithLlm(
|
|||||||
{
|
{
|
||||||
role: "system",
|
role: "system",
|
||||||
content:
|
content:
|
||||||
"당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
|
`당신은 ${skillName} 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
role: "user",
|
role: "user",
|
||||||
content: buildJudgePrompt(evalCase, responseText, derived),
|
content: buildJudgePrompt(skillName, evalCase, responseText, derived),
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
{
|
{
|
||||||
@@ -100,18 +108,18 @@ export async function judgeOnboardingWithLlm(
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const parsed = llmJudgeSchema.parse(JSON.parse(response.content));
|
const config = getSkillJudgeConfig(skillName);
|
||||||
|
const parsed = createJudgeSchema(config.axes).parse(
|
||||||
|
JSON.parse(response.content),
|
||||||
|
);
|
||||||
|
const axisScores = Object.fromEntries(
|
||||||
|
config.axes.map((axis) => [axis, Number(parsed[axis])]),
|
||||||
|
);
|
||||||
const scores: LlmJudgeScores = {
|
const scores: LlmJudgeScores = {
|
||||||
friendliness: parsed.friendliness,
|
...axisScores,
|
||||||
accuracy: parsed.accuracy,
|
|
||||||
concision: parsed.concision,
|
|
||||||
scope: parsed.scope,
|
|
||||||
overall: round(
|
overall: round(
|
||||||
(parsed.friendliness +
|
config.axes.reduce((sum, axis) => sum + Number(parsed[axis]), 0) /
|
||||||
parsed.accuracy +
|
config.axes.length,
|
||||||
parsed.concision +
|
|
||||||
parsed.scope) /
|
|
||||||
4,
|
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -121,7 +129,7 @@ export async function judgeOnboardingWithLlm(
|
|||||||
endpoint: response.endpoint,
|
endpoint: response.endpoint,
|
||||||
provider: response.provider,
|
provider: response.provider,
|
||||||
scores,
|
scores,
|
||||||
rationale: parsed.rationale,
|
rationale: String(parsed["rationale"]),
|
||||||
raw: response.content,
|
raw: response.content,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
@@ -132,3 +140,18 @@ export async function judgeOnboardingWithLlm(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function judgeOnboardingWithLlm(
|
||||||
|
llmClient: LlmClient,
|
||||||
|
evalCase: EvalCaseDefinition,
|
||||||
|
responseText: string,
|
||||||
|
derived: EvalDerivedSnapshot,
|
||||||
|
): Promise<LlmJudgeResult> {
|
||||||
|
return judgeSkillWithLlm(
|
||||||
|
llmClient,
|
||||||
|
"onboarding",
|
||||||
|
evalCase,
|
||||||
|
responseText,
|
||||||
|
derived,
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";
|
|||||||
import { afterEach, describe, expect, it } from "vitest";
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
|
|
||||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||||
|
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
|
||||||
import { judgeProcessRule } from "./process-judge.js";
|
import { judgeProcessRule } from "./process-judge.js";
|
||||||
import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js";
|
import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js";
|
||||||
|
|
||||||
@@ -86,6 +87,8 @@ function makeTrace(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("judgeProcessRule", () => {
|
describe("judgeProcessRule", () => {
|
||||||
|
const toolRegistry = createOnboardingToolRegistry();
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
delete process.env["EVAL_PRM_ENABLED"];
|
delete process.env["EVAL_PRM_ENABLED"];
|
||||||
delete process.env["EVAL_PRM_WEIGHTS_PATH"];
|
delete process.env["EVAL_PRM_WEIGHTS_PATH"];
|
||||||
@@ -102,6 +105,7 @@ describe("judgeProcessRule", () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
makeTrace(),
|
makeTrace(),
|
||||||
|
toolRegistry,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.score).toBe(1);
|
expect(result.score).toBe(1);
|
||||||
@@ -122,6 +126,7 @@ describe("judgeProcessRule", () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
makeTrace(),
|
makeTrace(),
|
||||||
|
toolRegistry,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
@@ -155,6 +160,7 @@ describe("judgeProcessRule", () => {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
|
toolRegistry,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
@@ -180,6 +186,7 @@ describe("judgeProcessRule", () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
toolRegistry,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
@@ -214,6 +221,7 @@ describe("judgeProcessRule", () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
toolRegistry,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
@@ -226,7 +234,7 @@ describe("judgeProcessRule", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("skips optional expectations when a golden case has not opted in", () => {
|
it("skips optional expectations when a golden case has not opted in", () => {
|
||||||
const result = judgeProcessRule({}, makeTrace());
|
const result = judgeProcessRule({}, makeTrace(), toolRegistry);
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
result.checks.find((check) => check.rule === "intent-match")?.skipped,
|
result.checks.find((check) => check.rule === "intent-match")?.skipped,
|
||||||
@@ -261,6 +269,7 @@ describe("judgeProcessRule", () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
toolRegistry,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.prmFeatures).toMatchObject({
|
expect(result.prmFeatures).toMatchObject({
|
||||||
|
|||||||
@@ -2,8 +2,7 @@ import { z } from "zod";
|
|||||||
|
|
||||||
import type { AgentRequest } from "@agent/types";
|
import type { AgentRequest } from "@agent/types";
|
||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
import type { ToolRegistry } from "@tools/registry";
|
||||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
|
||||||
import {
|
import {
|
||||||
getEvalProcessLlmJudgeMaxTokens,
|
getEvalProcessLlmJudgeMaxTokens,
|
||||||
getEvalPrmEnabled,
|
getEvalPrmEnabled,
|
||||||
@@ -12,6 +11,7 @@ import {
|
|||||||
} from "../../config.js";
|
} from "../../config.js";
|
||||||
import type {
|
import type {
|
||||||
EvalCaseExpected,
|
EvalCaseExpected,
|
||||||
|
EvalProcessTraceSnapshot,
|
||||||
ProcessJudgeCheck,
|
ProcessJudgeCheck,
|
||||||
ProcessJudgeResult,
|
ProcessJudgeResult,
|
||||||
} from "../types.js";
|
} from "../types.js";
|
||||||
@@ -80,7 +80,7 @@ function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudge
|
|||||||
|
|
||||||
function enrichWithPrm(
|
function enrichWithPrm(
|
||||||
result: ProcessJudgeResult,
|
result: ProcessJudgeResult,
|
||||||
trace: ProcessTraceSnapshot,
|
trace: EvalProcessTraceSnapshot,
|
||||||
): ProcessJudgeResult {
|
): ProcessJudgeResult {
|
||||||
if (!getEvalPrmEnabled()) {
|
if (!getEvalPrmEnabled()) {
|
||||||
return result;
|
return result;
|
||||||
@@ -125,7 +125,7 @@ function hasAllTools(
|
|||||||
return subset && actual.size === expected.size;
|
return subset && actual.size === expected.size;
|
||||||
}
|
}
|
||||||
|
|
||||||
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
function countRecordedSteps(trace: EvalProcessTraceSnapshot): number {
|
||||||
const scalarSteps = [
|
const scalarSteps = [
|
||||||
trace.steps.classify,
|
trace.steps.classify,
|
||||||
trace.steps.derive,
|
trace.steps.derive,
|
||||||
@@ -138,18 +138,20 @@ function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
|||||||
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
|
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
function validateToolArgs(
|
||||||
|
trace: EvalProcessTraceSnapshot,
|
||||||
|
toolRegistry: ToolRegistry,
|
||||||
|
): ProcessJudgeCheck {
|
||||||
const toolCalls = trace.steps.toolCalls ?? [];
|
const toolCalls = trace.steps.toolCalls ?? [];
|
||||||
if (toolCalls.length === 0) {
|
if (toolCalls.length === 0) {
|
||||||
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
|
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
|
||||||
}
|
}
|
||||||
|
|
||||||
const registry = createBuiltInToolRegistry();
|
|
||||||
const failures: string[] = [];
|
const failures: string[] = [];
|
||||||
|
|
||||||
for (const call of toolCalls) {
|
for (const call of toolCalls) {
|
||||||
try {
|
try {
|
||||||
registry.get(call.output.name).argsSchema.parse(call.output.args);
|
toolRegistry.get(call.output.name).argsSchema.parse(call.output.args);
|
||||||
} catch {
|
} catch {
|
||||||
failures.push(call.output.name);
|
failures.push(call.output.name);
|
||||||
}
|
}
|
||||||
@@ -167,7 +169,8 @@ function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
|||||||
|
|
||||||
export function judgeProcessRule(
|
export function judgeProcessRule(
|
||||||
expected: EvalCaseExpected,
|
expected: EvalCaseExpected,
|
||||||
trace: ProcessTraceSnapshot,
|
trace: EvalProcessTraceSnapshot,
|
||||||
|
toolRegistry: ToolRegistry,
|
||||||
): ProcessJudgeResult {
|
): ProcessJudgeResult {
|
||||||
const checks: ProcessJudgeCheck[] = [];
|
const checks: ProcessJudgeCheck[] = [];
|
||||||
|
|
||||||
@@ -244,7 +247,7 @@ export function judgeProcessRule(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
checks.push(validateToolArgs(trace));
|
checks.push(validateToolArgs(trace, toolRegistry));
|
||||||
|
|
||||||
const toolPlanStepCount = trace.steps.plan?.output.toolPlanStepCount;
|
const toolPlanStepCount = trace.steps.plan?.output.toolPlanStepCount;
|
||||||
if (toolPlanStepCount !== undefined) {
|
if (toolPlanStepCount !== undefined) {
|
||||||
@@ -318,7 +321,7 @@ export function judgeProcessRule(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildProcessJudgePrompt(
|
function buildProcessJudgePrompt(
|
||||||
trace: ProcessTraceSnapshot,
|
trace: EvalProcessTraceSnapshot,
|
||||||
options: { finalText: string; request: AgentRequest },
|
options: { finalText: string; request: AgentRequest },
|
||||||
): string {
|
): string {
|
||||||
return [
|
return [
|
||||||
@@ -341,7 +344,7 @@ function buildProcessJudgePrompt(
|
|||||||
|
|
||||||
export async function judgeProcessWithLlm(
|
export async function judgeProcessWithLlm(
|
||||||
llmClient: LlmClient,
|
llmClient: LlmClient,
|
||||||
trace: ProcessTraceSnapshot,
|
trace: EvalProcessTraceSnapshot,
|
||||||
options: { finalText: string; request: AgentRequest },
|
options: { finalText: string; request: AgentRequest },
|
||||||
): Promise<ProcessJudgeResult> {
|
): Promise<ProcessJudgeResult> {
|
||||||
if (!llmClient.hasEndpoints()) {
|
if (!llmClient.hasEndpoints()) {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
import { judgeOnboardingRuleCase } from "./rule-judge.js";
|
import { judgeRuleCase } from "./rule-judge.js";
|
||||||
|
import { getSkillJudgeConfig } from "./skill-config.js";
|
||||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||||
|
|
||||||
function makeDerived(
|
function makeDerived(
|
||||||
@@ -18,9 +19,12 @@ function makeDerived(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("judgeOnboardingRuleCase", () => {
|
describe("judgeRuleCase", () => {
|
||||||
|
const validator = getSkillJudgeConfig("onboarding").validator;
|
||||||
|
|
||||||
it("passes when active checks match", () => {
|
it("passes when active checks match", () => {
|
||||||
const result = judgeOnboardingRuleCase(
|
const derived = makeDerived();
|
||||||
|
const result = judgeRuleCase(
|
||||||
{
|
{
|
||||||
intent: "status-question",
|
intent: "status-question",
|
||||||
responseMode: "llm",
|
responseMode: "llm",
|
||||||
@@ -28,7 +32,12 @@ describe("judgeOnboardingRuleCase", () => {
|
|||||||
mustNotInclude: ["디스코드 서버"],
|
mustNotInclude: ["디스코드 서버"],
|
||||||
},
|
},
|
||||||
"이제 방 하트 인증만 남았어요.",
|
"이제 방 하트 인증만 남았어요.",
|
||||||
makeDerived(),
|
{
|
||||||
|
derived,
|
||||||
|
intent: derived.intent.kind,
|
||||||
|
responseMode: derived.responseMode,
|
||||||
|
validator,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.passed).toBe(true);
|
expect(result.passed).toBe(true);
|
||||||
@@ -37,13 +46,19 @@ describe("judgeOnboardingRuleCase", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("fails intent and mustNotInclude mismatches", () => {
|
it("fails intent and mustNotInclude mismatches", () => {
|
||||||
const result = judgeOnboardingRuleCase(
|
const derived = makeDerived();
|
||||||
|
const result = judgeRuleCase(
|
||||||
{
|
{
|
||||||
intent: "help-how",
|
intent: "help-how",
|
||||||
mustNotInclude: ["디스코드"],
|
mustNotInclude: ["디스코드"],
|
||||||
},
|
},
|
||||||
"디스코드 서버에 먼저 들어오시면 돼요.",
|
"디스코드 서버에 먼저 들어오시면 돼요.",
|
||||||
makeDerived(),
|
{
|
||||||
|
derived,
|
||||||
|
intent: derived.intent.kind,
|
||||||
|
responseMode: derived.responseMode,
|
||||||
|
validator,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.passed).toBe(false);
|
expect(result.passed).toBe(false);
|
||||||
@@ -56,12 +71,18 @@ describe("judgeOnboardingRuleCase", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("treats rejecting validator results as a failed rule check", () => {
|
it("treats rejecting validator results as a failed rule check", () => {
|
||||||
const result = judgeOnboardingRuleCase(
|
const derived = makeDerived({
|
||||||
|
intent: { kind: "photo-question" },
|
||||||
|
});
|
||||||
|
const result = judgeRuleCase(
|
||||||
{},
|
{},
|
||||||
"디스코드 서버에 들어가시면 돼요.",
|
"디스코드 서버에 들어가시면 돼요.",
|
||||||
makeDerived({
|
{
|
||||||
intent: { kind: "photo-question" },
|
derived,
|
||||||
}),
|
intent: derived.intent.kind,
|
||||||
|
responseMode: derived.responseMode,
|
||||||
|
validator,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.passed).toBe(false);
|
expect(result.passed).toBe(false);
|
||||||
|
|||||||
+26
-120
@@ -1,6 +1,5 @@
|
|||||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
|
||||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
|
||||||
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
|
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
|
||||||
|
import type { SkillJudgeConfig } from "./skill-config.js";
|
||||||
|
|
||||||
function normalizeText(value: string): string {
|
function normalizeText(value: string): string {
|
||||||
return value.trim().toLowerCase();
|
return value.trim().toLowerCase();
|
||||||
@@ -14,18 +13,35 @@ export function judgeRuleCase(
|
|||||||
expected: EvalCaseExpected,
|
expected: EvalCaseExpected,
|
||||||
responseText: string,
|
responseText: string,
|
||||||
actual: {
|
actual: {
|
||||||
|
derived?: unknown;
|
||||||
intent: string;
|
intent: string;
|
||||||
responseMode?: string | undefined;
|
responseMode?: string | undefined;
|
||||||
|
validator?: SkillJudgeConfig["validator"] | undefined;
|
||||||
},
|
},
|
||||||
): RuleJudgeResult {
|
): RuleJudgeResult {
|
||||||
const checks: RuleJudgeCheck[] = [];
|
const checks: RuleJudgeCheck[] = [];
|
||||||
|
const validation = actual.validator
|
||||||
|
? actual.validator(responseText, actual.derived)
|
||||||
|
: null;
|
||||||
|
|
||||||
checks.push({
|
if (validation) {
|
||||||
name: "validator",
|
checks.push({
|
||||||
passed: true,
|
name: "validator",
|
||||||
skipped: true,
|
passed: validation.status !== "reject",
|
||||||
detail: "not configured for this skill",
|
skipped: false,
|
||||||
});
|
detail:
|
||||||
|
validation.reasons.length > 0
|
||||||
|
? validation.reasons.join(", ")
|
||||||
|
: validation.status,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
checks.push({
|
||||||
|
name: "validator",
|
||||||
|
passed: true,
|
||||||
|
skipped: true,
|
||||||
|
detail: "not configured for this skill",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const expectedIntent = expected.intent?.trim();
|
const expectedIntent = expected.intent?.trim();
|
||||||
if (expectedIntent) {
|
if (expectedIntent) {
|
||||||
@@ -112,117 +128,7 @@ export function judgeRuleCase(
|
|||||||
passed: activeChecks.every((check) => check.passed),
|
passed: activeChecks.every((check) => check.passed),
|
||||||
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
||||||
checks,
|
checks,
|
||||||
validatorStatus: "pass",
|
validatorStatus: validation?.status ?? "pass",
|
||||||
validatorReasons: [],
|
validatorReasons: validation?.reasons ?? [],
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function judgeOnboardingRuleCase(
|
|
||||||
expected: EvalCaseExpected,
|
|
||||||
responseText: string,
|
|
||||||
derived: DerivedState,
|
|
||||||
): RuleJudgeResult {
|
|
||||||
const validation = validateOnboardingResponse(responseText, derived, {
|
|
||||||
allowShortResponse: derived.allowShortResponse,
|
|
||||||
});
|
|
||||||
const checks: RuleJudgeCheck[] = [];
|
|
||||||
|
|
||||||
checks.push({
|
|
||||||
name: "validator",
|
|
||||||
passed: validation.status !== "reject",
|
|
||||||
skipped: false,
|
|
||||||
detail:
|
|
||||||
validation.reasons.length > 0
|
|
||||||
? validation.reasons.join(", ")
|
|
||||||
: validation.status,
|
|
||||||
});
|
|
||||||
|
|
||||||
const expectedIntent = expected.intent?.trim();
|
|
||||||
if (expectedIntent) {
|
|
||||||
checks.push({
|
|
||||||
name: "intent",
|
|
||||||
passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind),
|
|
||||||
skipped: false,
|
|
||||||
detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
checks.push({
|
|
||||||
name: "intent",
|
|
||||||
passed: true,
|
|
||||||
skipped: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const expectedResponseMode = expected.responseMode?.trim();
|
|
||||||
if (expectedResponseMode) {
|
|
||||||
checks.push({
|
|
||||||
name: "responseMode",
|
|
||||||
passed:
|
|
||||||
normalizeText(expectedResponseMode) ===
|
|
||||||
normalizeText(derived.responseMode),
|
|
||||||
skipped: false,
|
|
||||||
detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
checks.push({
|
|
||||||
name: "responseMode",
|
|
||||||
passed: true,
|
|
||||||
skipped: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
|
|
||||||
if (mustIncludeAny.length > 0) {
|
|
||||||
const normalizedResponse = normalizeText(responseText);
|
|
||||||
const matched = mustIncludeAny.find((candidate) =>
|
|
||||||
normalizedResponse.includes(normalizeText(candidate)),
|
|
||||||
);
|
|
||||||
|
|
||||||
checks.push({
|
|
||||||
name: "mustIncludeAny",
|
|
||||||
passed: Boolean(matched),
|
|
||||||
skipped: false,
|
|
||||||
detail: matched
|
|
||||||
? `matched=${matched}`
|
|
||||||
: `expected one of: ${mustIncludeAny.join(", ")}`,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
checks.push({
|
|
||||||
name: "mustIncludeAny",
|
|
||||||
passed: true,
|
|
||||||
skipped: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
|
|
||||||
if (mustNotInclude.length > 0) {
|
|
||||||
const normalizedResponse = normalizeText(responseText);
|
|
||||||
const matched = mustNotInclude.find((candidate) =>
|
|
||||||
normalizedResponse.includes(normalizeText(candidate)),
|
|
||||||
);
|
|
||||||
|
|
||||||
checks.push({
|
|
||||||
name: "mustNotInclude",
|
|
||||||
passed: !matched,
|
|
||||||
skipped: false,
|
|
||||||
detail: matched ? `forbidden=${matched}` : undefined,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
checks.push({
|
|
||||||
name: "mustNotInclude",
|
|
||||||
passed: true,
|
|
||||||
skipped: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const activeChecks = checks.filter((check) => !check.skipped);
|
|
||||||
const passedChecks = activeChecks.filter((check) => check.passed).length;
|
|
||||||
|
|
||||||
return {
|
|
||||||
passed: activeChecks.every((check) => check.passed),
|
|
||||||
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
|
||||||
checks,
|
|
||||||
validatorStatus: validation.status,
|
|
||||||
validatorReasons: validation.reasons,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
import type { ToolRegistry } from "@tools/registry";
|
||||||
|
import { createModerationToolRegistry } from "@skills/moderation/tools";
|
||||||
|
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
|
||||||
|
import {
|
||||||
|
validateOnboardingResponse,
|
||||||
|
type ValidationResult,
|
||||||
|
} from "@skills/onboarding/runtime/validator";
|
||||||
|
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||||
|
|
||||||
|
export type SkillValidationResult = ValidationResult;
|
||||||
|
|
||||||
|
export interface SkillJudgeConfig {
|
||||||
|
axes: readonly string[];
|
||||||
|
promptTemplate?: string | undefined;
|
||||||
|
toolRegistryFactory: () => ToolRegistry;
|
||||||
|
tournamentInstruction: string;
|
||||||
|
validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined;
|
||||||
|
weights?: Record<string, number> | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const SKILL_JUDGE_CONFIGS: Record<string, SkillJudgeConfig> = {
|
||||||
|
moderation: {
|
||||||
|
axes: ["appropriateness", "accuracy", "helpfulness"],
|
||||||
|
toolRegistryFactory: createModerationToolRegistry,
|
||||||
|
tournamentInstruction:
|
||||||
|
"커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.",
|
||||||
|
},
|
||||||
|
onboarding: {
|
||||||
|
axes: ["friendliness", "accuracy", "concision", "scope"],
|
||||||
|
toolRegistryFactory: createOnboardingToolRegistry,
|
||||||
|
tournamentInstruction:
|
||||||
|
"온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.",
|
||||||
|
validator: (text, derived) =>
|
||||||
|
validateOnboardingResponse(text, derived as DerivedState, {
|
||||||
|
allowShortResponse: (derived as DerivedState).allowShortResponse,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export function getSkillJudgeConfig(skillName: string): SkillJudgeConfig {
|
||||||
|
const config = SKILL_JUDGE_CONFIGS[skillName] ?? SKILL_JUDGE_CONFIGS["onboarding"];
|
||||||
|
if (!config) {
|
||||||
|
throw new Error("Missing onboarding judge config");
|
||||||
|
}
|
||||||
|
return config;
|
||||||
|
}
|
||||||
+41
-45
@@ -5,15 +5,11 @@ import { dirname, resolve } from "node:path";
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import { SkillRegistry } from "@skills/skill-loader";
|
import { SkillRegistry } from "@skills/skill-loader";
|
||||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
|
||||||
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
|
|
||||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
|
||||||
import {
|
import {
|
||||||
PromptBuilder,
|
|
||||||
type PromptLayerDiff,
|
type PromptLayerDiff,
|
||||||
type PromptLayerName,
|
type PromptLayerName,
|
||||||
type PromptVariantManifest,
|
type PromptVariantManifest,
|
||||||
} from "@skills/onboarding/prompts/builder";
|
} from "@eval/prompt-variant";
|
||||||
import {
|
import {
|
||||||
getEvalTournamentMinCases,
|
getEvalTournamentMinCases,
|
||||||
getLlmProposeMaxTokens,
|
getLlmProposeMaxTokens,
|
||||||
@@ -26,6 +22,8 @@ import { queryArchivedReviewLabels } from "../review/label-archive.js";
|
|||||||
import type { ReviewLabel } from "../review/queue.js";
|
import type { ReviewLabel } from "../review/queue.js";
|
||||||
import { runG1Static, runG4Permission } from "../governance/gates.js";
|
import { runG1Static, runG4Permission } from "../governance/gates.js";
|
||||||
import { promoteTrustTier } from "../governance/trust.js";
|
import { promoteTrustTier } from "../governance/trust.js";
|
||||||
|
import { getSkillJudgeConfig } from "../judges/skill-config.js";
|
||||||
|
import { getSkillProposeAdapter } from "./skill-adapters.js";
|
||||||
|
|
||||||
export interface ProposalInput {
|
export interface ProposalInput {
|
||||||
slice: SliceKey;
|
slice: SliceKey;
|
||||||
@@ -129,44 +127,12 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
|||||||
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
||||||
}
|
}
|
||||||
|
|
||||||
function emptyBlackboard(): Blackboard {
|
|
||||||
return {
|
|
||||||
flaggedIssues: [],
|
|
||||||
openQuestions: [],
|
|
||||||
recentNicknameAttempts: [],
|
|
||||||
toneMemory: {},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function derivedFromSlice(slice: SliceKey): DerivedState {
|
|
||||||
return {
|
|
||||||
allowShortResponse: false,
|
|
||||||
intent: { kind: slice.intent } as Intent,
|
|
||||||
phase: { kind: slice.phase } as OnboardingPhase,
|
|
||||||
responseMode: "llm",
|
|
||||||
shouldRespond: true,
|
|
||||||
suggestedMaxTokens: 384,
|
|
||||||
tier: slice.tier,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function defaultPromptContext(): PromptContext {
|
|
||||||
return {
|
|
||||||
blackboard: emptyBlackboard(),
|
|
||||||
conversationHistory: [],
|
|
||||||
userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildCurrentPromptSnapshot(
|
export function buildCurrentPromptSnapshot(
|
||||||
slice: SliceKey,
|
slice: SliceKey,
|
||||||
archiveEntry?: EvalArchiveEntry,
|
archiveEntry?: EvalArchiveEntry,
|
||||||
): string {
|
): string {
|
||||||
return PromptBuilder.forDerived(
|
return getSkillProposeAdapter(slice.skill ?? archiveEntry?.skill ?? "onboarding")
|
||||||
archiveEntry?.derived ?? derivedFromSlice(slice),
|
.buildSnapshot(slice, archiveEntry);
|
||||||
null,
|
|
||||||
defaultPromptContext(),
|
|
||||||
).buildSystemPrompt();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
|
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
|
||||||
@@ -233,8 +199,9 @@ export function summarizeRootCauses(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildProposalPrompt(input: ProposalInput): string {
|
function buildProposalPrompt(input: ProposalInput): string {
|
||||||
|
const skill = input.slice.skill ?? "onboarding";
|
||||||
return [
|
return [
|
||||||
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
|
`${skill} 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.`,
|
||||||
"반드시 JSON object만 출력하세요.",
|
"반드시 JSON object만 출력하세요.",
|
||||||
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
|
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
|
||||||
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
|
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
|
||||||
@@ -271,7 +238,7 @@ export async function proposeVariants(
|
|||||||
{
|
{
|
||||||
role: "system",
|
role: "system",
|
||||||
content:
|
content:
|
||||||
"당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.",
|
`당신은 ${input.slice.skill ?? "onboarding"} 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
role: "user",
|
role: "user",
|
||||||
@@ -296,10 +263,12 @@ export async function proposeVariants(
|
|||||||
return {
|
return {
|
||||||
...proposal,
|
...proposal,
|
||||||
diff,
|
diff,
|
||||||
|
skill: input.slice.skill ?? "onboarding",
|
||||||
variantId: stableHash({
|
variantId: stableHash({
|
||||||
description: proposal.description,
|
description: proposal.description,
|
||||||
diff,
|
diff,
|
||||||
rationale: proposal.rationale,
|
rationale: proposal.rationale,
|
||||||
|
skill: input.slice.skill ?? "onboarding",
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
@@ -325,11 +294,34 @@ export async function buildProposalInputFromArchive(
|
|||||||
},
|
},
|
||||||
): Promise<ProposalInput> {
|
): Promise<ProposalInput> {
|
||||||
const limit = options?.limit ?? 50;
|
const limit = options?.limit ?? 50;
|
||||||
|
const skill = slice.skill ?? "onboarding";
|
||||||
|
const skillFilter =
|
||||||
|
skill === "onboarding"
|
||||||
|
? {
|
||||||
|
$or: [
|
||||||
|
{ skill },
|
||||||
|
{ skill: { $exists: false } },
|
||||||
|
],
|
||||||
|
}
|
||||||
|
: { skill };
|
||||||
const entries = await getEvalArchiveCollection(mongoConnection)
|
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||||
.find({
|
.find({
|
||||||
tier: slice.tier,
|
$and: [
|
||||||
"derived.intent.kind": slice.intent,
|
skillFilter,
|
||||||
"derived.phase.kind": slice.phase,
|
{ tier: slice.tier },
|
||||||
|
{
|
||||||
|
$or: [
|
||||||
|
{
|
||||||
|
"derived.intent.kind": slice.intent,
|
||||||
|
"derived.phase.kind": slice.phase,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"derived.intent": slice.intent,
|
||||||
|
"derived.phase": slice.phase,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
})
|
})
|
||||||
.sort({ evaluatedAt: -1 })
|
.sort({ evaluatedAt: -1 })
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
@@ -373,13 +365,17 @@ export async function writePromptVariantProposal(
|
|||||||
await mkdir(dirname(outputPath), { recursive: true });
|
await mkdir(dirname(outputPath), { recursive: true });
|
||||||
const manifest: PromptVariantProposal = {
|
const manifest: PromptVariantProposal = {
|
||||||
...proposal,
|
...proposal,
|
||||||
|
skill: proposal.skill ?? "onboarding",
|
||||||
trustTier: proposal.trustTier ?? "T1",
|
trustTier: proposal.trustTier ?? "T1",
|
||||||
};
|
};
|
||||||
const g1 = runG1Static(manifest);
|
const g1 = runG1Static(manifest);
|
||||||
|
const skill = manifest.skill ?? "onboarding";
|
||||||
|
const judgeConfig = getSkillJudgeConfig(skill);
|
||||||
const g4 = runG4Permission(
|
const g4 = runG4Permission(
|
||||||
manifest,
|
manifest,
|
||||||
createBuiltInToolRegistry(),
|
judgeConfig.toolRegistryFactory(),
|
||||||
SkillRegistry.load(resolve(process.cwd(), "src", "skills")),
|
SkillRegistry.load(resolve(process.cwd(), "src", "skills")),
|
||||||
|
skill,
|
||||||
);
|
);
|
||||||
const gatedManifest: PromptVariantProposal = {
|
const gatedManifest: PromptVariantProposal = {
|
||||||
...manifest,
|
...manifest,
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { moderationProposeAdapter } from "@skills/moderation/eval/propose-adapter";
|
||||||
|
import { onboardingProposeAdapter } from "@skills/onboarding/eval/propose-adapter";
|
||||||
|
import type { SkillProposeAdapter } from "./skill-propose-adapter.js";
|
||||||
|
|
||||||
|
const SKILL_PROPOSE_ADAPTERS: Record<string, SkillProposeAdapter> = {
|
||||||
|
[moderationProposeAdapter.skill]: moderationProposeAdapter,
|
||||||
|
[onboardingProposeAdapter.skill]: onboardingProposeAdapter,
|
||||||
|
};
|
||||||
|
|
||||||
|
export function getSkillProposeAdapter(skill: string): SkillProposeAdapter {
|
||||||
|
const adapter = SKILL_PROPOSE_ADAPTERS[skill] ?? SKILL_PROPOSE_ADAPTERS["onboarding"];
|
||||||
|
if (!adapter) {
|
||||||
|
throw new Error("Missing onboarding propose adapter");
|
||||||
|
}
|
||||||
|
return adapter;
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
import type { AgentRequest } from "@agent/types";
|
||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import type { ChatMessage } from "@llm/types";
|
||||||
|
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||||
|
import type { EvalArchiveEntry } from "../archive.js";
|
||||||
|
import type { SliceKey } from "../aggregator.js";
|
||||||
|
|
||||||
|
export interface SkillProposeAdapter {
|
||||||
|
readonly skill: string;
|
||||||
|
buildSnapshot(slice: SliceKey, archiveEntry?: EvalArchiveEntry): string;
|
||||||
|
derivedFromSlice(slice: SliceKey): unknown;
|
||||||
|
generateResponse(
|
||||||
|
llmClient: LlmClient,
|
||||||
|
entry: EvalArchiveEntry,
|
||||||
|
variantId: string,
|
||||||
|
manifests?: Record<string, PromptVariantManifest>,
|
||||||
|
): Promise<string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getVariantManifest(
|
||||||
|
variantId: string,
|
||||||
|
manifests?: Record<string, PromptVariantManifest>,
|
||||||
|
): PromptVariantManifest | string | null {
|
||||||
|
if (variantId === "control") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return manifests?.[variantId] ?? variantId;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function basicPromptSnapshot(messages: ChatMessage[]): string {
|
||||||
|
return messages
|
||||||
|
.map((message) => `[${message.role}]\n${message.content}`)
|
||||||
|
.join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getRequestText(request: AgentRequest): string {
|
||||||
|
return request.userMessage?.trim() || "";
|
||||||
|
}
|
||||||
+60
-62
@@ -2,11 +2,12 @@ import { createHash } from "node:crypto";
|
|||||||
|
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
|
||||||
import { getEvalTournamentMaxMatches } from "../../config.js";
|
import { getEvalTournamentMaxMatches } from "../../config.js";
|
||||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
||||||
import { getEvalArchiveCollection } from "../archive.js";
|
import { getEvalArchiveCollection } from "../archive.js";
|
||||||
|
import { getSkillJudgeConfig } from "../judges/skill-config.js";
|
||||||
|
import { getSkillProposeAdapter } from "./skill-adapters.js";
|
||||||
import {
|
import {
|
||||||
createEloRating,
|
createEloRating,
|
||||||
scoreForVerdict,
|
scoreForVerdict,
|
||||||
@@ -30,6 +31,7 @@ export interface TournamentMatch {
|
|||||||
export interface TournamentResult {
|
export interface TournamentResult {
|
||||||
generatedAt: string;
|
generatedAt: string;
|
||||||
matches: TournamentMatch[];
|
matches: TournamentMatch[];
|
||||||
|
skill?: string | undefined;
|
||||||
variants: EloRating[];
|
variants: EloRating[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,6 +60,10 @@ const sideBySideJudgeSchema = z.object({
|
|||||||
verdict: z.enum(["A", "B", "tie"]),
|
verdict: z.enum(["A", "B", "tie"]),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function entrySkill(entry: EvalArchiveEntry): string {
|
||||||
|
return entry.skill || entry.request.skill || "onboarding";
|
||||||
|
}
|
||||||
|
|
||||||
function stableHash(value: unknown): string {
|
function stableHash(value: unknown): string {
|
||||||
return createHash("sha256")
|
return createHash("sha256")
|
||||||
.update(JSON.stringify(value))
|
.update(JSON.stringify(value))
|
||||||
@@ -65,56 +71,18 @@ function stableHash(value: unknown): string {
|
|||||||
.slice(0, 12);
|
.slice(0, 12);
|
||||||
}
|
}
|
||||||
|
|
||||||
function emptyBlackboard(): Blackboard {
|
|
||||||
return {
|
|
||||||
flaggedIssues: [],
|
|
||||||
openQuestions: [],
|
|
||||||
recentNicknameAttempts: [],
|
|
||||||
toneMemory: {},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function promptContextForCase(entry: EvalArchiveEntry): PromptContext {
|
|
||||||
return {
|
|
||||||
blackboard: emptyBlackboard(),
|
|
||||||
conversationHistory: [],
|
|
||||||
userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function getVariantManifest(
|
|
||||||
variantId: string,
|
|
||||||
manifests?: Record<string, PromptVariantManifest>,
|
|
||||||
): PromptVariantManifest | string | null {
|
|
||||||
if (variantId === "control") {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return manifests?.[variantId] ?? variantId;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function generateTournamentResponse(
|
async function generateTournamentResponse(
|
||||||
llmClient: LlmClient,
|
llmClient: LlmClient,
|
||||||
entry: EvalArchiveEntry,
|
entry: EvalArchiveEntry,
|
||||||
variantId: string,
|
variantId: string,
|
||||||
manifests?: Record<string, PromptVariantManifest>,
|
manifests?: Record<string, PromptVariantManifest>,
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
let builder = PromptBuilder.forDerived(
|
return getSkillProposeAdapter(entrySkill(entry)).generateResponse(
|
||||||
entry.derived,
|
llmClient,
|
||||||
null,
|
entry,
|
||||||
promptContextForCase(entry),
|
variantId,
|
||||||
|
manifests,
|
||||||
);
|
);
|
||||||
const manifest = getVariantManifest(variantId, manifests);
|
|
||||||
if (manifest) {
|
|
||||||
builder = builder.applyVariant(manifest);
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await llmClient.chatCompletion(builder.build(), {
|
|
||||||
maxTokens: entry.derived.suggestedMaxTokens,
|
|
||||||
temperature: 0,
|
|
||||||
tier: entry.derived.tier,
|
|
||||||
});
|
|
||||||
|
|
||||||
return response?.content.trim() ?? "";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildJudgePrompt(
|
function buildJudgePrompt(
|
||||||
@@ -122,9 +90,12 @@ function buildJudgePrompt(
|
|||||||
responseA: string,
|
responseA: string,
|
||||||
responseB: string,
|
responseB: string,
|
||||||
): string {
|
): string {
|
||||||
|
const skill = entrySkill(entry);
|
||||||
|
const config = getSkillJudgeConfig(skill);
|
||||||
return [
|
return [
|
||||||
"아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.",
|
`아래 두 ${skill} 응답 중 더 나은 쪽을 고르세요.`,
|
||||||
"평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.",
|
config.tournamentInstruction,
|
||||||
|
`평가 축: ${config.axes.join(", ")}`,
|
||||||
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
|
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
|
||||||
"",
|
"",
|
||||||
"[request]",
|
"[request]",
|
||||||
@@ -157,7 +128,7 @@ async function judgeTournamentMatch(
|
|||||||
{
|
{
|
||||||
role: "system",
|
role: "system",
|
||||||
content:
|
content:
|
||||||
"당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.",
|
`당신은 ${entrySkill(entry)} 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
role: "user",
|
role: "user",
|
||||||
@@ -201,6 +172,7 @@ async function ensureTournamentIndexes(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
collection.createIndex({ generatedAt: -1 }),
|
collection.createIndex({ generatedAt: -1 }),
|
||||||
|
collection.createIndex({ skill: 1, generatedAt: -1 }),
|
||||||
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
|
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
@@ -225,6 +197,7 @@ export async function runTournament(
|
|||||||
cases: EvalArchiveEntry[];
|
cases: EvalArchiveEntry[];
|
||||||
maxMatches?: number | undefined;
|
maxMatches?: number | undefined;
|
||||||
nowMs?: number | undefined;
|
nowMs?: number | undefined;
|
||||||
|
skill?: string | undefined;
|
||||||
variantIds: string[];
|
variantIds: string[];
|
||||||
variantManifests?: Record<string, PromptVariantManifest> | undefined;
|
variantManifests?: Record<string, PromptVariantManifest> | undefined;
|
||||||
},
|
},
|
||||||
@@ -303,9 +276,11 @@ export async function runTournament(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const skill = options.skill ?? options.cases[0]?.skill;
|
||||||
return {
|
return {
|
||||||
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
|
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
|
||||||
matches,
|
matches,
|
||||||
|
...(skill ? { skill } : {}),
|
||||||
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
|
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -315,22 +290,35 @@ export async function loadTournamentCases(
|
|||||||
options: {
|
options: {
|
||||||
caseIds?: string[] | undefined;
|
caseIds?: string[] | undefined;
|
||||||
limit?: number | undefined;
|
limit?: number | undefined;
|
||||||
|
skill?: string | undefined;
|
||||||
},
|
},
|
||||||
): Promise<EvalArchiveEntry[]> {
|
): Promise<EvalArchiveEntry[]> {
|
||||||
const filter =
|
const filter: Record<string, unknown> = {};
|
||||||
options.caseIds && options.caseIds.length > 0
|
if (options.caseIds && options.caseIds.length > 0) {
|
||||||
? {
|
filter.streamId = {
|
||||||
streamId: {
|
$in: options.caseIds,
|
||||||
$in: options.caseIds,
|
};
|
||||||
},
|
}
|
||||||
}
|
if (options.skill) {
|
||||||
: {};
|
if (options.skill === "onboarding") {
|
||||||
|
filter.$or = [
|
||||||
|
{ skill: options.skill },
|
||||||
|
{ skill: { $exists: false } },
|
||||||
|
];
|
||||||
|
} else {
|
||||||
|
filter.skill = options.skill;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return getEvalArchiveCollection(mongoConnection)
|
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||||
.find(filter)
|
.find(filter)
|
||||||
.sort({ evaluatedAt: -1 })
|
.sort({ evaluatedAt: -1 })
|
||||||
.limit(options.limit ?? getEvalTournamentMaxMatches())
|
.limit(options.limit ?? getEvalTournamentMaxMatches())
|
||||||
.toArray();
|
.toArray();
|
||||||
|
return entries.map((entry) => ({
|
||||||
|
...entry,
|
||||||
|
skill: entry.skill || entry.request.skill || "onboarding",
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function saveTournamentResult(
|
export async function saveTournamentResult(
|
||||||
@@ -346,14 +334,24 @@ export async function queryTournamentResults(
|
|||||||
mongoConnection: MongoConnectionLike,
|
mongoConnection: MongoConnectionLike,
|
||||||
options?: {
|
options?: {
|
||||||
limit?: number | undefined;
|
limit?: number | undefined;
|
||||||
|
skill?: string | undefined;
|
||||||
variantId?: string | undefined;
|
variantId?: string | undefined;
|
||||||
},
|
},
|
||||||
): Promise<TournamentResult[]> {
|
): Promise<TournamentResult[]> {
|
||||||
const filter = options?.variantId
|
const filter: Record<string, unknown> = {};
|
||||||
? {
|
if (options?.variantId) {
|
||||||
"variants.id": options.variantId,
|
filter["variants.id"] = options.variantId;
|
||||||
}
|
}
|
||||||
: {};
|
if (options?.skill) {
|
||||||
|
if (options.skill === "onboarding") {
|
||||||
|
filter.$or = [
|
||||||
|
{ skill: options.skill },
|
||||||
|
{ skill: { $exists: false } },
|
||||||
|
];
|
||||||
|
} else {
|
||||||
|
filter.skill = options.skill;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return getTournamentCollection(mongoConnection)
|
return getTournamentCollection(mongoConnection)
|
||||||
.find(filter)
|
.find(filter)
|
||||||
|
|||||||
+8
-4
@@ -16,7 +16,11 @@ import {
|
|||||||
} from "./anonymize.js";
|
} from "./anonymize.js";
|
||||||
import type { EvalCaseDefinition } from "./types.js";
|
import type { EvalCaseDefinition } from "./types.js";
|
||||||
import type { EvalArchiveEntry, MongoConnectionLike } from "./archive.js";
|
import type { EvalArchiveEntry, MongoConnectionLike } from "./archive.js";
|
||||||
import { getEvalArchiveCollection } from "./archive.js";
|
import {
|
||||||
|
getArchiveDerivedIntent,
|
||||||
|
getArchiveDerivedPhase,
|
||||||
|
getEvalArchiveCollection,
|
||||||
|
} from "./archive.js";
|
||||||
import { queryArchivedReviewLabels } from "./review/label-archive.js";
|
import { queryArchivedReviewLabels } from "./review/label-archive.js";
|
||||||
import type { ReviewLabel } from "./review/queue.js";
|
import type { ReviewLabel } from "./review/queue.js";
|
||||||
|
|
||||||
@@ -59,7 +63,7 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function sliceKey(entry: EvalArchiveEntry): string {
|
function sliceKey(entry: EvalArchiveEntry): string {
|
||||||
return `${entry.tier}:${entry.derived.intent.kind}:${entry.derived.phase.kind}`;
|
return `${entry.skill || entry.request.skill || "onboarding"}:${entry.tier}:${getArchiveDerivedIntent(entry.derived)}:${getArchiveDerivedPhase(entry.derived)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function sortByLowestScore(
|
function sortByLowestScore(
|
||||||
@@ -286,8 +290,8 @@ function candidateToEvalCase(candidate: PromotionCandidate): EvalCaseDefinition
|
|||||||
candidate.labelRationale === undefined
|
candidate.labelRationale === undefined
|
||||||
? null
|
? null
|
||||||
: `humanRationale=${candidate.labelRationale}`,
|
: `humanRationale=${candidate.labelRationale}`,
|
||||||
`derivedIntent=${candidate.derived.intent.kind}`,
|
`derivedIntent=${getArchiveDerivedIntent(candidate.derived)}`,
|
||||||
`derivedPhase=${candidate.derived.phase.kind}`,
|
`derivedPhase=${getArchiveDerivedPhase(candidate.derived)}`,
|
||||||
`assistantResponse=${candidate.response.text}`,
|
`assistantResponse=${candidate.response.text}`,
|
||||||
candidate.betterResponse === undefined
|
candidate.betterResponse === undefined
|
||||||
? null
|
? null
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
import { existsSync, readFileSync } from "node:fs";
|
||||||
|
import { resolve } from "node:path";
|
||||||
|
|
||||||
|
export type PromptLayerName =
|
||||||
|
| "context"
|
||||||
|
| "examples"
|
||||||
|
| "persona"
|
||||||
|
| "policy"
|
||||||
|
| "state"
|
||||||
|
| "tool-policy";
|
||||||
|
|
||||||
|
export type PromptLayerOperation = "append" | "remove" | "replace";
|
||||||
|
|
||||||
|
export interface PromptLayerDiff {
|
||||||
|
layer: PromptLayerName;
|
||||||
|
op: PromptLayerOperation;
|
||||||
|
reason?: string | undefined;
|
||||||
|
text?: string | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PromptVariantManifest {
|
||||||
|
description?: string | undefined;
|
||||||
|
diff: PromptLayerDiff[];
|
||||||
|
expectedImprovement?: string | undefined;
|
||||||
|
rationale?: string | undefined;
|
||||||
|
requiredResources?: string[] | undefined;
|
||||||
|
requiredTools?: string[] | undefined;
|
||||||
|
skill?: string | undefined;
|
||||||
|
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
|
||||||
|
variantId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||||
|
|
||||||
|
export function loadPromptVariantManifest(
|
||||||
|
variantId: string,
|
||||||
|
): PromptVariantManifest | null {
|
||||||
|
const normalized = variantId.trim();
|
||||||
|
if (!normalized) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (variantCache.has(normalized)) {
|
||||||
|
return variantCache.get(normalized) ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const path = resolve(
|
||||||
|
process.cwd(),
|
||||||
|
"src",
|
||||||
|
"eval",
|
||||||
|
"proposals",
|
||||||
|
`${normalized}.json`,
|
||||||
|
);
|
||||||
|
if (!existsSync(path)) {
|
||||||
|
variantCache.set(normalized, null);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
||||||
|
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
||||||
|
variantCache.set(normalized, null);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
variantCache.set(normalized, parsed);
|
||||||
|
return parsed;
|
||||||
|
} catch {
|
||||||
|
variantCache.set(normalized, null);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -81,11 +81,12 @@ function toCalibrationPairs(
|
|||||||
pairs.push({
|
pairs.push({
|
||||||
evalStreamId: label.evalStreamId,
|
evalStreamId: label.evalStreamId,
|
||||||
humanScore: label.humanScore,
|
humanScore: label.humanScore,
|
||||||
llmAccuracy: archive.llmJudge.scores.accuracy,
|
llmAccuracy: archive.llmJudge.scores.accuracy ?? archive.llmJudge.scores.overall,
|
||||||
llmConcision: archive.llmJudge.scores.concision,
|
llmConcision: archive.llmJudge.scores.concision ?? archive.llmJudge.scores.overall,
|
||||||
llmFriendliness: archive.llmJudge.scores.friendliness,
|
llmFriendliness:
|
||||||
|
archive.llmJudge.scores.friendliness ?? archive.llmJudge.scores.overall,
|
||||||
llmOverall: archive.llmJudge.scores.overall,
|
llmOverall: archive.llmJudge.scores.overall,
|
||||||
llmScope: archive.llmJudge.scores.scope,
|
llmScope: archive.llmJudge.scores.scope ?? archive.llmJudge.scores.overall,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,26 @@ import {
|
|||||||
} from "@skills/onboarding/runtime/router";
|
} from "@skills/onboarding/runtime/router";
|
||||||
import type { EvalArchiveEntry } from "./archive.js";
|
import type { EvalArchiveEntry } from "./archive.js";
|
||||||
|
|
||||||
|
function isOnboardingArchiveEntry(
|
||||||
|
entry: EvalArchiveEntry,
|
||||||
|
): entry is EvalArchiveEntry & {
|
||||||
|
classified: ClassifiedIntent;
|
||||||
|
derived: DerivedState;
|
||||||
|
} {
|
||||||
|
return (
|
||||||
|
(entry.skill === undefined ||
|
||||||
|
entry.skill === "onboarding" ||
|
||||||
|
entry.request.skill === "onboarding") &&
|
||||||
|
typeof entry.derived === "object" &&
|
||||||
|
entry.derived !== null &&
|
||||||
|
typeof (entry.derived as { intent?: unknown }).intent === "object" &&
|
||||||
|
typeof (entry.derived as { phase?: unknown }).phase === "object" &&
|
||||||
|
typeof entry.classified === "object" &&
|
||||||
|
entry.classified !== null &&
|
||||||
|
typeof (entry.classified as { intent?: unknown }).intent === "object"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||||
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
||||||
}
|
}
|
||||||
@@ -21,6 +41,9 @@ function requestFromEntry(entry: EvalArchiveEntry): AgentRequest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null {
|
function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null {
|
||||||
|
if (!isOnboardingArchiveEntry(entry)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
const staticDecision = routerDecisionKind(
|
const staticDecision = routerDecisionKind(
|
||||||
decidePreGenerationRouting({
|
decidePreGenerationRouting({
|
||||||
classified: entry.classified,
|
classified: entry.classified,
|
||||||
@@ -57,11 +80,11 @@ function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function toClassified(entry: EvalArchiveEntry): ClassifiedIntent {
|
function toClassified(entry: EvalArchiveEntry): ClassifiedIntent {
|
||||||
return entry.classified;
|
return entry.classified as ClassifiedIntent;
|
||||||
}
|
}
|
||||||
|
|
||||||
function toDerived(entry: EvalArchiveEntry): DerivedState {
|
function toDerived(entry: EvalArchiveEntry): DerivedState {
|
||||||
return entry.derived;
|
return entry.derived as DerivedState;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildRouterTrainingSamples(
|
export function buildRouterTrainingSamples(
|
||||||
|
|||||||
+59
-19
@@ -30,12 +30,10 @@ import type {
|
|||||||
} from "@skills/onboarding/state/onboarding-state";
|
} from "@skills/onboarding/state/onboarding-state";
|
||||||
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
||||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
||||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||||
import { judgeProcessRule } from "./judges/process-judge.js";
|
import { judgeProcessRule } from "./judges/process-judge.js";
|
||||||
import {
|
import { judgeRuleCase } from "./judges/rule-judge.js";
|
||||||
judgeOnboardingRuleCase,
|
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||||
judgeRuleCase,
|
|
||||||
} from "./judges/rule-judge.js";
|
|
||||||
import type {
|
import type {
|
||||||
EvalBaselineFile,
|
EvalBaselineFile,
|
||||||
EvalCaseDefinition,
|
EvalCaseDefinition,
|
||||||
@@ -161,11 +159,18 @@ function round(value: number): number {
|
|||||||
return Number(value.toFixed(3));
|
return Number(value.toFixed(3));
|
||||||
}
|
}
|
||||||
|
|
||||||
function average(values: number[]): number | null {
|
function average(values: Array<number | undefined>): number | null {
|
||||||
if (values.length === 0) {
|
const finiteValues = values.filter(
|
||||||
|
(value): value is number =>
|
||||||
|
typeof value === "number" && Number.isFinite(value),
|
||||||
|
);
|
||||||
|
if (finiteValues.length === 0) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return round(values.reduce((sum, value) => sum + value, 0) / values.length);
|
return round(
|
||||||
|
finiteValues.reduce((sum, value) => sum + value, 0) /
|
||||||
|
finiteValues.length,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function averageOrZero(values: number[]): number {
|
function averageOrZero(values: number[]): number {
|
||||||
@@ -741,8 +746,16 @@ async function evaluateCase(
|
|||||||
skill,
|
skill,
|
||||||
llmClient,
|
llmClient,
|
||||||
enableLlmJudge,
|
enableLlmJudge,
|
||||||
|
effectiveSkill,
|
||||||
)
|
)
|
||||||
: evaluateGenericSkillCase(evalCase, input, skill),
|
: evaluateGenericSkillCase(
|
||||||
|
evalCase,
|
||||||
|
input,
|
||||||
|
skill,
|
||||||
|
llmClient,
|
||||||
|
enableLlmJudge,
|
||||||
|
effectiveSkill,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -752,9 +765,11 @@ async function evaluateOnboardingCase(
|
|||||||
skill: Skill,
|
skill: Skill,
|
||||||
llmClient: LlmClient,
|
llmClient: LlmClient,
|
||||||
enableLlmJudge: boolean,
|
enableLlmJudge: boolean,
|
||||||
|
skillName: string,
|
||||||
): Promise<EvalCaseResult> {
|
): Promise<EvalCaseResult> {
|
||||||
const classifier = new OnboardingIntentClassifier(llmClient);
|
const classifier = new OnboardingIntentClassifier(llmClient);
|
||||||
const processTrace = new ProcessTraceCollector();
|
const processTrace = new ProcessTraceCollector();
|
||||||
|
const judgeConfig = getSkillJudgeConfig(skillName);
|
||||||
|
|
||||||
return runWithProcessTrace(processTrace, async () => {
|
return runWithProcessTrace(processTrace, async () => {
|
||||||
const classified = await classifier.classify(
|
const classified = await classifier.classify(
|
||||||
@@ -775,14 +790,16 @@ async function evaluateOnboardingCase(
|
|||||||
);
|
);
|
||||||
const latencyMs = Date.now() - startedAt;
|
const latencyMs = Date.now() - startedAt;
|
||||||
const derivedSnapshot = toDerivedSnapshot(derived);
|
const derivedSnapshot = toDerivedSnapshot(derived);
|
||||||
const ruleJudge = judgeOnboardingRuleCase(
|
const ruleJudge = judgeRuleCase(evalCase.expected, response.text, {
|
||||||
evalCase.expected,
|
|
||||||
response.text,
|
|
||||||
derived,
|
derived,
|
||||||
);
|
intent: derived.intent.kind,
|
||||||
|
responseMode: derived.responseMode,
|
||||||
|
validator: judgeConfig.validator,
|
||||||
|
});
|
||||||
const llmJudge = enableLlmJudge
|
const llmJudge = enableLlmJudge
|
||||||
? await judgeOnboardingWithLlm(
|
? await judgeSkillWithLlm(
|
||||||
llmClient,
|
llmClient,
|
||||||
|
skillName,
|
||||||
evalCase,
|
evalCase,
|
||||||
response.text,
|
response.text,
|
||||||
derivedSnapshot,
|
derivedSnapshot,
|
||||||
@@ -795,6 +812,7 @@ async function evaluateOnboardingCase(
|
|||||||
const processJudge = judgeProcessRule(
|
const processJudge = judgeProcessRule(
|
||||||
evalCase.expected,
|
evalCase.expected,
|
||||||
processTraceSnapshot,
|
processTraceSnapshot,
|
||||||
|
judgeConfig.toolRegistryFactory(),
|
||||||
);
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -834,7 +852,12 @@ async function evaluateOnboardingCase(
|
|||||||
error: errorMessage,
|
error: errorMessage,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
|
ruleJudge: judgeRuleCase(evalCase.expected, "", {
|
||||||
|
derived,
|
||||||
|
intent: derived.intent.kind,
|
||||||
|
responseMode: derived.responseMode,
|
||||||
|
validator: judgeConfig.validator,
|
||||||
|
}),
|
||||||
llmJudge: {
|
llmJudge: {
|
||||||
skipped: true,
|
skipped: true,
|
||||||
reason: "skill-error",
|
reason: "skill-error",
|
||||||
@@ -842,6 +865,7 @@ async function evaluateOnboardingCase(
|
|||||||
processJudge: judgeProcessRule(
|
processJudge: judgeProcessRule(
|
||||||
evalCase.expected,
|
evalCase.expected,
|
||||||
processTraceSnapshot,
|
processTraceSnapshot,
|
||||||
|
judgeConfig.toolRegistryFactory(),
|
||||||
),
|
),
|
||||||
processTrace: processTraceSnapshot,
|
processTrace: processTraceSnapshot,
|
||||||
error: errorMessage,
|
error: errorMessage,
|
||||||
@@ -854,8 +878,12 @@ async function evaluateGenericSkillCase(
|
|||||||
evalCase: EvalCaseDefinition,
|
evalCase: EvalCaseDefinition,
|
||||||
input: EvalCaseInput,
|
input: EvalCaseInput,
|
||||||
skill: Skill,
|
skill: Skill,
|
||||||
|
llmClient: LlmClient,
|
||||||
|
enableLlmJudge: boolean,
|
||||||
|
skillName: string,
|
||||||
): Promise<EvalCaseResult> {
|
): Promise<EvalCaseResult> {
|
||||||
const classified = classifyModerationIntent(input.request.userMessage ?? "");
|
const classified = classifyModerationIntent(input.request.userMessage ?? "");
|
||||||
|
const judgeConfig = getSkillJudgeConfig(skillName);
|
||||||
const derived: EvalDerivedSnapshot = {
|
const derived: EvalDerivedSnapshot = {
|
||||||
allowShortResponse: true,
|
allowShortResponse: true,
|
||||||
intent: classified.intent.kind,
|
intent: classified.intent.kind,
|
||||||
@@ -887,13 +915,23 @@ async function evaluateGenericSkillCase(
|
|||||||
request: input.request,
|
request: input.request,
|
||||||
response,
|
response,
|
||||||
ruleJudge: judgeRuleCase(evalCase.expected, response.text, {
|
ruleJudge: judgeRuleCase(evalCase.expected, response.text, {
|
||||||
|
derived,
|
||||||
intent: classified.intent.kind,
|
intent: classified.intent.kind,
|
||||||
responseMode: derived.responseMode,
|
responseMode: derived.responseMode,
|
||||||
|
validator: judgeConfig.validator,
|
||||||
}),
|
}),
|
||||||
llmJudge: {
|
llmJudge: enableLlmJudge
|
||||||
skipped: true,
|
? await judgeSkillWithLlm(
|
||||||
reason: "unsupported-skill",
|
llmClient,
|
||||||
},
|
skillName,
|
||||||
|
evalCase,
|
||||||
|
response.text,
|
||||||
|
derived,
|
||||||
|
)
|
||||||
|
: {
|
||||||
|
skipped: true,
|
||||||
|
reason: "disabled",
|
||||||
|
},
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const latencyMs = Date.now() - startedAt;
|
const latencyMs = Date.now() - startedAt;
|
||||||
@@ -920,8 +958,10 @@ async function evaluateGenericSkillCase(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
ruleJudge: judgeRuleCase(evalCase.expected, "", {
|
ruleJudge: judgeRuleCase(evalCase.expected, "", {
|
||||||
|
derived,
|
||||||
intent: classified.intent.kind,
|
intent: classified.intent.kind,
|
||||||
responseMode: derived.responseMode,
|
responseMode: derived.responseMode,
|
||||||
|
validator: judgeConfig.validator,
|
||||||
}),
|
}),
|
||||||
llmJudge: {
|
llmJudge: {
|
||||||
skipped: true,
|
skipped: true,
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ describe("ShadowEvalBus", () => {
|
|||||||
derived,
|
derived,
|
||||||
request,
|
request,
|
||||||
response,
|
response,
|
||||||
|
skill: "onboarding",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(appendStream).toHaveBeenCalledTimes(1);
|
expect(appendStream).toHaveBeenCalledTimes(1);
|
||||||
@@ -68,7 +69,9 @@ describe("ShadowEvalBus", () => {
|
|||||||
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
|
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
|
||||||
type: "shadow-eval",
|
type: "shadow-eval",
|
||||||
requestId: "req-1",
|
requestId: "req-1",
|
||||||
|
skill: "onboarding",
|
||||||
tier: "0",
|
tier: "0",
|
||||||
|
version: "2",
|
||||||
});
|
});
|
||||||
|
|
||||||
randomSpy.mockRestore();
|
randomSpy.mockRestore();
|
||||||
@@ -86,6 +89,7 @@ describe("ShadowEvalBus", () => {
|
|||||||
...response,
|
...response,
|
||||||
text: "",
|
text: "",
|
||||||
},
|
},
|
||||||
|
skill: "onboarding",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(appendStream).not.toHaveBeenCalled();
|
expect(appendStream).not.toHaveBeenCalled();
|
||||||
|
|||||||
+55
-18
@@ -1,11 +1,6 @@
|
|||||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||||
import type { GarnetClient } from "@common/garnet";
|
import type { GarnetClient } from "@common/garnet";
|
||||||
import { getRequestLogger } from "@obs/context";
|
import { getRequestLogger } from "@obs/context";
|
||||||
import type {
|
|
||||||
ClassifiedIntent,
|
|
||||||
DerivedState,
|
|
||||||
} from "@skills/onboarding/state/onboarding-state";
|
|
||||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
|
||||||
import {
|
import {
|
||||||
getBackgroundStreamMaxLen,
|
getBackgroundStreamMaxLen,
|
||||||
getShadowEvalSamplingRate,
|
getShadowEvalSamplingRate,
|
||||||
@@ -14,13 +9,15 @@ import {
|
|||||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||||
|
|
||||||
export interface ShadowEvalEnvelope {
|
export interface ShadowEvalEnvelope {
|
||||||
classified: ClassifiedIntent;
|
classified: unknown;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
derived: DerivedState;
|
derived: unknown;
|
||||||
latencyMs?: number | undefined;
|
latencyMs?: number | undefined;
|
||||||
process?: ProcessTraceSnapshot | undefined;
|
process?: unknown;
|
||||||
request: AgentRequest;
|
request: AgentRequest;
|
||||||
response: AgentResponse;
|
response: AgentResponse;
|
||||||
|
skill: string;
|
||||||
|
version?: 2 | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ShadowEvalPublisher {
|
export interface ShadowEvalPublisher {
|
||||||
@@ -38,10 +35,46 @@ export function shouldSampleShadowEval(
|
|||||||
return randomValue < getShadowEvalSamplingRate(tier);
|
return randomValue < getShadowEvalSamplingRate(tier);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
|
function asRecord(value: unknown): Record<string, unknown> {
|
||||||
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
return typeof value === "object" && value !== null
|
||||||
| undefined {
|
? (value as Record<string, unknown>)
|
||||||
return process?.steps.classify?.output.abstention;
|
: {};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
||||||
|
const parsed = typeof value === "number" ? value : Number(value);
|
||||||
|
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDerivedTier(derived: unknown): 0 | 1 | 2 | null {
|
||||||
|
return parseTier(asRecord(derived)["tier"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTraceAbstention(process: unknown): Record<string, unknown> | null {
|
||||||
|
const steps = asRecord(asRecord(process)["steps"]);
|
||||||
|
const classify = asRecord(steps["classify"]);
|
||||||
|
const output = asRecord(classify["output"]);
|
||||||
|
const abstention = asRecord(output["abstention"]);
|
||||||
|
return Object.keys(abstention).length > 0 ? abstention : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAbstentionActionKind(abstention: Record<string, unknown> | null): string | null {
|
||||||
|
const action = asRecord(abstention?.["action"]);
|
||||||
|
const kind = action["kind"];
|
||||||
|
return typeof kind === "string" && kind.trim() ? kind : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAbstentionReason(abstention: Record<string, unknown> | null): string | null {
|
||||||
|
const reason = abstention?.["reason"];
|
||||||
|
return typeof reason === "string" && reason.trim() ? reason : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAbstentionScore(abstention: Record<string, unknown> | null): number | null {
|
||||||
|
const score = abstention?.["score"];
|
||||||
|
return typeof score === "number" && Number.isFinite(score) ? score : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ShadowEvalBus implements ShadowEvalPublisher {
|
export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||||
@@ -51,11 +84,13 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
|
|||||||
|
|
||||||
publish(envelope: ShadowEvalEnvelope): void {
|
publish(envelope: ShadowEvalEnvelope): void {
|
||||||
const abstention = getTraceAbstention(envelope.process);
|
const abstention = getTraceAbstention(envelope.process);
|
||||||
if (!envelope.response.text.trim() && abstention?.action.kind !== "no-response") {
|
const actionKind = getAbstentionActionKind(abstention);
|
||||||
|
if (!envelope.response.text.trim() && actionKind !== "no-response") {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!shouldSampleShadowEval(envelope.derived.tier)) {
|
const tier = getDerivedTier(envelope.derived);
|
||||||
|
if (tier === null || !shouldSampleShadowEval(tier)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,7 +101,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
|
|||||||
requestId: envelope.request.requestId,
|
requestId: envelope.request.requestId,
|
||||||
chatroomId: envelope.request.chatroomId,
|
chatroomId: envelope.request.chatroomId,
|
||||||
userId: envelope.request.userId,
|
userId: envelope.request.userId,
|
||||||
tier: String(envelope.derived.tier),
|
skill: envelope.skill,
|
||||||
|
version: String(envelope.version ?? 2),
|
||||||
|
tier: String(tier),
|
||||||
latencyMs:
|
latencyMs:
|
||||||
envelope.latencyMs !== undefined ? String(envelope.latencyMs) : "",
|
envelope.latencyMs !== undefined ? String(envelope.latencyMs) : "",
|
||||||
input: JSON.stringify({
|
input: JSON.stringify({
|
||||||
@@ -79,9 +116,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (abstention) {
|
if (abstention) {
|
||||||
fields.abstentionAction = abstention.action.kind;
|
fields.abstentionAction = actionKind ?? "";
|
||||||
fields.abstentionReason = abstention.reason ?? "";
|
fields.abstentionReason = getAbstentionReason(abstention) ?? "";
|
||||||
fields.abstentionScore = String(abstention.score);
|
fields.abstentionScore = String(getAbstentionScore(abstention) ?? "");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (envelope.process) {
|
if (envelope.process) {
|
||||||
|
|||||||
+100
-41
@@ -12,12 +12,6 @@ import { summarizeConversation } from "@conv/summarizer";
|
|||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import { getLogger } from "@obs/logger";
|
import { getLogger } from "@obs/logger";
|
||||||
import type { AlertSender } from "@obs/alert";
|
import type { AlertSender } from "@obs/alert";
|
||||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
|
||||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
|
||||||
import type {
|
|
||||||
ClassifiedIntent,
|
|
||||||
DerivedState,
|
|
||||||
} from "@skills/onboarding/state/onboarding-state";
|
|
||||||
import type { ConversationStore } from "@conv/conversation-store";
|
import type { ConversationStore } from "@conv/conversation-store";
|
||||||
import {
|
import {
|
||||||
getAlertThresholdDriftPct,
|
getAlertThresholdDriftPct,
|
||||||
@@ -49,13 +43,14 @@ import {
|
|||||||
type MongoConnectionLike,
|
type MongoConnectionLike,
|
||||||
} from "./archive.js";
|
} from "./archive.js";
|
||||||
import { computeDriftReports } from "./drift.js";
|
import { computeDriftReports } from "./drift.js";
|
||||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||||
import { writeProcessAttributions } from "./judges/attribution.js";
|
import { writeProcessAttributions } from "./judges/attribution.js";
|
||||||
import { loadPrmModel } from "./judges/prm.js";
|
import { loadPrmModel } from "./judges/prm.js";
|
||||||
import {
|
import {
|
||||||
judgeProcessRule,
|
judgeProcessRule,
|
||||||
judgeProcessWithLlm,
|
judgeProcessWithLlm,
|
||||||
} from "./judges/process-judge.js";
|
} from "./judges/process-judge.js";
|
||||||
|
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||||
import {
|
import {
|
||||||
appendCandidatesToJsonl,
|
appendCandidatesToJsonl,
|
||||||
selectPromotionCandidates,
|
selectPromotionCandidates,
|
||||||
@@ -72,6 +67,7 @@ import type {
|
|||||||
EvalBaselineFile,
|
EvalBaselineFile,
|
||||||
EvalCaseDefinition,
|
EvalCaseDefinition,
|
||||||
EvalDerivedSnapshot,
|
EvalDerivedSnapshot,
|
||||||
|
EvalProcessTraceSnapshot,
|
||||||
ProcessJudgeResult,
|
ProcessJudgeResult,
|
||||||
} from "./types.js";
|
} from "./types.js";
|
||||||
|
|
||||||
@@ -100,13 +96,15 @@ const STOP_TIMEOUT_MS = 10_000;
|
|||||||
type BackgroundJobType = "shadow-eval" | "summarize";
|
type BackgroundJobType = "shadow-eval" | "summarize";
|
||||||
|
|
||||||
interface ShadowEvalJob {
|
interface ShadowEvalJob {
|
||||||
classified: ClassifiedIntent;
|
classified: unknown;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
derived: DerivedState;
|
derived: unknown;
|
||||||
latencyMs?: number | undefined;
|
latencyMs?: number | undefined;
|
||||||
process?: ProcessTraceSnapshot | undefined;
|
process?: EvalProcessTraceSnapshot | undefined;
|
||||||
request: AgentRequest;
|
request: AgentRequest;
|
||||||
response: AgentResponse;
|
response: AgentResponse;
|
||||||
|
skill: string;
|
||||||
|
version: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface EvalObservation {
|
interface EvalObservation {
|
||||||
@@ -168,26 +166,75 @@ function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
|||||||
return JSON.parse(value) as T;
|
return JSON.parse(value) as T;
|
||||||
}
|
}
|
||||||
|
|
||||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
function asRecord(value: unknown): Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null
|
||||||
|
? (value as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getStringField(
|
||||||
|
record: Record<string, unknown>,
|
||||||
|
key: string,
|
||||||
|
): string | null {
|
||||||
|
const value = record[key];
|
||||||
|
return typeof value === "string" && value.trim() ? value : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNestedString(
|
||||||
|
record: Record<string, unknown>,
|
||||||
|
first: string,
|
||||||
|
second: string,
|
||||||
|
): string | null {
|
||||||
|
return getStringField(asRecord(record[first]), second);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseTier(value: unknown): 0 | 1 | 2 {
|
||||||
|
const parsed = typeof value === "number" ? value : Number(value);
|
||||||
|
return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] {
|
||||||
|
return value === "skip" ||
|
||||||
|
value === "template" ||
|
||||||
|
value === "llm" ||
|
||||||
|
value === "llm-with-tools"
|
||||||
|
? value
|
||||||
|
: "llm";
|
||||||
|
}
|
||||||
|
|
||||||
|
function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot {
|
||||||
|
const record = asRecord(derived);
|
||||||
|
const intent = getStringField(record, "intent") ??
|
||||||
|
getNestedString(record, "intent", "kind") ??
|
||||||
|
"unknown";
|
||||||
|
const phase = getStringField(record, "phase") ??
|
||||||
|
getNestedString(record, "phase", "kind") ??
|
||||||
|
"unknown";
|
||||||
|
const abstentionScore = record["abstentionScore"];
|
||||||
|
const suggestedMaxTokens = record["suggestedMaxTokens"];
|
||||||
|
|
||||||
return {
|
return {
|
||||||
...(derived.abstentionScore !== undefined
|
...(typeof abstentionScore === "number"
|
||||||
? { abstentionScore: derived.abstentionScore }
|
? { abstentionScore }
|
||||||
: {}),
|
: {}),
|
||||||
allowShortResponse: derived.allowShortResponse,
|
allowShortResponse: record["allowShortResponse"] === true,
|
||||||
intent: derived.intent.kind,
|
intent,
|
||||||
phase: derived.phase.kind,
|
phase,
|
||||||
responseMode: derived.responseMode,
|
responseMode: parseResponseMode(record["responseMode"]),
|
||||||
...(derived.routingDecision
|
...(record["routingDecision"]
|
||||||
? { routingDecision: derived.routingDecision }
|
? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] }
|
||||||
: {}),
|
: {}),
|
||||||
shouldRespond: derived.shouldRespond,
|
shouldRespond: record["shouldRespond"] !== false,
|
||||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
suggestedMaxTokens:
|
||||||
tier: derived.tier,
|
typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens)
|
||||||
|
? suggestedMaxTokens
|
||||||
|
: 192,
|
||||||
|
tier: parseTier(record["tier"]),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
|
function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined):
|
||||||
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
| NonNullable<EvalProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
||||||
| undefined {
|
| undefined {
|
||||||
return process?.steps.classify?.output.abstention;
|
return process?.steps.classify?.output.abstention;
|
||||||
}
|
}
|
||||||
@@ -208,17 +255,17 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
|||||||
message.message["input"],
|
message.message["input"],
|
||||||
"input",
|
"input",
|
||||||
);
|
);
|
||||||
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
|
const process = parseOptionalJsonField<EvalProcessTraceSnapshot>(
|
||||||
message.message["process"],
|
message.message["process"],
|
||||||
);
|
);
|
||||||
|
|
||||||
const job: ShadowEvalJob = {
|
const job: ShadowEvalJob = {
|
||||||
classified: parseJsonField<ClassifiedIntent>(
|
classified: parseJsonField<unknown>(
|
||||||
message.message["classified"],
|
message.message["classified"],
|
||||||
"classified",
|
"classified",
|
||||||
),
|
),
|
||||||
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
||||||
derived: parseJsonField<DerivedState>(
|
derived: parseJsonField<unknown>(
|
||||||
message.message["derived"],
|
message.message["derived"],
|
||||||
"derived",
|
"derived",
|
||||||
),
|
),
|
||||||
@@ -228,6 +275,8 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
|||||||
message.message["output"],
|
message.message["output"],
|
||||||
"output",
|
"output",
|
||||||
),
|
),
|
||||||
|
skill: message.message["skill"]?.trim() || input.request.skill || "onboarding",
|
||||||
|
version: Number(message.message["version"] ?? 1),
|
||||||
};
|
};
|
||||||
|
|
||||||
if (process) {
|
if (process) {
|
||||||
@@ -500,6 +549,8 @@ export class ShadowWorker {
|
|||||||
|
|
||||||
private async processShadowEval(message: StreamEntry): Promise<void> {
|
private async processShadowEval(message: StreamEntry): Promise<void> {
|
||||||
const job = parseShadowEvalJob(message);
|
const job = parseShadowEvalJob(message);
|
||||||
|
const judgeConfig = getSkillJudgeConfig(job.skill);
|
||||||
|
const derivedSnapshot = toDerivedSnapshot(job.derived);
|
||||||
const abstention = getTraceAbstention(job.process);
|
const abstention = getTraceAbstention(job.process);
|
||||||
const responseTextForValidation =
|
const responseTextForValidation =
|
||||||
job.response.text.trim()
|
job.response.text.trim()
|
||||||
@@ -507,18 +558,19 @@ export class ShadowWorker {
|
|||||||
: abstention?.action.kind === "no-response"
|
: abstention?.action.kind === "no-response"
|
||||||
? "[NO_RESPONSE]"
|
? "[NO_RESPONSE]"
|
||||||
: job.response.text;
|
: job.response.text;
|
||||||
const validation = validateOnboardingResponse(
|
const validation = judgeConfig.validator
|
||||||
responseTextForValidation,
|
? judgeConfig.validator(responseTextForValidation, job.derived)
|
||||||
job.derived,
|
: {
|
||||||
{
|
reasons: [],
|
||||||
allowShortResponse: job.derived.allowShortResponse,
|
status: "pass" as const,
|
||||||
},
|
text: responseTextForValidation,
|
||||||
);
|
};
|
||||||
const llmJudge = await judgeOnboardingWithLlm(
|
const llmJudge = await judgeSkillWithLlm(
|
||||||
this.llmClient,
|
this.llmClient,
|
||||||
|
job.skill,
|
||||||
toEvalCase(job, message.id),
|
toEvalCase(job, message.id),
|
||||||
job.response.text,
|
job.response.text,
|
||||||
toDerivedSnapshot(job.derived),
|
derivedSnapshot,
|
||||||
);
|
);
|
||||||
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
||||||
const llmJudgeErrored =
|
const llmJudgeErrored =
|
||||||
@@ -541,10 +593,12 @@ export class ShadowWorker {
|
|||||||
requestId: job.request.requestId,
|
requestId: job.request.requestId,
|
||||||
chatroomId: job.request.chatroomId,
|
chatroomId: job.request.chatroomId,
|
||||||
userId: job.request.userId,
|
userId: job.request.userId,
|
||||||
tier: String(job.derived.tier),
|
skill: job.skill,
|
||||||
intent: job.derived.intent.kind,
|
version: String(Math.max(2, job.version)),
|
||||||
phase: job.derived.phase.kind,
|
tier: String(derivedSnapshot.tier),
|
||||||
responseMode: job.derived.responseMode,
|
intent: derivedSnapshot.intent,
|
||||||
|
phase: derivedSnapshot.phase,
|
||||||
|
responseMode: derivedSnapshot.responseMode,
|
||||||
validatorStatus: validation.status,
|
validatorStatus: validation.status,
|
||||||
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||||
llmOverall: llmOverall === null ? "" : String(llmOverall),
|
llmOverall: llmOverall === null ? "" : String(llmOverall),
|
||||||
@@ -589,7 +643,12 @@ export class ShadowWorker {
|
|||||||
throw new Error("Missing process trace");
|
throw new Error("Missing process trace");
|
||||||
}
|
}
|
||||||
|
|
||||||
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
|
const judgeConfig = getSkillJudgeConfig(job.skill);
|
||||||
|
const ruleResult = judgeProcessRule(
|
||||||
|
toEvalCase(job, messageId).expected,
|
||||||
|
job.process,
|
||||||
|
judgeConfig.toolRegistryFactory(),
|
||||||
|
);
|
||||||
if (
|
if (
|
||||||
!getEvalProcessLlmJudgeEnabled() ||
|
!getEvalProcessLlmJudgeEnabled() ||
|
||||||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
|
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
|
||||||
|
|||||||
+3
-4
@@ -11,6 +11,8 @@ import type {
|
|||||||
} from "@skills/onboarding/runtime/trace";
|
} from "@skills/onboarding/runtime/trace";
|
||||||
import type { PrmFeatureVector } from "./judges/prm.js";
|
import type { PrmFeatureVector } from "./judges/prm.js";
|
||||||
|
|
||||||
|
export type EvalProcessTraceSnapshot = ProcessTraceSnapshot;
|
||||||
|
|
||||||
export interface EvalRoomContextMessage {
|
export interface EvalRoomContextMessage {
|
||||||
senderId: string;
|
senderId: string;
|
||||||
nickname: string;
|
nickname: string;
|
||||||
@@ -70,10 +72,7 @@ export interface RuleJudgeResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface LlmJudgeScores {
|
export interface LlmJudgeScores {
|
||||||
friendliness: number;
|
[axis: string]: number;
|
||||||
accuracy: number;
|
|
||||||
concision: number;
|
|
||||||
scope: number;
|
|
||||||
overall: number;
|
overall: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,211 @@
|
|||||||
|
import type { ConversationMessage } from "@agent/types";
|
||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import type { ChatMessage } from "@llm/types";
|
||||||
|
import {
|
||||||
|
loadPromptVariantManifest,
|
||||||
|
type PromptLayerDiff,
|
||||||
|
type PromptVariantManifest,
|
||||||
|
} from "@eval/prompt-variant";
|
||||||
|
import type { EvalArchiveEntry } from "@eval/archive";
|
||||||
|
import type { SliceKey } from "@eval/aggregator";
|
||||||
|
import {
|
||||||
|
basicPromptSnapshot,
|
||||||
|
getRequestText,
|
||||||
|
getVariantManifest,
|
||||||
|
type SkillProposeAdapter,
|
||||||
|
} from "../../../eval/loop/skill-propose-adapter.js";
|
||||||
|
import {
|
||||||
|
classifyModerationIntent,
|
||||||
|
type ModerationIntent,
|
||||||
|
type ModerationIntentKind,
|
||||||
|
} from "../intents.js";
|
||||||
|
import { MODERATION_PROMPT_DATA } from "../prompt-data.js";
|
||||||
|
|
||||||
|
interface ModerationDerivedSnapshot {
|
||||||
|
allowShortResponse: boolean;
|
||||||
|
intent: string;
|
||||||
|
phase: string;
|
||||||
|
responseMode: "skip" | "template" | "llm" | "llm-with-tools";
|
||||||
|
shouldRespond: boolean;
|
||||||
|
suggestedMaxTokens: number;
|
||||||
|
tier: 0 | 1 | 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatHistory(history: ConversationMessage[]): string {
|
||||||
|
if (history.length === 0) {
|
||||||
|
return "(none)";
|
||||||
|
}
|
||||||
|
|
||||||
|
return history
|
||||||
|
.slice(-5)
|
||||||
|
.map((message) => `${message.role}: ${message.content}`)
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatExamples(intent: ModerationIntent): string {
|
||||||
|
return MODERATION_PROMPT_DATA.examples[intent.kind]
|
||||||
|
.slice(0, 2)
|
||||||
|
.map((example) => `user: ${example.user}\nassistant: ${example.assistant}`)
|
||||||
|
.join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyDiff(
|
||||||
|
layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>,
|
||||||
|
diff: PromptLayerDiff,
|
||||||
|
): void {
|
||||||
|
const target = diff.layer === "tool-policy" ? "policy" : diff.layer;
|
||||||
|
if (target !== "context" &&
|
||||||
|
target !== "examples" &&
|
||||||
|
target !== "persona" &&
|
||||||
|
target !== "policy" &&
|
||||||
|
target !== "state") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (diff.op === "remove") {
|
||||||
|
layers[target] = "";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const text = diff.text?.trim();
|
||||||
|
if (!text) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (diff.op === "replace") {
|
||||||
|
layers[target] = text;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getManifest(
|
||||||
|
variantId: string,
|
||||||
|
manifests?: Record<string, PromptVariantManifest>,
|
||||||
|
): PromptVariantManifest | null {
|
||||||
|
const manifest = getVariantManifest(variantId, manifests);
|
||||||
|
if (!manifest) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return typeof manifest === "string"
|
||||||
|
? loadPromptVariantManifest(manifest)
|
||||||
|
: manifest;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMessages(
|
||||||
|
entry: Pick<EvalArchiveEntry, "request">,
|
||||||
|
variant?: PromptVariantManifest | null,
|
||||||
|
): ChatMessage[] {
|
||||||
|
const classified = classifyModerationIntent(getRequestText(entry.request));
|
||||||
|
const intent = classified.intent;
|
||||||
|
const layers = {
|
||||||
|
persona: MODERATION_PROMPT_DATA.persona.join("\n"),
|
||||||
|
policy: [
|
||||||
|
...MODERATION_PROMPT_DATA.policy.base,
|
||||||
|
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
|
||||||
|
].join("\n"),
|
||||||
|
state: ["[intent]", intent.kind].join("\n"),
|
||||||
|
examples: formatExamples(intent),
|
||||||
|
context: "",
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const diff of variant?.diff ?? []) {
|
||||||
|
applyDiff(layers, diff);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
role: "system",
|
||||||
|
content: [
|
||||||
|
layers.persona,
|
||||||
|
"",
|
||||||
|
"[policy]",
|
||||||
|
layers.policy,
|
||||||
|
"",
|
||||||
|
layers.state,
|
||||||
|
"",
|
||||||
|
"[examples]",
|
||||||
|
layers.examples,
|
||||||
|
...(layers.context ? ["", "[context]", layers.context] : []),
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
"[history]",
|
||||||
|
formatHistory([]),
|
||||||
|
"",
|
||||||
|
"[message]",
|
||||||
|
getRequestText(entry.request),
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot {
|
||||||
|
return {
|
||||||
|
allowShortResponse: true,
|
||||||
|
intent: slice.intent,
|
||||||
|
phase: slice.phase,
|
||||||
|
responseMode: slice.intent === "off-topic" ? "template" : "llm",
|
||||||
|
shouldRespond: slice.intent !== "off-topic",
|
||||||
|
suggestedMaxTokens: 192,
|
||||||
|
tier: slice.tier,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSuggestedMaxTokens(entry: EvalArchiveEntry): number {
|
||||||
|
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
|
||||||
|
return typeof derived.suggestedMaxTokens === "number"
|
||||||
|
? derived.suggestedMaxTokens
|
||||||
|
: 192;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 {
|
||||||
|
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
|
||||||
|
return derived.tier === 0 || derived.tier === 1 || derived.tier === 2
|
||||||
|
? derived.tier
|
||||||
|
: entry.tier;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const moderationProposeAdapter: SkillProposeAdapter = {
|
||||||
|
skill: "moderation",
|
||||||
|
|
||||||
|
buildSnapshot(slice, archiveEntry) {
|
||||||
|
const request = archiveEntry?.request ?? {
|
||||||
|
requestId: "proposal-snapshot",
|
||||||
|
skill: "moderation",
|
||||||
|
chatroomId: "proposal-room",
|
||||||
|
userId: "proposal-user",
|
||||||
|
nickname: "문의자",
|
||||||
|
trigger: "user-message" as const,
|
||||||
|
userMessage: `slice=${slice.intent}/${slice.phase}`,
|
||||||
|
};
|
||||||
|
return basicPromptSnapshot(buildMessages({ request }));
|
||||||
|
},
|
||||||
|
|
||||||
|
derivedFromSlice,
|
||||||
|
|
||||||
|
async generateResponse(
|
||||||
|
llmClient: LlmClient,
|
||||||
|
entry: EvalArchiveEntry,
|
||||||
|
variantId: string,
|
||||||
|
manifests?: Record<string, PromptVariantManifest>,
|
||||||
|
): Promise<string> {
|
||||||
|
const response = await llmClient.chatCompletion(
|
||||||
|
buildMessages(entry, getManifest(variantId, manifests)),
|
||||||
|
{
|
||||||
|
maxTokens: getSuggestedMaxTokens(entry),
|
||||||
|
temperature: 0,
|
||||||
|
tier: getTier(entry),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return response?.content.trim() ?? "";
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export function normalizeModerationIntentKind(value: string): ModerationIntentKind {
|
||||||
|
return value in MODERATION_PROMPT_DATA.examples
|
||||||
|
? (value as ModerationIntentKind)
|
||||||
|
: "off-topic";
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import { MODERATION_INTENT_TABLE } from "./intents.js";
|
||||||
|
import { createModerationToolRegistry } from "./tools.js";
|
||||||
|
|
||||||
|
export function getIntentKinds(): string[] {
|
||||||
|
return Object.keys(MODERATION_INTENT_TABLE);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getToolNames(): string[] {
|
||||||
|
return createModerationToolRegistry()
|
||||||
|
.list()
|
||||||
|
.map((tool) => tool.name);
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||||
|
import type { EvalArchiveEntry } from "@eval/archive";
|
||||||
|
import type { SliceKey } from "@eval/aggregator";
|
||||||
|
import {
|
||||||
|
getVariantManifest,
|
||||||
|
type SkillProposeAdapter,
|
||||||
|
} from "../../../eval/loop/skill-propose-adapter.js";
|
||||||
|
import {
|
||||||
|
PromptBuilder,
|
||||||
|
} from "../prompts/builder.js";
|
||||||
|
import type { DerivedState, Intent, OnboardingPhase } from "../state/onboarding-state.js";
|
||||||
|
import type { Blackboard, PromptContext } from "../types.js";
|
||||||
|
|
||||||
|
function emptyBlackboard(): Blackboard {
|
||||||
|
return {
|
||||||
|
flaggedIssues: [],
|
||||||
|
openQuestions: [],
|
||||||
|
recentNicknameAttempts: [],
|
||||||
|
toneMemory: {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function derivedFromSlice(slice: SliceKey): DerivedState {
|
||||||
|
return {
|
||||||
|
allowShortResponse: false,
|
||||||
|
intent: { kind: slice.intent } as Intent,
|
||||||
|
phase: { kind: slice.phase } as OnboardingPhase,
|
||||||
|
responseMode: "llm",
|
||||||
|
shouldRespond: true,
|
||||||
|
suggestedMaxTokens: 384,
|
||||||
|
tier: slice.tier,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultPromptContext(userMessage?: string): PromptContext {
|
||||||
|
return {
|
||||||
|
blackboard: emptyBlackboard(),
|
||||||
|
conversationHistory: [],
|
||||||
|
userMessage:
|
||||||
|
userMessage?.trim() ||
|
||||||
|
"현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDerived(entry: EvalArchiveEntry, slice?: SliceKey): DerivedState {
|
||||||
|
const derived = entry.derived as Partial<DerivedState>;
|
||||||
|
if (derived.intent && derived.phase) {
|
||||||
|
return derived as DerivedState;
|
||||||
|
}
|
||||||
|
return derivedFromSlice(
|
||||||
|
slice ?? {
|
||||||
|
skill: "onboarding",
|
||||||
|
tier: entry.tier,
|
||||||
|
intent: "unknown",
|
||||||
|
phase: "unknown",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export const onboardingProposeAdapter: SkillProposeAdapter = {
|
||||||
|
skill: "onboarding",
|
||||||
|
|
||||||
|
buildSnapshot(slice, archiveEntry) {
|
||||||
|
return PromptBuilder.forDerived(
|
||||||
|
archiveEntry ? getDerived(archiveEntry, slice) : derivedFromSlice(slice),
|
||||||
|
null,
|
||||||
|
defaultPromptContext(archiveEntry?.request.userMessage),
|
||||||
|
).buildSystemPrompt();
|
||||||
|
},
|
||||||
|
|
||||||
|
derivedFromSlice,
|
||||||
|
|
||||||
|
async generateResponse(
|
||||||
|
llmClient: LlmClient,
|
||||||
|
entry: EvalArchiveEntry,
|
||||||
|
variantId: string,
|
||||||
|
manifests?: Record<string, PromptVariantManifest>,
|
||||||
|
): Promise<string> {
|
||||||
|
let builder = PromptBuilder.forDerived(
|
||||||
|
getDerived(entry),
|
||||||
|
null,
|
||||||
|
defaultPromptContext(entry.request.userMessage),
|
||||||
|
);
|
||||||
|
const manifest = getVariantManifest(variantId, manifests);
|
||||||
|
if (manifest) {
|
||||||
|
builder = builder.applyVariant(manifest);
|
||||||
|
}
|
||||||
|
|
||||||
|
const derived = getDerived(entry);
|
||||||
|
const response = await llmClient.chatCompletion(builder.build(), {
|
||||||
|
maxTokens: derived.suggestedMaxTokens,
|
||||||
|
temperature: 0,
|
||||||
|
tier: derived.tier,
|
||||||
|
});
|
||||||
|
|
||||||
|
return response?.content.trim() ?? "";
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import { ONBOARDING_INTENT_TABLE } from "./intents.js";
|
||||||
|
import { createOnboardingToolRegistry } from "./tools.js";
|
||||||
|
|
||||||
|
export function getIntentKinds(): string[] {
|
||||||
|
return Object.keys(ONBOARDING_INTENT_TABLE);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getToolNames(): string[] {
|
||||||
|
return createOnboardingToolRegistry()
|
||||||
|
.list()
|
||||||
|
.map((tool) => tool.name);
|
||||||
|
}
|
||||||
@@ -1,6 +1,11 @@
|
|||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { resolve } from "node:path";
|
import { resolve } from "node:path";
|
||||||
|
|
||||||
|
import {
|
||||||
|
loadPromptVariantManifest,
|
||||||
|
type PromptLayerDiff,
|
||||||
|
type PromptLayerName,
|
||||||
|
type PromptVariantManifest,
|
||||||
|
} from "@eval/prompt-variant";
|
||||||
import { SkillRegistry } from "@skills/skill-loader";
|
import { SkillRegistry } from "@skills/skill-loader";
|
||||||
import type { ChatMessage } from "@llm/types";
|
import type { ChatMessage } from "@llm/types";
|
||||||
import {
|
import {
|
||||||
@@ -19,35 +24,12 @@ import {
|
|||||||
import { buildToolPolicy } from "./tool-policy.js";
|
import { buildToolPolicy } from "./tool-policy.js";
|
||||||
import { defaultExampleRetriever } from "./example-store.js";
|
import { defaultExampleRetriever } from "./example-store.js";
|
||||||
|
|
||||||
export type PromptLayerName =
|
export {
|
||||||
| "context"
|
loadPromptVariantManifest,
|
||||||
| "examples"
|
type PromptLayerDiff,
|
||||||
| "persona"
|
type PromptLayerName,
|
||||||
| "policy"
|
type PromptVariantManifest,
|
||||||
| "state"
|
} from "@eval/prompt-variant";
|
||||||
| "tool-policy";
|
|
||||||
|
|
||||||
export type PromptLayerOperation = "append" | "remove" | "replace";
|
|
||||||
|
|
||||||
export interface PromptLayerDiff {
|
|
||||||
layer: PromptLayerName;
|
|
||||||
op: PromptLayerOperation;
|
|
||||||
reason?: string | undefined;
|
|
||||||
text?: string | undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PromptVariantManifest {
|
|
||||||
description?: string | undefined;
|
|
||||||
diff: PromptLayerDiff[];
|
|
||||||
expectedImprovement?: string | undefined;
|
|
||||||
rationale?: string | undefined;
|
|
||||||
requiredResources?: string[] | undefined;
|
|
||||||
requiredTools?: string[] | undefined;
|
|
||||||
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
|
|
||||||
variantId: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
|
||||||
|
|
||||||
function getSkillRootDir(): string {
|
function getSkillRootDir(): string {
|
||||||
return resolve(process.cwd(), "src", "skills");
|
return resolve(process.cwd(), "src", "skills");
|
||||||
@@ -85,43 +67,6 @@ function applyLayerDiff(
|
|||||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function loadPromptVariantManifest(
|
|
||||||
variantId: string,
|
|
||||||
): PromptVariantManifest | null {
|
|
||||||
const normalized = variantId.trim();
|
|
||||||
if (!normalized) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (variantCache.has(normalized)) {
|
|
||||||
return variantCache.get(normalized) ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const path = resolve(
|
|
||||||
process.cwd(),
|
|
||||||
"src",
|
|
||||||
"eval",
|
|
||||||
"proposals",
|
|
||||||
`${normalized}.json`,
|
|
||||||
);
|
|
||||||
if (!existsSync(path)) {
|
|
||||||
variantCache.set(normalized, null);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
|
||||||
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
|
||||||
variantCache.set(normalized, null);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
variantCache.set(normalized, parsed);
|
|
||||||
return parsed;
|
|
||||||
} catch {
|
|
||||||
variantCache.set(normalized, null);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function skillIndexLayer(registry: SkillRegistry): string {
|
function skillIndexLayer(registry: SkillRegistry): string {
|
||||||
const lines = registry.list().map(
|
const lines = registry.list().map(
|
||||||
(manifest) =>
|
(manifest) =>
|
||||||
|
|||||||
@@ -189,6 +189,7 @@ export class OnboardingSkill implements Skill {
|
|||||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||||
request,
|
request,
|
||||||
response,
|
response,
|
||||||
|
skill: this.name,
|
||||||
});
|
});
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
@@ -222,6 +223,7 @@ export class OnboardingSkill implements Skill {
|
|||||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||||
request,
|
request,
|
||||||
response,
|
response,
|
||||||
|
skill: this.name,
|
||||||
});
|
});
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
@@ -294,6 +296,7 @@ export class OnboardingSkill implements Skill {
|
|||||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||||
request,
|
request,
|
||||||
response,
|
response,
|
||||||
|
skill: this.name,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,3 @@
|
|||||||
import { readFileSync } from "node:fs";
|
|
||||||
import { resolve } from "node:path";
|
|
||||||
|
|
||||||
import { checkOnboardingProgressTool } from "@tools/onboarding/check-onboarding-progress";
|
import { checkOnboardingProgressTool } from "@tools/onboarding/check-onboarding-progress";
|
||||||
import { getRoomContextTool } from "@tools/onboarding/get-room-context";
|
import { getRoomContextTool } from "@tools/onboarding/get-room-context";
|
||||||
import { lookupCommunityRuleTool } from "@tools/onboarding/lookup-community-rule";
|
import { lookupCommunityRuleTool } from "@tools/onboarding/lookup-community-rule";
|
||||||
@@ -17,32 +14,6 @@ const ONBOARDING_TOOLS = [
|
|||||||
verifyDiscordNicknameTool,
|
verifyDiscordNicknameTool,
|
||||||
] as const satisfies readonly Tool<unknown, unknown>[];
|
] as const satisfies readonly Tool<unknown, unknown>[];
|
||||||
|
|
||||||
function readSkillManifestToolNames(): string[] {
|
|
||||||
const skillPath = resolve(process.cwd(), "src", "skills", "onboarding", "SKILL.md");
|
|
||||||
const raw = readFileSync(skillPath, "utf8");
|
|
||||||
const match = raw.match(/^tools:\s*\[([^\]]*)\]/mu);
|
|
||||||
if (!match?.[1]) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
return match[1]
|
|
||||||
.split(",")
|
|
||||||
.map((item) => item.trim())
|
|
||||||
.filter(Boolean);
|
|
||||||
}
|
|
||||||
|
|
||||||
function assertToolManifestMatchesRegistry(): void {
|
|
||||||
const manifestNames = readSkillManifestToolNames().sort();
|
|
||||||
const registryNames = ONBOARDING_TOOLS.map((tool) => tool.name).sort();
|
|
||||||
if (manifestNames.join("\n") !== registryNames.join("\n")) {
|
|
||||||
throw new Error(
|
|
||||||
`Onboarding SKILL.md tools do not match registry: manifest=${manifestNames.join(",")} registry=${registryNames.join(",")}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
assertToolManifestMatchesRegistry();
|
|
||||||
|
|
||||||
export function createOnboardingToolRegistry(): ToolRegistry {
|
export function createOnboardingToolRegistry(): ToolRegistry {
|
||||||
const registry = new ToolRegistry();
|
const registry = new ToolRegistry();
|
||||||
for (const tool of ONBOARDING_TOOLS) {
|
for (const tool of ONBOARDING_TOOLS) {
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
|
||||||
|
import * as moderationBinding from "./moderation/manifest-binding.js";
|
||||||
|
import * as onboardingBinding from "./onboarding/manifest-binding.js";
|
||||||
|
|
||||||
|
interface ManifestBinding {
|
||||||
|
getIntentKinds(): string[];
|
||||||
|
getToolNames(): string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const BINDINGS: Record<string, ManifestBinding> = {
|
||||||
|
moderation: moderationBinding,
|
||||||
|
onboarding: onboardingBinding,
|
||||||
|
};
|
||||||
|
|
||||||
|
function sortUnique(values: readonly string[]): string[] {
|
||||||
|
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
|
||||||
|
}
|
||||||
|
|
||||||
|
function format(values: readonly string[]): string {
|
||||||
|
return values.length === 0 ? "(none)" : values.join(", ");
|
||||||
|
}
|
||||||
|
|
||||||
|
function diff(left: readonly string[], right: readonly string[]): string[] {
|
||||||
|
const rightSet = new Set(right);
|
||||||
|
return left.filter((item) => !rightSet.has(item));
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateManifest(
|
||||||
|
manifest: SkillManifest,
|
||||||
|
binding: ManifestBinding | undefined,
|
||||||
|
): string[] {
|
||||||
|
if (!binding) {
|
||||||
|
return [`${manifest.name}: missing manifest binding`];
|
||||||
|
}
|
||||||
|
|
||||||
|
const errors: string[] = [];
|
||||||
|
const manifestIntents = sortUnique(manifest.intents);
|
||||||
|
const codeIntents = sortUnique(binding.getIntentKinds());
|
||||||
|
const unknownIntents = diff(manifestIntents, codeIntents);
|
||||||
|
if (unknownIntents.length > 0) {
|
||||||
|
errors.push(
|
||||||
|
`${manifest.name}: SKILL.md intents not found in code: ${format(unknownIntents)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const manifestTools = sortUnique(manifest.tools);
|
||||||
|
const codeTools = sortUnique(binding.getToolNames());
|
||||||
|
const missingTools = diff(manifestTools, codeTools);
|
||||||
|
const undocumentedTools = diff(codeTools, manifestTools);
|
||||||
|
if (missingTools.length > 0 || undocumentedTools.length > 0) {
|
||||||
|
errors.push(
|
||||||
|
`${manifest.name}: SKILL.md tools mismatch; missingInRegistry=${format(missingTools)} missingInManifest=${format(undocumentedTools)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [key, path] of Object.entries(manifest.resources)) {
|
||||||
|
if (!existsSync(path)) {
|
||||||
|
errors.push(`${manifest.name}: resource not found ${key} -> ${path}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||||
|
const byPriority = new Map<number, string[]>();
|
||||||
|
for (const manifest of manifests) {
|
||||||
|
const current = byPriority.get(manifest.priority) ?? [];
|
||||||
|
current.push(manifest.name);
|
||||||
|
byPriority.set(manifest.priority, current);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...byPriority.entries()].flatMap(([priority, names]) =>
|
||||||
|
names.length > 1
|
||||||
|
? [`priority=${priority} is shared by skills: ${format(names.sort())}`]
|
||||||
|
: [],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
|
||||||
|
const registry = SkillRegistry.load(rootDir);
|
||||||
|
const manifests = registry.list();
|
||||||
|
return [
|
||||||
|
...manifests.flatMap((manifest) =>
|
||||||
|
validateManifest(manifest, BINDINGS[manifest.name]),
|
||||||
|
),
|
||||||
|
...validatePriorityConflicts(manifests),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
const invokedPath = process.argv[1] ? resolve(process.argv[1]) : "";
|
||||||
|
if (invokedPath === resolve(fileURLToPath(import.meta.url))) {
|
||||||
|
const errors = validateSkillManifests();
|
||||||
|
if (errors.length > 0) {
|
||||||
|
for (const error of errors) {
|
||||||
|
console.error(`[skills] ${error}`);
|
||||||
|
}
|
||||||
|
process.exitCode = 1;
|
||||||
|
} else {
|
||||||
|
console.info("[skills] manifests ok");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user