Make eval pipeline skill-aware and validate SKILL manifests
This commit is contained in:
1 parent
12e731a428
commit
54bc295ead
35 files changed
+1280
-499
No files matched your search
+36
-6
@@ -9,6 +9,7 @@ export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor";
|
||||
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
|
||||
|
||||
export interface SliceKey {
|
||||
skill?: string | undefined;
|
||||
tier: 0 | 1 | 2;
|
||||
intent: string;
|
||||
phase: string;
|
||||
@@ -111,6 +112,14 @@ function getNestedString(
|
||||
return typeof value === "string" && value.trim() ? value : null;
|
||||
}
|
||||
|
||||
function getStringField(
|
||||
record: Record<string, unknown>,
|
||||
key: string,
|
||||
): string | null {
|
||||
const value = record[key];
|
||||
return typeof value === "string" && value.trim() ? value : null;
|
||||
}
|
||||
|
||||
function extractLlmOverall(message: Record<string, string>): number | null {
|
||||
const direct = parseFiniteNumber(message["llmOverall"]);
|
||||
if (direct !== null) {
|
||||
@@ -177,9 +186,13 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
||||
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
|
||||
const intent =
|
||||
entry.message["intent"]?.trim() ||
|
||||
getStringField(derived, "intent") ||
|
||||
getNestedString(derived, "intent", "kind");
|
||||
const phase =
|
||||
entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind");
|
||||
entry.message["phase"]?.trim() ||
|
||||
getStringField(derived, "phase") ||
|
||||
getNestedString(derived, "phase", "kind");
|
||||
const skill = entry.message["skill"]?.trim();
|
||||
|
||||
if (tier === null || !intent || !phase) {
|
||||
return null;
|
||||
@@ -194,6 +207,7 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
||||
llmOverall: extractLlmOverall(entry.message),
|
||||
processJudgeScore: extractProcessJudgeScore(entry.message),
|
||||
slice: {
|
||||
...(skill ? { skill } : {}),
|
||||
tier,
|
||||
intent,
|
||||
phase,
|
||||
@@ -341,17 +355,28 @@ function average(sum: number, count: number): number | null {
|
||||
|
||||
function sliceKeyFromString(value: string): SliceKey | null {
|
||||
const parts = value.split(":");
|
||||
const [tierRaw, intent, phase] = parts;
|
||||
let skill: string | undefined;
|
||||
let offset = 0;
|
||||
let tierRaw = parts[0];
|
||||
if (parseTier(tierRaw) === null && parts.length >= 4) {
|
||||
skill = parts[0];
|
||||
offset = 1;
|
||||
tierRaw = parts[offset];
|
||||
}
|
||||
const intent = parts[offset + 1];
|
||||
const phase = parts[offset + 2];
|
||||
const tier = parseTier(tierRaw);
|
||||
if (tier === null || !intent || !phase) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const routingDecision = parts.length >= 6 ? parts[3] : undefined;
|
||||
const abstentionAction = parts.length >= 6 ? parts[4] : parts[3];
|
||||
const abstentionReason = parts.length >= 6 ? parts[5] : parts[4];
|
||||
const tail = parts.slice(offset + 3);
|
||||
const routingDecision = tail.length >= 3 ? tail[0] : undefined;
|
||||
const abstentionAction = tail.length >= 3 ? tail[1] : tail[0];
|
||||
const abstentionReason = tail.length >= 3 ? tail[2] : tail[1];
|
||||
|
||||
return {
|
||||
...(skill ? { skill } : {}),
|
||||
tier,
|
||||
intent,
|
||||
phase,
|
||||
@@ -376,7 +401,12 @@ function sliceKeyFromRedisKey(key: string): SliceKey | null {
|
||||
}
|
||||
|
||||
export function sliceKeyToString(key: SliceKey): string {
|
||||
const base = `${key.tier}:${key.intent}:${key.phase}`;
|
||||
const base = [
|
||||
...(key.skill ? [key.skill] : []),
|
||||
String(key.tier),
|
||||
key.intent,
|
||||
key.phase,
|
||||
].join(":");
|
||||
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
|
||||
return base;
|
||||
}
|
||||
|
||||
+53
-9
@@ -17,17 +17,19 @@ import {
|
||||
type SliceKey,
|
||||
type SliceReportRow,
|
||||
} from "./aggregator.js";
|
||||
import type { EvalClassifiedSnapshot, EvalDerivedSnapshot } from "./types.js";
|
||||
|
||||
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
|
||||
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
|
||||
|
||||
export interface EvalArchiveEntry {
|
||||
streamId: string;
|
||||
skill?: string | undefined;
|
||||
evaluatedAt: Date;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
derived: DerivedState;
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState | EvalDerivedSnapshot;
|
||||
classified: ClassifiedIntent | EvalClassifiedSnapshot;
|
||||
validation: ValidationResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
@@ -127,6 +129,44 @@ function getAbstention(
|
||||
};
|
||||
}
|
||||
|
||||
function getStringField(
|
||||
record: Record<string, unknown>,
|
||||
key: string,
|
||||
): string | null {
|
||||
const value = record[key];
|
||||
return typeof value === "string" && value.trim() ? value : null;
|
||||
}
|
||||
|
||||
function getNestedString(
|
||||
record: Record<string, unknown>,
|
||||
first: string,
|
||||
second: string,
|
||||
): string | null {
|
||||
const nested = asRecord(record[first]);
|
||||
return getStringField(nested, second);
|
||||
}
|
||||
|
||||
export function getArchiveDerivedIntent(derived: EvalArchiveEntry["derived"]): string {
|
||||
const record = asRecord(derived);
|
||||
return getStringField(record, "intent") ??
|
||||
getNestedString(record, "intent", "kind") ??
|
||||
"unknown";
|
||||
}
|
||||
|
||||
export function getArchiveDerivedPhase(derived: EvalArchiveEntry["derived"]): string {
|
||||
const record = asRecord(derived);
|
||||
return getStringField(record, "phase") ??
|
||||
getNestedString(record, "phase", "kind") ??
|
||||
"unknown";
|
||||
}
|
||||
|
||||
export function getArchiveRoutingDecision(
|
||||
derived: EvalArchiveEntry["derived"],
|
||||
): string | null {
|
||||
const record = asRecord(derived);
|
||||
return getNestedString(record, "routingDecision", "kind");
|
||||
}
|
||||
|
||||
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
|
||||
if (!value?.trim()) {
|
||||
return new Date(nowMs);
|
||||
@@ -164,13 +204,13 @@ function calculatePercentile(values: number[], percentile: number): number | nul
|
||||
|
||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
const abstention = getAbstention(entry);
|
||||
const routingDecision = getArchiveRoutingDecision(entry.derived);
|
||||
return {
|
||||
skill: entry.skill || entry.request.skill || "onboarding",
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
...(entry.derived.routingDecision
|
||||
? { routingDecision: entry.derived.routingDecision.kind }
|
||||
: {}),
|
||||
intent: getArchiveDerivedIntent(entry.derived),
|
||||
phase: getArchiveDerivedPhase(entry.derived),
|
||||
...(routingDecision ? { routingDecision } : {}),
|
||||
...(abstention.action ? { abstentionAction: abstention.action } : {}),
|
||||
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
|
||||
};
|
||||
@@ -192,7 +232,7 @@ function parseEvalArchiveEntry(
|
||||
entry.message["input"],
|
||||
"input",
|
||||
);
|
||||
const derived = parseJsonField<DerivedState>(
|
||||
const derived = parseJsonField<DerivedState | EvalDerivedSnapshot>(
|
||||
entry.message["derived"],
|
||||
"derived",
|
||||
);
|
||||
@@ -208,11 +248,12 @@ function parseEvalArchiveEntry(
|
||||
|
||||
const archiveEntry: EvalArchiveEntry = {
|
||||
streamId: entry.id,
|
||||
skill: entry.message["skill"]?.trim() || input.request.skill || "onboarding",
|
||||
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
|
||||
request: input.request,
|
||||
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
|
||||
derived,
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
classified: parseJsonField<ClassifiedIntent | EvalClassifiedSnapshot>(
|
||||
entry.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
@@ -248,9 +289,12 @@ async function ensureEvalArchiveIndexes(
|
||||
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
|
||||
),
|
||||
collection.createIndex({
|
||||
skill: 1,
|
||||
tier: 1,
|
||||
"derived.intent.kind": 1,
|
||||
"derived.intent": 1,
|
||||
"derived.phase.kind": 1,
|
||||
"derived.phase": 1,
|
||||
evaluatedAt: -1,
|
||||
}),
|
||||
]);
|
||||
|
||||
+12
-1
@@ -65,6 +65,7 @@ function parseSlice(value: string | undefined): SliceKey {
|
||||
const tier = Number(fields["tier"]);
|
||||
const intent = fields["intent"];
|
||||
const phase = fields["phase"];
|
||||
const skill = fields["skill"]?.trim();
|
||||
|
||||
if (
|
||||
(tier !== 0 && tier !== 1 && tier !== 2) ||
|
||||
@@ -77,6 +78,7 @@ function parseSlice(value: string | undefined): SliceKey {
|
||||
}
|
||||
|
||||
return {
|
||||
...(skill ? { skill } : {}),
|
||||
tier,
|
||||
intent,
|
||||
phase,
|
||||
@@ -620,9 +622,14 @@ async function main(): Promise<void> {
|
||||
} = await import("./loop/propose.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const slice = parseSlice(getFlagString(flags, "slice"));
|
||||
const skill = getFlagString(flags, "skill");
|
||||
const input = await buildProposalInputFromArchive(
|
||||
mongo.connection,
|
||||
parseSlice(getFlagString(flags, "slice")),
|
||||
{
|
||||
...slice,
|
||||
...(skill ? { skill } : {}),
|
||||
},
|
||||
{
|
||||
force: hasFlag(flags, "force"),
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
@@ -668,13 +675,16 @@ async function main(): Promise<void> {
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
|
||||
const skill = getFlagString(flags, "skill");
|
||||
const cases = await loadTournamentCases(mongo.connection, {
|
||||
caseIds: parseCsv(getFlagString(flags, "case-ids")),
|
||||
limit: maxMatches,
|
||||
skill,
|
||||
});
|
||||
const result = await runTournament(createEvalLlmClient(), {
|
||||
cases,
|
||||
maxMatches,
|
||||
skill,
|
||||
variantIds,
|
||||
});
|
||||
await saveTournamentResult(mongo.connection, result);
|
||||
@@ -691,6 +701,7 @@ async function main(): Promise<void> {
|
||||
try {
|
||||
const results = await queryTournamentResults(mongo.connection, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
skill: getFlagString(flags, "skill"),
|
||||
variantId: getFlagString(flags, "variant"),
|
||||
});
|
||||
for (const result of results) {
|
||||
|
||||
+12
-6
@@ -3,7 +3,12 @@ import type {
|
||||
EvalArchiveEntry,
|
||||
MongoConnectionLike,
|
||||
} from "./archive.js";
|
||||
import { getEvalArchiveCollection } from "./archive.js";
|
||||
import {
|
||||
getArchiveDerivedIntent,
|
||||
getArchiveDerivedPhase,
|
||||
getArchiveRoutingDecision,
|
||||
getEvalArchiveCollection,
|
||||
} from "./archive.js";
|
||||
import type { SliceKey } from "./aggregator.js";
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
@@ -41,6 +46,7 @@ function average(values: number[]): number | null {
|
||||
|
||||
function sliceKeyToString(slice: SliceKey): string {
|
||||
return [
|
||||
slice.skill ?? "onboarding",
|
||||
slice.tier,
|
||||
slice.intent,
|
||||
slice.phase,
|
||||
@@ -51,13 +57,13 @@ function sliceKeyToString(slice: SliceKey): string {
|
||||
}
|
||||
|
||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
const routingDecision = getArchiveRoutingDecision(entry.derived);
|
||||
return {
|
||||
skill: entry.skill || entry.request.skill || "onboarding",
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
...(entry.derived.routingDecision
|
||||
? { routingDecision: entry.derived.routingDecision.kind }
|
||||
: {}),
|
||||
intent: getArchiveDerivedIntent(entry.derived),
|
||||
phase: getArchiveDerivedPhase(entry.derived),
|
||||
...(routingDecision ? { routingDecision } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||
import { SkillRegistry } from "@skills/skill-loader";
|
||||
import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import { createModerationToolRegistry } from "@skills/moderation/tools";
|
||||
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
|
||||
import { runG1Static, runG4Permission } from "./gates.js";
|
||||
|
||||
function manifest(
|
||||
@@ -69,7 +70,7 @@ describe("governance gates", () => {
|
||||
|
||||
it("checks required tool and resource permissions in G4", () => {
|
||||
const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`);
|
||||
const toolRegistry = createBuiltInToolRegistry();
|
||||
const toolRegistry = createOnboardingToolRegistry();
|
||||
|
||||
expect(
|
||||
runG4Permission(
|
||||
@@ -80,6 +81,7 @@ describe("governance gates", () => {
|
||||
} as PromptVariantManifest,
|
||||
toolRegistry,
|
||||
skillRegistry,
|
||||
"onboarding",
|
||||
),
|
||||
).toMatchObject({
|
||||
passed: true,
|
||||
@@ -94,10 +96,31 @@ describe("governance gates", () => {
|
||||
} as PromptVariantManifest,
|
||||
toolRegistry,
|
||||
skillRegistry,
|
||||
"onboarding",
|
||||
),
|
||||
).toMatchObject({
|
||||
passed: false,
|
||||
reason: "missing-required-capability",
|
||||
});
|
||||
});
|
||||
|
||||
it("checks moderation resources against the target skill", () => {
|
||||
const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`);
|
||||
|
||||
expect(
|
||||
runG4Permission(
|
||||
{
|
||||
...manifest(),
|
||||
requiredResources: ["community-rules"],
|
||||
requiredTools: ["lookup-rule"],
|
||||
skill: "moderation",
|
||||
} as PromptVariantManifest,
|
||||
createModerationToolRegistry(),
|
||||
skillRegistry,
|
||||
"moderation",
|
||||
),
|
||||
).toMatchObject({
|
||||
passed: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
||||
import type { RuntimeFlagManifest } from "@skills/onboarding/runtime/runtime-flag-manifest";
|
||||
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||
import type { SkillRegistry } from "@skills/skill-loader";
|
||||
import type { ToolRegistry } from "@tools/registry";
|
||||
|
||||
@@ -10,6 +9,12 @@ export interface GateResult {
|
||||
reason?: string | undefined;
|
||||
}
|
||||
|
||||
interface RuntimeFlagManifest {
|
||||
description: string;
|
||||
flags: Record<string, unknown>;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
type GovernedManifest = PromptVariantManifest | RuntimeFlagManifest;
|
||||
|
||||
const riskyPatterns: Array<{ pattern: RegExp; reason: string }> = [
|
||||
@@ -94,14 +99,15 @@ export function runG4Permission(
|
||||
manifest: GovernedManifest,
|
||||
toolRegistry: ToolRegistry,
|
||||
skillRegistry: SkillRegistry,
|
||||
targetSkill: string,
|
||||
): GateResult {
|
||||
const record = asRecord(manifest);
|
||||
const requiredTools = readStringArray(record, "requiredTools");
|
||||
const requiredResources = readStringArray(record, "requiredResources");
|
||||
const missingTools = requiredTools.filter((toolName) => !toolRegistry.has(toolName));
|
||||
const onboarding = skillRegistry.get("onboarding");
|
||||
const skillManifest = skillRegistry.get(targetSkill);
|
||||
const missingResources = requiredResources.filter(
|
||||
(resource) => !onboarding?.resources[resource],
|
||||
(resource) => !skillManifest?.resources[resource],
|
||||
);
|
||||
|
||||
if (missingTools.length > 0 || missingResources.length > 0) {
|
||||
|
||||
@@ -2,7 +2,13 @@ import type {
|
||||
ProcessTraceSnapshot,
|
||||
ProcessTraceStepName,
|
||||
} from "@skills/onboarding/runtime/trace";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
||||
import {
|
||||
getArchiveDerivedIntent,
|
||||
getArchiveDerivedPhase,
|
||||
getArchiveRoutingDecision,
|
||||
type EvalArchiveEntry,
|
||||
type MongoConnectionLike,
|
||||
} from "../archive.js";
|
||||
import type { SliceKey } from "../aggregator.js";
|
||||
import type { PrmModel, PrmFeatureVector } from "./prm.js";
|
||||
import {
|
||||
@@ -136,14 +142,14 @@ export function attributeTraceWithPrm(
|
||||
function sliceFromEntry(entry: EvalArchiveEntry): SliceKey {
|
||||
const abstention = (entry.process as ProcessTraceSnapshot | undefined)
|
||||
?.steps.classify?.output.abstention;
|
||||
const routingDecision = getArchiveRoutingDecision(entry.derived);
|
||||
|
||||
return {
|
||||
skill: entry.skill || entry.request.skill || "onboarding",
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
...(entry.derived.routingDecision
|
||||
? { routingDecision: entry.derived.routingDecision.kind }
|
||||
: {}),
|
||||
intent: getArchiveDerivedIntent(entry.derived),
|
||||
phase: getArchiveDerivedPhase(entry.derived),
|
||||
...(routingDecision ? { routingDecision } : {}),
|
||||
...(abstention?.action.kind !== undefined &&
|
||||
abstention.action.kind !== "proceed"
|
||||
? { abstentionAction: abstention.action.kind }
|
||||
|
||||
@@ -7,29 +7,36 @@ import type {
|
||||
LlmJudgeResult,
|
||||
LlmJudgeScores,
|
||||
} from "../types.js";
|
||||
|
||||
const llmJudgeSchema = z.object({
|
||||
friendliness: z.number().int().min(1).max(5),
|
||||
accuracy: z.number().int().min(1).max(5),
|
||||
concision: z.number().int().min(1).max(5),
|
||||
scope: z.number().int().min(1).max(5),
|
||||
rationale: z.string().trim().min(1).max(200),
|
||||
});
|
||||
import { getSkillJudgeConfig } from "./skill-config.js";
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function createJudgeSchema(axes: readonly string[]): z.ZodObject<z.ZodRawShape> {
|
||||
return z.object({
|
||||
...Object.fromEntries(
|
||||
axes.map((axis) => [axis, z.number().int().min(1).max(5)]),
|
||||
),
|
||||
rationale: z.string().trim().min(1).max(200),
|
||||
});
|
||||
}
|
||||
|
||||
function buildJudgePrompt(
|
||||
skillName: string,
|
||||
evalCase: EvalCaseDefinition,
|
||||
responseText: string,
|
||||
derived: EvalDerivedSnapshot,
|
||||
): string {
|
||||
const config = getSkillJudgeConfig(skillName);
|
||||
const axes = config.axes.join(", ");
|
||||
const scoreShape = Object.fromEntries(config.axes.map((axis) => [axis, 1]));
|
||||
return [
|
||||
"다음 온보딩 에이전트 응답을 평가해 주세요.",
|
||||
"평가 축은 friendliness, accuracy, concision, scope 입니다.",
|
||||
`다음 ${skillName} 에이전트 응답을 평가해 주세요.`,
|
||||
`평가 축은 ${axes} 입니다.`,
|
||||
"각 점수는 1~5의 정수만 사용하세요.",
|
||||
"scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.",
|
||||
config.promptTemplate ??
|
||||
"accuracy는 기대 조건과 실제 답변의 사실성을, concision은 과도한 부가 설명 여부를 포함해 평가하세요.",
|
||||
"",
|
||||
"[request]",
|
||||
JSON.stringify(evalCase.input.request),
|
||||
@@ -48,12 +55,13 @@ function buildJudgePrompt(
|
||||
"[assistant_response]",
|
||||
responseText,
|
||||
"",
|
||||
'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}',
|
||||
`JSON object only: ${JSON.stringify({ ...scoreShape, rationale: "..." })}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function judgeOnboardingWithLlm(
|
||||
export async function judgeSkillWithLlm(
|
||||
llmClient: LlmClient,
|
||||
skillName: string,
|
||||
evalCase: EvalCaseDefinition,
|
||||
responseText: string,
|
||||
derived: EvalDerivedSnapshot,
|
||||
@@ -77,11 +85,11 @@ export async function judgeOnboardingWithLlm(
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
|
||||
`당신은 ${skillName} 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.`,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildJudgePrompt(evalCase, responseText, derived),
|
||||
content: buildJudgePrompt(skillName, evalCase, responseText, derived),
|
||||
},
|
||||
],
|
||||
{
|
||||
@@ -100,18 +108,18 @@ export async function judgeOnboardingWithLlm(
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = llmJudgeSchema.parse(JSON.parse(response.content));
|
||||
const config = getSkillJudgeConfig(skillName);
|
||||
const parsed = createJudgeSchema(config.axes).parse(
|
||||
JSON.parse(response.content),
|
||||
);
|
||||
const axisScores = Object.fromEntries(
|
||||
config.axes.map((axis) => [axis, Number(parsed[axis])]),
|
||||
);
|
||||
const scores: LlmJudgeScores = {
|
||||
friendliness: parsed.friendliness,
|
||||
accuracy: parsed.accuracy,
|
||||
concision: parsed.concision,
|
||||
scope: parsed.scope,
|
||||
...axisScores,
|
||||
overall: round(
|
||||
(parsed.friendliness +
|
||||
parsed.accuracy +
|
||||
parsed.concision +
|
||||
parsed.scope) /
|
||||
4,
|
||||
config.axes.reduce((sum, axis) => sum + Number(parsed[axis]), 0) /
|
||||
config.axes.length,
|
||||
),
|
||||
};
|
||||
|
||||
@@ -121,7 +129,7 @@ export async function judgeOnboardingWithLlm(
|
||||
endpoint: response.endpoint,
|
||||
provider: response.provider,
|
||||
scores,
|
||||
rationale: parsed.rationale,
|
||||
rationale: String(parsed["rationale"]),
|
||||
raw: response.content,
|
||||
};
|
||||
} catch {
|
||||
@@ -132,3 +140,18 @@ export async function judgeOnboardingWithLlm(
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function judgeOnboardingWithLlm(
|
||||
llmClient: LlmClient,
|
||||
evalCase: EvalCaseDefinition,
|
||||
responseText: string,
|
||||
derived: EvalDerivedSnapshot,
|
||||
): Promise<LlmJudgeResult> {
|
||||
return judgeSkillWithLlm(
|
||||
llmClient,
|
||||
"onboarding",
|
||||
evalCase,
|
||||
responseText,
|
||||
derived,
|
||||
);
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
|
||||
import { judgeProcessRule } from "./process-judge.js";
|
||||
import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js";
|
||||
|
||||
@@ -86,6 +87,8 @@ function makeTrace(
|
||||
}
|
||||
|
||||
describe("judgeProcessRule", () => {
|
||||
const toolRegistry = createOnboardingToolRegistry();
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env["EVAL_PRM_ENABLED"];
|
||||
delete process.env["EVAL_PRM_WEIGHTS_PATH"];
|
||||
@@ -102,6 +105,7 @@ describe("judgeProcessRule", () => {
|
||||
},
|
||||
},
|
||||
makeTrace(),
|
||||
toolRegistry,
|
||||
);
|
||||
|
||||
expect(result.score).toBe(1);
|
||||
@@ -122,6 +126,7 @@ describe("judgeProcessRule", () => {
|
||||
},
|
||||
},
|
||||
makeTrace(),
|
||||
toolRegistry,
|
||||
);
|
||||
|
||||
expect(
|
||||
@@ -155,6 +160,7 @@ describe("judgeProcessRule", () => {
|
||||
},
|
||||
],
|
||||
}),
|
||||
toolRegistry,
|
||||
);
|
||||
|
||||
expect(
|
||||
@@ -180,6 +186,7 @@ describe("judgeProcessRule", () => {
|
||||
},
|
||||
},
|
||||
}),
|
||||
toolRegistry,
|
||||
);
|
||||
|
||||
expect(
|
||||
@@ -214,6 +221,7 @@ describe("judgeProcessRule", () => {
|
||||
},
|
||||
},
|
||||
}),
|
||||
toolRegistry,
|
||||
);
|
||||
|
||||
expect(
|
||||
@@ -226,7 +234,7 @@ describe("judgeProcessRule", () => {
|
||||
});
|
||||
|
||||
it("skips optional expectations when a golden case has not opted in", () => {
|
||||
const result = judgeProcessRule({}, makeTrace());
|
||||
const result = judgeProcessRule({}, makeTrace(), toolRegistry);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "intent-match")?.skipped,
|
||||
@@ -261,6 +269,7 @@ describe("judgeProcessRule", () => {
|
||||
},
|
||||
},
|
||||
}),
|
||||
toolRegistry,
|
||||
);
|
||||
|
||||
expect(result.prmFeatures).toMatchObject({
|
||||
|
||||
@@ -2,8 +2,7 @@ import { z } from "zod";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import type { ToolRegistry } from "@tools/registry";
|
||||
import {
|
||||
getEvalProcessLlmJudgeMaxTokens,
|
||||
getEvalPrmEnabled,
|
||||
@@ -12,6 +11,7 @@ import {
|
||||
} from "../../config.js";
|
||||
import type {
|
||||
EvalCaseExpected,
|
||||
EvalProcessTraceSnapshot,
|
||||
ProcessJudgeCheck,
|
||||
ProcessJudgeResult,
|
||||
} from "../types.js";
|
||||
@@ -80,7 +80,7 @@ function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudge
|
||||
|
||||
function enrichWithPrm(
|
||||
result: ProcessJudgeResult,
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
): ProcessJudgeResult {
|
||||
if (!getEvalPrmEnabled()) {
|
||||
return result;
|
||||
@@ -125,7 +125,7 @@ function hasAllTools(
|
||||
return subset && actual.size === expected.size;
|
||||
}
|
||||
|
||||
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
||||
function countRecordedSteps(trace: EvalProcessTraceSnapshot): number {
|
||||
const scalarSteps = [
|
||||
trace.steps.classify,
|
||||
trace.steps.derive,
|
||||
@@ -138,18 +138,20 @@ function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
||||
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
|
||||
}
|
||||
|
||||
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
||||
function validateToolArgs(
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
toolRegistry: ToolRegistry,
|
||||
): ProcessJudgeCheck {
|
||||
const toolCalls = trace.steps.toolCalls ?? [];
|
||||
if (toolCalls.length === 0) {
|
||||
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
|
||||
}
|
||||
|
||||
const registry = createBuiltInToolRegistry();
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const call of toolCalls) {
|
||||
try {
|
||||
registry.get(call.output.name).argsSchema.parse(call.output.args);
|
||||
toolRegistry.get(call.output.name).argsSchema.parse(call.output.args);
|
||||
} catch {
|
||||
failures.push(call.output.name);
|
||||
}
|
||||
@@ -167,7 +169,8 @@ function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
||||
|
||||
export function judgeProcessRule(
|
||||
expected: EvalCaseExpected,
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
toolRegistry: ToolRegistry,
|
||||
): ProcessJudgeResult {
|
||||
const checks: ProcessJudgeCheck[] = [];
|
||||
|
||||
@@ -244,7 +247,7 @@ export function judgeProcessRule(
|
||||
);
|
||||
}
|
||||
|
||||
checks.push(validateToolArgs(trace));
|
||||
checks.push(validateToolArgs(trace, toolRegistry));
|
||||
|
||||
const toolPlanStepCount = trace.steps.plan?.output.toolPlanStepCount;
|
||||
if (toolPlanStepCount !== undefined) {
|
||||
@@ -318,7 +321,7 @@ export function judgeProcessRule(
|
||||
}
|
||||
|
||||
function buildProcessJudgePrompt(
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): string {
|
||||
return [
|
||||
@@ -341,7 +344,7 @@ function buildProcessJudgePrompt(
|
||||
|
||||
export async function judgeProcessWithLlm(
|
||||
llmClient: LlmClient,
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): Promise<ProcessJudgeResult> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { judgeOnboardingRuleCase } from "./rule-judge.js";
|
||||
import { judgeRuleCase } from "./rule-judge.js";
|
||||
import { getSkillJudgeConfig } from "./skill-config.js";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
function makeDerived(
|
||||
@@ -18,9 +19,12 @@ function makeDerived(
|
||||
};
|
||||
}
|
||||
|
||||
describe("judgeOnboardingRuleCase", () => {
|
||||
describe("judgeRuleCase", () => {
|
||||
const validator = getSkillJudgeConfig("onboarding").validator;
|
||||
|
||||
it("passes when active checks match", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
const derived = makeDerived();
|
||||
const result = judgeRuleCase(
|
||||
{
|
||||
intent: "status-question",
|
||||
responseMode: "llm",
|
||||
@@ -28,7 +32,12 @@ describe("judgeOnboardingRuleCase", () => {
|
||||
mustNotInclude: ["디스코드 서버"],
|
||||
},
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
makeDerived(),
|
||||
{
|
||||
derived,
|
||||
intent: derived.intent.kind,
|
||||
responseMode: derived.responseMode,
|
||||
validator,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
@@ -37,13 +46,19 @@ describe("judgeOnboardingRuleCase", () => {
|
||||
});
|
||||
|
||||
it("fails intent and mustNotInclude mismatches", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
const derived = makeDerived();
|
||||
const result = judgeRuleCase(
|
||||
{
|
||||
intent: "help-how",
|
||||
mustNotInclude: ["디스코드"],
|
||||
},
|
||||
"디스코드 서버에 먼저 들어오시면 돼요.",
|
||||
makeDerived(),
|
||||
{
|
||||
derived,
|
||||
intent: derived.intent.kind,
|
||||
responseMode: derived.responseMode,
|
||||
validator,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
@@ -56,12 +71,18 @@ describe("judgeOnboardingRuleCase", () => {
|
||||
});
|
||||
|
||||
it("treats rejecting validator results as a failed rule check", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
const derived = makeDerived({
|
||||
intent: { kind: "photo-question" },
|
||||
});
|
||||
const result = judgeRuleCase(
|
||||
{},
|
||||
"디스코드 서버에 들어가시면 돼요.",
|
||||
makeDerived({
|
||||
intent: { kind: "photo-question" },
|
||||
}),
|
||||
{
|
||||
derived,
|
||||
intent: derived.intent.kind,
|
||||
responseMode: derived.responseMode,
|
||||
validator,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
|
||||
+26
-120
@@ -1,6 +1,5 @@
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
|
||||
import type { SkillJudgeConfig } from "./skill-config.js";
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
@@ -14,18 +13,35 @@ export function judgeRuleCase(
|
||||
expected: EvalCaseExpected,
|
||||
responseText: string,
|
||||
actual: {
|
||||
derived?: unknown;
|
||||
intent: string;
|
||||
responseMode?: string | undefined;
|
||||
validator?: SkillJudgeConfig["validator"] | undefined;
|
||||
},
|
||||
): RuleJudgeResult {
|
||||
const checks: RuleJudgeCheck[] = [];
|
||||
const validation = actual.validator
|
||||
? actual.validator(responseText, actual.derived)
|
||||
: null;
|
||||
|
||||
checks.push({
|
||||
name: "validator",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
detail: "not configured for this skill",
|
||||
});
|
||||
if (validation) {
|
||||
checks.push({
|
||||
name: "validator",
|
||||
passed: validation.status !== "reject",
|
||||
skipped: false,
|
||||
detail:
|
||||
validation.reasons.length > 0
|
||||
? validation.reasons.join(", ")
|
||||
: validation.status,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "validator",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
detail: "not configured for this skill",
|
||||
});
|
||||
}
|
||||
|
||||
const expectedIntent = expected.intent?.trim();
|
||||
if (expectedIntent) {
|
||||
@@ -112,117 +128,7 @@ export function judgeRuleCase(
|
||||
passed: activeChecks.every((check) => check.passed),
|
||||
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
||||
checks,
|
||||
validatorStatus: "pass",
|
||||
validatorReasons: [],
|
||||
};
|
||||
}
|
||||
|
||||
export function judgeOnboardingRuleCase(
|
||||
expected: EvalCaseExpected,
|
||||
responseText: string,
|
||||
derived: DerivedState,
|
||||
): RuleJudgeResult {
|
||||
const validation = validateOnboardingResponse(responseText, derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
});
|
||||
const checks: RuleJudgeCheck[] = [];
|
||||
|
||||
checks.push({
|
||||
name: "validator",
|
||||
passed: validation.status !== "reject",
|
||||
skipped: false,
|
||||
detail:
|
||||
validation.reasons.length > 0
|
||||
? validation.reasons.join(", ")
|
||||
: validation.status,
|
||||
});
|
||||
|
||||
const expectedIntent = expected.intent?.trim();
|
||||
if (expectedIntent) {
|
||||
checks.push({
|
||||
name: "intent",
|
||||
passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind),
|
||||
skipped: false,
|
||||
detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "intent",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const expectedResponseMode = expected.responseMode?.trim();
|
||||
if (expectedResponseMode) {
|
||||
checks.push({
|
||||
name: "responseMode",
|
||||
passed:
|
||||
normalizeText(expectedResponseMode) ===
|
||||
normalizeText(derived.responseMode),
|
||||
skipped: false,
|
||||
detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "responseMode",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
|
||||
if (mustIncludeAny.length > 0) {
|
||||
const normalizedResponse = normalizeText(responseText);
|
||||
const matched = mustIncludeAny.find((candidate) =>
|
||||
normalizedResponse.includes(normalizeText(candidate)),
|
||||
);
|
||||
|
||||
checks.push({
|
||||
name: "mustIncludeAny",
|
||||
passed: Boolean(matched),
|
||||
skipped: false,
|
||||
detail: matched
|
||||
? `matched=${matched}`
|
||||
: `expected one of: ${mustIncludeAny.join(", ")}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "mustIncludeAny",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
|
||||
if (mustNotInclude.length > 0) {
|
||||
const normalizedResponse = normalizeText(responseText);
|
||||
const matched = mustNotInclude.find((candidate) =>
|
||||
normalizedResponse.includes(normalizeText(candidate)),
|
||||
);
|
||||
|
||||
checks.push({
|
||||
name: "mustNotInclude",
|
||||
passed: !matched,
|
||||
skipped: false,
|
||||
detail: matched ? `forbidden=${matched}` : undefined,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "mustNotInclude",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const activeChecks = checks.filter((check) => !check.skipped);
|
||||
const passedChecks = activeChecks.filter((check) => check.passed).length;
|
||||
|
||||
return {
|
||||
passed: activeChecks.every((check) => check.passed),
|
||||
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
||||
checks,
|
||||
validatorStatus: validation.status,
|
||||
validatorReasons: validation.reasons,
|
||||
validatorStatus: validation?.status ?? "pass",
|
||||
validatorReasons: validation?.reasons ?? [],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { ToolRegistry } from "@tools/registry";
|
||||
import { createModerationToolRegistry } from "@skills/moderation/tools";
|
||||
import { createOnboardingToolRegistry } from "@skills/onboarding/tools";
|
||||
import {
|
||||
validateOnboardingResponse,
|
||||
type ValidationResult,
|
||||
} from "@skills/onboarding/runtime/validator";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
export type SkillValidationResult = ValidationResult;
|
||||
|
||||
export interface SkillJudgeConfig {
|
||||
axes: readonly string[];
|
||||
promptTemplate?: string | undefined;
|
||||
toolRegistryFactory: () => ToolRegistry;
|
||||
tournamentInstruction: string;
|
||||
validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined;
|
||||
weights?: Record<string, number> | undefined;
|
||||
}
|
||||
|
||||
export const SKILL_JUDGE_CONFIGS: Record<string, SkillJudgeConfig> = {
|
||||
moderation: {
|
||||
axes: ["appropriateness", "accuracy", "helpfulness"],
|
||||
toolRegistryFactory: createModerationToolRegistry,
|
||||
tournamentInstruction:
|
||||
"커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.",
|
||||
},
|
||||
onboarding: {
|
||||
axes: ["friendliness", "accuracy", "concision", "scope"],
|
||||
toolRegistryFactory: createOnboardingToolRegistry,
|
||||
tournamentInstruction:
|
||||
"온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.",
|
||||
validator: (text, derived) =>
|
||||
validateOnboardingResponse(text, derived as DerivedState, {
|
||||
allowShortResponse: (derived as DerivedState).allowShortResponse,
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
export function getSkillJudgeConfig(skillName: string): SkillJudgeConfig {
|
||||
const config = SKILL_JUDGE_CONFIGS[skillName] ?? SKILL_JUDGE_CONFIGS["onboarding"];
|
||||
if (!config) {
|
||||
throw new Error("Missing onboarding judge config");
|
||||
}
|
||||
return config;
|
||||
}
|
||||
+41
-45
@@ -5,15 +5,11 @@ import { dirname, resolve } from "node:path";
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { SkillRegistry } from "@skills/skill-loader";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
||||
import {
|
||||
PromptBuilder,
|
||||
type PromptLayerDiff,
|
||||
type PromptLayerName,
|
||||
type PromptVariantManifest,
|
||||
} from "@skills/onboarding/prompts/builder";
|
||||
} from "@eval/prompt-variant";
|
||||
import {
|
||||
getEvalTournamentMinCases,
|
||||
getLlmProposeMaxTokens,
|
||||
@@ -26,6 +22,8 @@ import { queryArchivedReviewLabels } from "../review/label-archive.js";
|
||||
import type { ReviewLabel } from "../review/queue.js";
|
||||
import { runG1Static, runG4Permission } from "../governance/gates.js";
|
||||
import { promoteTrustTier } from "../governance/trust.js";
|
||||
import { getSkillJudgeConfig } from "../judges/skill-config.js";
|
||||
import { getSkillProposeAdapter } from "./skill-adapters.js";
|
||||
|
||||
export interface ProposalInput {
|
||||
slice: SliceKey;
|
||||
@@ -129,44 +127,12 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function derivedFromSlice(slice: SliceKey): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: { kind: slice.intent } as Intent,
|
||||
phase: { kind: slice.phase } as OnboardingPhase,
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 384,
|
||||
tier: slice.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function defaultPromptContext(): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
export function buildCurrentPromptSnapshot(
|
||||
slice: SliceKey,
|
||||
archiveEntry?: EvalArchiveEntry,
|
||||
): string {
|
||||
return PromptBuilder.forDerived(
|
||||
archiveEntry?.derived ?? derivedFromSlice(slice),
|
||||
null,
|
||||
defaultPromptContext(),
|
||||
).buildSystemPrompt();
|
||||
return getSkillProposeAdapter(slice.skill ?? archiveEntry?.skill ?? "onboarding")
|
||||
.buildSnapshot(slice, archiveEntry);
|
||||
}
|
||||
|
||||
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
|
||||
@@ -233,8 +199,9 @@ export function summarizeRootCauses(
|
||||
}
|
||||
|
||||
function buildProposalPrompt(input: ProposalInput): string {
|
||||
const skill = input.slice.skill ?? "onboarding";
|
||||
return [
|
||||
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
|
||||
`${skill} 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.`,
|
||||
"반드시 JSON object만 출력하세요.",
|
||||
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
|
||||
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
|
||||
@@ -271,7 +238,7 @@ export async function proposeVariants(
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.",
|
||||
`당신은 ${input.slice.skill ?? "onboarding"} 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.`,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
@@ -296,10 +263,12 @@ export async function proposeVariants(
|
||||
return {
|
||||
...proposal,
|
||||
diff,
|
||||
skill: input.slice.skill ?? "onboarding",
|
||||
variantId: stableHash({
|
||||
description: proposal.description,
|
||||
diff,
|
||||
rationale: proposal.rationale,
|
||||
skill: input.slice.skill ?? "onboarding",
|
||||
}),
|
||||
};
|
||||
});
|
||||
@@ -325,11 +294,34 @@ export async function buildProposalInputFromArchive(
|
||||
},
|
||||
): Promise<ProposalInput> {
|
||||
const limit = options?.limit ?? 50;
|
||||
const skill = slice.skill ?? "onboarding";
|
||||
const skillFilter =
|
||||
skill === "onboarding"
|
||||
? {
|
||||
$or: [
|
||||
{ skill },
|
||||
{ skill: { $exists: false } },
|
||||
],
|
||||
}
|
||||
: { skill };
|
||||
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
tier: slice.tier,
|
||||
"derived.intent.kind": slice.intent,
|
||||
"derived.phase.kind": slice.phase,
|
||||
$and: [
|
||||
skillFilter,
|
||||
{ tier: slice.tier },
|
||||
{
|
||||
$or: [
|
||||
{
|
||||
"derived.intent.kind": slice.intent,
|
||||
"derived.phase.kind": slice.phase,
|
||||
},
|
||||
{
|
||||
"derived.intent": slice.intent,
|
||||
"derived.phase": slice.phase,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(limit)
|
||||
@@ -373,13 +365,17 @@ export async function writePromptVariantProposal(
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
const manifest: PromptVariantProposal = {
|
||||
...proposal,
|
||||
skill: proposal.skill ?? "onboarding",
|
||||
trustTier: proposal.trustTier ?? "T1",
|
||||
};
|
||||
const g1 = runG1Static(manifest);
|
||||
const skill = manifest.skill ?? "onboarding";
|
||||
const judgeConfig = getSkillJudgeConfig(skill);
|
||||
const g4 = runG4Permission(
|
||||
manifest,
|
||||
createBuiltInToolRegistry(),
|
||||
judgeConfig.toolRegistryFactory(),
|
||||
SkillRegistry.load(resolve(process.cwd(), "src", "skills")),
|
||||
skill,
|
||||
);
|
||||
const gatedManifest: PromptVariantProposal = {
|
||||
...manifest,
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { moderationProposeAdapter } from "@skills/moderation/eval/propose-adapter";
|
||||
import { onboardingProposeAdapter } from "@skills/onboarding/eval/propose-adapter";
|
||||
import type { SkillProposeAdapter } from "./skill-propose-adapter.js";
|
||||
|
||||
const SKILL_PROPOSE_ADAPTERS: Record<string, SkillProposeAdapter> = {
|
||||
[moderationProposeAdapter.skill]: moderationProposeAdapter,
|
||||
[onboardingProposeAdapter.skill]: onboardingProposeAdapter,
|
||||
};
|
||||
|
||||
export function getSkillProposeAdapter(skill: string): SkillProposeAdapter {
|
||||
const adapter = SKILL_PROPOSE_ADAPTERS[skill] ?? SKILL_PROPOSE_ADAPTERS["onboarding"];
|
||||
if (!adapter) {
|
||||
throw new Error("Missing onboarding propose adapter");
|
||||
}
|
||||
return adapter;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||
import type { EvalArchiveEntry } from "../archive.js";
|
||||
import type { SliceKey } from "../aggregator.js";
|
||||
|
||||
export interface SkillProposeAdapter {
|
||||
readonly skill: string;
|
||||
buildSnapshot(slice: SliceKey, archiveEntry?: EvalArchiveEntry): string;
|
||||
derivedFromSlice(slice: SliceKey): unknown;
|
||||
generateResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string>;
|
||||
}
|
||||
|
||||
export function getVariantManifest(
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): PromptVariantManifest | string | null {
|
||||
if (variantId === "control") {
|
||||
return null;
|
||||
}
|
||||
return manifests?.[variantId] ?? variantId;
|
||||
}
|
||||
|
||||
export function basicPromptSnapshot(messages: ChatMessage[]): string {
|
||||
return messages
|
||||
.map((message) => `[${message.role}]\n${message.content}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
export function getRequestText(request: AgentRequest): string {
|
||||
return request.userMessage?.trim() || "";
|
||||
}
|
||||
+60
-62
@@ -2,11 +2,12 @@ import { createHash } from "node:crypto";
|
||||
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
||||
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||
import { getEvalTournamentMaxMatches } from "../../config.js";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
||||
import { getEvalArchiveCollection } from "../archive.js";
|
||||
import { getSkillJudgeConfig } from "../judges/skill-config.js";
|
||||
import { getSkillProposeAdapter } from "./skill-adapters.js";
|
||||
import {
|
||||
createEloRating,
|
||||
scoreForVerdict,
|
||||
@@ -30,6 +31,7 @@ export interface TournamentMatch {
|
||||
export interface TournamentResult {
|
||||
generatedAt: string;
|
||||
matches: TournamentMatch[];
|
||||
skill?: string | undefined;
|
||||
variants: EloRating[];
|
||||
}
|
||||
|
||||
@@ -58,6 +60,10 @@ const sideBySideJudgeSchema = z.object({
|
||||
verdict: z.enum(["A", "B", "tie"]),
|
||||
});
|
||||
|
||||
function entrySkill(entry: EvalArchiveEntry): string {
|
||||
return entry.skill || entry.request.skill || "onboarding";
|
||||
}
|
||||
|
||||
function stableHash(value: unknown): string {
|
||||
return createHash("sha256")
|
||||
.update(JSON.stringify(value))
|
||||
@@ -65,56 +71,18 @@ function stableHash(value: unknown): string {
|
||||
.slice(0, 12);
|
||||
}
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function promptContextForCase(entry: EvalArchiveEntry): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
function getVariantManifest(
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): PromptVariantManifest | string | null {
|
||||
if (variantId === "control") {
|
||||
return null;
|
||||
}
|
||||
return manifests?.[variantId] ?? variantId;
|
||||
}
|
||||
|
||||
async function generateTournamentResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string> {
|
||||
let builder = PromptBuilder.forDerived(
|
||||
entry.derived,
|
||||
null,
|
||||
promptContextForCase(entry),
|
||||
return getSkillProposeAdapter(entrySkill(entry)).generateResponse(
|
||||
llmClient,
|
||||
entry,
|
||||
variantId,
|
||||
manifests,
|
||||
);
|
||||
const manifest = getVariantManifest(variantId, manifests);
|
||||
if (manifest) {
|
||||
builder = builder.applyVariant(manifest);
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(builder.build(), {
|
||||
maxTokens: entry.derived.suggestedMaxTokens,
|
||||
temperature: 0,
|
||||
tier: entry.derived.tier,
|
||||
});
|
||||
|
||||
return response?.content.trim() ?? "";
|
||||
}
|
||||
|
||||
function buildJudgePrompt(
|
||||
@@ -122,9 +90,12 @@ function buildJudgePrompt(
|
||||
responseA: string,
|
||||
responseB: string,
|
||||
): string {
|
||||
const skill = entrySkill(entry);
|
||||
const config = getSkillJudgeConfig(skill);
|
||||
return [
|
||||
"아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.",
|
||||
"평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.",
|
||||
`아래 두 ${skill} 응답 중 더 나은 쪽을 고르세요.`,
|
||||
config.tournamentInstruction,
|
||||
`평가 축: ${config.axes.join(", ")}`,
|
||||
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
|
||||
"",
|
||||
"[request]",
|
||||
@@ -157,7 +128,7 @@ async function judgeTournamentMatch(
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.",
|
||||
`당신은 ${entrySkill(entry)} 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.`,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
@@ -201,6 +172,7 @@ async function ensureTournamentIndexes(
|
||||
): Promise<void> {
|
||||
await Promise.all([
|
||||
collection.createIndex({ generatedAt: -1 }),
|
||||
collection.createIndex({ skill: 1, generatedAt: -1 }),
|
||||
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
|
||||
]);
|
||||
}
|
||||
@@ -225,6 +197,7 @@ export async function runTournament(
|
||||
cases: EvalArchiveEntry[];
|
||||
maxMatches?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
skill?: string | undefined;
|
||||
variantIds: string[];
|
||||
variantManifests?: Record<string, PromptVariantManifest> | undefined;
|
||||
},
|
||||
@@ -303,9 +276,11 @@ export async function runTournament(
|
||||
}
|
||||
}
|
||||
|
||||
const skill = options.skill ?? options.cases[0]?.skill;
|
||||
return {
|
||||
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
|
||||
matches,
|
||||
...(skill ? { skill } : {}),
|
||||
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
|
||||
};
|
||||
}
|
||||
@@ -315,22 +290,35 @@ export async function loadTournamentCases(
|
||||
options: {
|
||||
caseIds?: string[] | undefined;
|
||||
limit?: number | undefined;
|
||||
skill?: string | undefined;
|
||||
},
|
||||
): Promise<EvalArchiveEntry[]> {
|
||||
const filter =
|
||||
options.caseIds && options.caseIds.length > 0
|
||||
? {
|
||||
streamId: {
|
||||
$in: options.caseIds,
|
||||
},
|
||||
}
|
||||
: {};
|
||||
const filter: Record<string, unknown> = {};
|
||||
if (options.caseIds && options.caseIds.length > 0) {
|
||||
filter.streamId = {
|
||||
$in: options.caseIds,
|
||||
};
|
||||
}
|
||||
if (options.skill) {
|
||||
if (options.skill === "onboarding") {
|
||||
filter.$or = [
|
||||
{ skill: options.skill },
|
||||
{ skill: { $exists: false } },
|
||||
];
|
||||
} else {
|
||||
filter.skill = options.skill;
|
||||
}
|
||||
}
|
||||
|
||||
return getEvalArchiveCollection(mongoConnection)
|
||||
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find(filter)
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(options.limit ?? getEvalTournamentMaxMatches())
|
||||
.toArray();
|
||||
return entries.map((entry) => ({
|
||||
...entry,
|
||||
skill: entry.skill || entry.request.skill || "onboarding",
|
||||
}));
|
||||
}
|
||||
|
||||
export async function saveTournamentResult(
|
||||
@@ -346,14 +334,24 @@ export async function queryTournamentResults(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
limit?: number | undefined;
|
||||
skill?: string | undefined;
|
||||
variantId?: string | undefined;
|
||||
},
|
||||
): Promise<TournamentResult[]> {
|
||||
const filter = options?.variantId
|
||||
? {
|
||||
"variants.id": options.variantId,
|
||||
}
|
||||
: {};
|
||||
const filter: Record<string, unknown> = {};
|
||||
if (options?.variantId) {
|
||||
filter["variants.id"] = options.variantId;
|
||||
}
|
||||
if (options?.skill) {
|
||||
if (options.skill === "onboarding") {
|
||||
filter.$or = [
|
||||
{ skill: options.skill },
|
||||
{ skill: { $exists: false } },
|
||||
];
|
||||
} else {
|
||||
filter.skill = options.skill;
|
||||
}
|
||||
}
|
||||
|
||||
return getTournamentCollection(mongoConnection)
|
||||
.find(filter)
|
||||
|
||||
+8
-4
@@ -16,7 +16,11 @@ import {
|
||||
} from "./anonymize.js";
|
||||
import type { EvalCaseDefinition } from "./types.js";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "./archive.js";
|
||||
import { getEvalArchiveCollection } from "./archive.js";
|
||||
import {
|
||||
getArchiveDerivedIntent,
|
||||
getArchiveDerivedPhase,
|
||||
getEvalArchiveCollection,
|
||||
} from "./archive.js";
|
||||
import { queryArchivedReviewLabels } from "./review/label-archive.js";
|
||||
import type { ReviewLabel } from "./review/queue.js";
|
||||
|
||||
@@ -59,7 +63,7 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
}
|
||||
|
||||
function sliceKey(entry: EvalArchiveEntry): string {
|
||||
return `${entry.tier}:${entry.derived.intent.kind}:${entry.derived.phase.kind}`;
|
||||
return `${entry.skill || entry.request.skill || "onboarding"}:${entry.tier}:${getArchiveDerivedIntent(entry.derived)}:${getArchiveDerivedPhase(entry.derived)}`;
|
||||
}
|
||||
|
||||
function sortByLowestScore(
|
||||
@@ -286,8 +290,8 @@ function candidateToEvalCase(candidate: PromotionCandidate): EvalCaseDefinition
|
||||
candidate.labelRationale === undefined
|
||||
? null
|
||||
: `humanRationale=${candidate.labelRationale}`,
|
||||
`derivedIntent=${candidate.derived.intent.kind}`,
|
||||
`derivedPhase=${candidate.derived.phase.kind}`,
|
||||
`derivedIntent=${getArchiveDerivedIntent(candidate.derived)}`,
|
||||
`derivedPhase=${getArchiveDerivedPhase(candidate.derived)}`,
|
||||
`assistantResponse=${candidate.response.text}`,
|
||||
candidate.betterResponse === undefined
|
||||
? null
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
export type PromptLayerName =
|
||||
| "context"
|
||||
| "examples"
|
||||
| "persona"
|
||||
| "policy"
|
||||
| "state"
|
||||
| "tool-policy";
|
||||
|
||||
export type PromptLayerOperation = "append" | "remove" | "replace";
|
||||
|
||||
export interface PromptLayerDiff {
|
||||
layer: PromptLayerName;
|
||||
op: PromptLayerOperation;
|
||||
reason?: string | undefined;
|
||||
text?: string | undefined;
|
||||
}
|
||||
|
||||
export interface PromptVariantManifest {
|
||||
description?: string | undefined;
|
||||
diff: PromptLayerDiff[];
|
||||
expectedImprovement?: string | undefined;
|
||||
rationale?: string | undefined;
|
||||
requiredResources?: string[] | undefined;
|
||||
requiredTools?: string[] | undefined;
|
||||
skill?: string | undefined;
|
||||
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||
|
||||
export function loadPromptVariantManifest(
|
||||
variantId: string,
|
||||
): PromptVariantManifest | null {
|
||||
const normalized = variantId.trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (variantCache.has(normalized)) {
|
||||
return variantCache.get(normalized) ?? null;
|
||||
}
|
||||
|
||||
const path = resolve(
|
||||
process.cwd(),
|
||||
"src",
|
||||
"eval",
|
||||
"proposals",
|
||||
`${normalized}.json`,
|
||||
);
|
||||
if (!existsSync(path)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
||||
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
variantCache.set(normalized, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -81,11 +81,12 @@ function toCalibrationPairs(
|
||||
pairs.push({
|
||||
evalStreamId: label.evalStreamId,
|
||||
humanScore: label.humanScore,
|
||||
llmAccuracy: archive.llmJudge.scores.accuracy,
|
||||
llmConcision: archive.llmJudge.scores.concision,
|
||||
llmFriendliness: archive.llmJudge.scores.friendliness,
|
||||
llmAccuracy: archive.llmJudge.scores.accuracy ?? archive.llmJudge.scores.overall,
|
||||
llmConcision: archive.llmJudge.scores.concision ?? archive.llmJudge.scores.overall,
|
||||
llmFriendliness:
|
||||
archive.llmJudge.scores.friendliness ?? archive.llmJudge.scores.overall,
|
||||
llmOverall: archive.llmJudge.scores.overall,
|
||||
llmScope: archive.llmJudge.scores.scope,
|
||||
llmScope: archive.llmJudge.scores.scope ?? archive.llmJudge.scores.overall,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,26 @@ import {
|
||||
} from "@skills/onboarding/runtime/router";
|
||||
import type { EvalArchiveEntry } from "./archive.js";
|
||||
|
||||
function isOnboardingArchiveEntry(
|
||||
entry: EvalArchiveEntry,
|
||||
): entry is EvalArchiveEntry & {
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState;
|
||||
} {
|
||||
return (
|
||||
(entry.skill === undefined ||
|
||||
entry.skill === "onboarding" ||
|
||||
entry.request.skill === "onboarding") &&
|
||||
typeof entry.derived === "object" &&
|
||||
entry.derived !== null &&
|
||||
typeof (entry.derived as { intent?: unknown }).intent === "object" &&
|
||||
typeof (entry.derived as { phase?: unknown }).phase === "object" &&
|
||||
typeof entry.classified === "object" &&
|
||||
entry.classified !== null &&
|
||||
typeof (entry.classified as { intent?: unknown }).intent === "object"
|
||||
);
|
||||
}
|
||||
|
||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
||||
}
|
||||
@@ -21,6 +41,9 @@ function requestFromEntry(entry: EvalArchiveEntry): AgentRequest {
|
||||
}
|
||||
|
||||
function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null {
|
||||
if (!isOnboardingArchiveEntry(entry)) {
|
||||
return null;
|
||||
}
|
||||
const staticDecision = routerDecisionKind(
|
||||
decidePreGenerationRouting({
|
||||
classified: entry.classified,
|
||||
@@ -57,11 +80,11 @@ function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null {
|
||||
}
|
||||
|
||||
function toClassified(entry: EvalArchiveEntry): ClassifiedIntent {
|
||||
return entry.classified;
|
||||
return entry.classified as ClassifiedIntent;
|
||||
}
|
||||
|
||||
function toDerived(entry: EvalArchiveEntry): DerivedState {
|
||||
return entry.derived;
|
||||
return entry.derived as DerivedState;
|
||||
}
|
||||
|
||||
export function buildRouterTrainingSamples(
|
||||
|
||||
+59
-19
@@ -30,12 +30,10 @@ import type {
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeProcessRule } from "./judges/process-judge.js";
|
||||
import {
|
||||
judgeOnboardingRuleCase,
|
||||
judgeRuleCase,
|
||||
} from "./judges/rule-judge.js";
|
||||
import { judgeRuleCase } from "./judges/rule-judge.js";
|
||||
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
@@ -161,11 +159,18 @@ function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function average(values: number[]): number | null {
|
||||
if (values.length === 0) {
|
||||
function average(values: Array<number | undefined>): number | null {
|
||||
const finiteValues = values.filter(
|
||||
(value): value is number =>
|
||||
typeof value === "number" && Number.isFinite(value),
|
||||
);
|
||||
if (finiteValues.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return round(values.reduce((sum, value) => sum + value, 0) / values.length);
|
||||
return round(
|
||||
finiteValues.reduce((sum, value) => sum + value, 0) /
|
||||
finiteValues.length,
|
||||
);
|
||||
}
|
||||
|
||||
function averageOrZero(values: number[]): number {
|
||||
@@ -741,8 +746,16 @@ async function evaluateCase(
|
||||
skill,
|
||||
llmClient,
|
||||
enableLlmJudge,
|
||||
effectiveSkill,
|
||||
)
|
||||
: evaluateGenericSkillCase(evalCase, input, skill),
|
||||
: evaluateGenericSkillCase(
|
||||
evalCase,
|
||||
input,
|
||||
skill,
|
||||
llmClient,
|
||||
enableLlmJudge,
|
||||
effectiveSkill,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -752,9 +765,11 @@ async function evaluateOnboardingCase(
|
||||
skill: Skill,
|
||||
llmClient: LlmClient,
|
||||
enableLlmJudge: boolean,
|
||||
skillName: string,
|
||||
): Promise<EvalCaseResult> {
|
||||
const classifier = new OnboardingIntentClassifier(llmClient);
|
||||
const processTrace = new ProcessTraceCollector();
|
||||
const judgeConfig = getSkillJudgeConfig(skillName);
|
||||
|
||||
return runWithProcessTrace(processTrace, async () => {
|
||||
const classified = await classifier.classify(
|
||||
@@ -775,14 +790,16 @@ async function evaluateOnboardingCase(
|
||||
);
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const derivedSnapshot = toDerivedSnapshot(derived);
|
||||
const ruleJudge = judgeOnboardingRuleCase(
|
||||
evalCase.expected,
|
||||
response.text,
|
||||
const ruleJudge = judgeRuleCase(evalCase.expected, response.text, {
|
||||
derived,
|
||||
);
|
||||
intent: derived.intent.kind,
|
||||
responseMode: derived.responseMode,
|
||||
validator: judgeConfig.validator,
|
||||
});
|
||||
const llmJudge = enableLlmJudge
|
||||
? await judgeOnboardingWithLlm(
|
||||
? await judgeSkillWithLlm(
|
||||
llmClient,
|
||||
skillName,
|
||||
evalCase,
|
||||
response.text,
|
||||
derivedSnapshot,
|
||||
@@ -795,6 +812,7 @@ async function evaluateOnboardingCase(
|
||||
const processJudge = judgeProcessRule(
|
||||
evalCase.expected,
|
||||
processTraceSnapshot,
|
||||
judgeConfig.toolRegistryFactory(),
|
||||
);
|
||||
|
||||
return {
|
||||
@@ -834,7 +852,12 @@ async function evaluateOnboardingCase(
|
||||
error: errorMessage,
|
||||
},
|
||||
},
|
||||
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
|
||||
ruleJudge: judgeRuleCase(evalCase.expected, "", {
|
||||
derived,
|
||||
intent: derived.intent.kind,
|
||||
responseMode: derived.responseMode,
|
||||
validator: judgeConfig.validator,
|
||||
}),
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
reason: "skill-error",
|
||||
@@ -842,6 +865,7 @@ async function evaluateOnboardingCase(
|
||||
processJudge: judgeProcessRule(
|
||||
evalCase.expected,
|
||||
processTraceSnapshot,
|
||||
judgeConfig.toolRegistryFactory(),
|
||||
),
|
||||
processTrace: processTraceSnapshot,
|
||||
error: errorMessage,
|
||||
@@ -854,8 +878,12 @@ async function evaluateGenericSkillCase(
|
||||
evalCase: EvalCaseDefinition,
|
||||
input: EvalCaseInput,
|
||||
skill: Skill,
|
||||
llmClient: LlmClient,
|
||||
enableLlmJudge: boolean,
|
||||
skillName: string,
|
||||
): Promise<EvalCaseResult> {
|
||||
const classified = classifyModerationIntent(input.request.userMessage ?? "");
|
||||
const judgeConfig = getSkillJudgeConfig(skillName);
|
||||
const derived: EvalDerivedSnapshot = {
|
||||
allowShortResponse: true,
|
||||
intent: classified.intent.kind,
|
||||
@@ -887,13 +915,23 @@ async function evaluateGenericSkillCase(
|
||||
request: input.request,
|
||||
response,
|
||||
ruleJudge: judgeRuleCase(evalCase.expected, response.text, {
|
||||
derived,
|
||||
intent: classified.intent.kind,
|
||||
responseMode: derived.responseMode,
|
||||
validator: judgeConfig.validator,
|
||||
}),
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
reason: "unsupported-skill",
|
||||
},
|
||||
llmJudge: enableLlmJudge
|
||||
? await judgeSkillWithLlm(
|
||||
llmClient,
|
||||
skillName,
|
||||
evalCase,
|
||||
response.text,
|
||||
derived,
|
||||
)
|
||||
: {
|
||||
skipped: true,
|
||||
reason: "disabled",
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
@@ -920,8 +958,10 @@ async function evaluateGenericSkillCase(
|
||||
},
|
||||
},
|
||||
ruleJudge: judgeRuleCase(evalCase.expected, "", {
|
||||
derived,
|
||||
intent: classified.intent.kind,
|
||||
responseMode: derived.responseMode,
|
||||
validator: judgeConfig.validator,
|
||||
}),
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
|
||||
@@ -61,6 +61,7 @@ describe("ShadowEvalBus", () => {
|
||||
derived,
|
||||
request,
|
||||
response,
|
||||
skill: "onboarding",
|
||||
});
|
||||
|
||||
expect(appendStream).toHaveBeenCalledTimes(1);
|
||||
@@ -68,7 +69,9 @@ describe("ShadowEvalBus", () => {
|
||||
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
|
||||
type: "shadow-eval",
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
tier: "0",
|
||||
version: "2",
|
||||
});
|
||||
|
||||
randomSpy.mockRestore();
|
||||
@@ -86,6 +89,7 @@ describe("ShadowEvalBus", () => {
|
||||
...response,
|
||||
text: "",
|
||||
},
|
||||
skill: "onboarding",
|
||||
});
|
||||
|
||||
expect(appendStream).not.toHaveBeenCalled();
|
||||
|
||||
+55
-18
@@ -1,11 +1,6 @@
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import {
|
||||
getBackgroundStreamMaxLen,
|
||||
getShadowEvalSamplingRate,
|
||||
@@ -14,13 +9,15 @@ import {
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
|
||||
export interface ShadowEvalEnvelope {
|
||||
classified: ClassifiedIntent;
|
||||
classified: unknown;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
derived: unknown;
|
||||
latencyMs?: number | undefined;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
process?: unknown;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: string;
|
||||
version?: 2 | undefined;
|
||||
}
|
||||
|
||||
export interface ShadowEvalPublisher {
|
||||
@@ -38,10 +35,46 @@ export function shouldSampleShadowEval(
|
||||
return randomValue < getShadowEvalSamplingRate(tier);
|
||||
}
|
||||
|
||||
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
|
||||
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
||||
| undefined {
|
||||
return process?.steps.classify?.output.abstention;
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
||||
const parsed = typeof value === "number" ? value : Number(value);
|
||||
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
||||
return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getDerivedTier(derived: unknown): 0 | 1 | 2 | null {
|
||||
return parseTier(asRecord(derived)["tier"]);
|
||||
}
|
||||
|
||||
function getTraceAbstention(process: unknown): Record<string, unknown> | null {
|
||||
const steps = asRecord(asRecord(process)["steps"]);
|
||||
const classify = asRecord(steps["classify"]);
|
||||
const output = asRecord(classify["output"]);
|
||||
const abstention = asRecord(output["abstention"]);
|
||||
return Object.keys(abstention).length > 0 ? abstention : null;
|
||||
}
|
||||
|
||||
function getAbstentionActionKind(abstention: Record<string, unknown> | null): string | null {
|
||||
const action = asRecord(abstention?.["action"]);
|
||||
const kind = action["kind"];
|
||||
return typeof kind === "string" && kind.trim() ? kind : null;
|
||||
}
|
||||
|
||||
function getAbstentionReason(abstention: Record<string, unknown> | null): string | null {
|
||||
const reason = abstention?.["reason"];
|
||||
return typeof reason === "string" && reason.trim() ? reason : null;
|
||||
}
|
||||
|
||||
function getAbstentionScore(abstention: Record<string, unknown> | null): number | null {
|
||||
const score = abstention?.["score"];
|
||||
return typeof score === "number" && Number.isFinite(score) ? score : null;
|
||||
}
|
||||
|
||||
export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||
@@ -51,11 +84,13 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||
|
||||
publish(envelope: ShadowEvalEnvelope): void {
|
||||
const abstention = getTraceAbstention(envelope.process);
|
||||
if (!envelope.response.text.trim() && abstention?.action.kind !== "no-response") {
|
||||
const actionKind = getAbstentionActionKind(abstention);
|
||||
if (!envelope.response.text.trim() && actionKind !== "no-response") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!shouldSampleShadowEval(envelope.derived.tier)) {
|
||||
const tier = getDerivedTier(envelope.derived);
|
||||
if (tier === null || !shouldSampleShadowEval(tier)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -66,7 +101,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||
requestId: envelope.request.requestId,
|
||||
chatroomId: envelope.request.chatroomId,
|
||||
userId: envelope.request.userId,
|
||||
tier: String(envelope.derived.tier),
|
||||
skill: envelope.skill,
|
||||
version: String(envelope.version ?? 2),
|
||||
tier: String(tier),
|
||||
latencyMs:
|
||||
envelope.latencyMs !== undefined ? String(envelope.latencyMs) : "",
|
||||
input: JSON.stringify({
|
||||
@@ -79,9 +116,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||
};
|
||||
|
||||
if (abstention) {
|
||||
fields.abstentionAction = abstention.action.kind;
|
||||
fields.abstentionReason = abstention.reason ?? "";
|
||||
fields.abstentionScore = String(abstention.score);
|
||||
fields.abstentionAction = actionKind ?? "";
|
||||
fields.abstentionReason = getAbstentionReason(abstention) ?? "";
|
||||
fields.abstentionScore = String(getAbstentionScore(abstention) ?? "");
|
||||
}
|
||||
|
||||
if (envelope.process) {
|
||||
|
||||
+100
-41
@@ -12,12 +12,6 @@ import { summarizeConversation } from "@conv/summarizer";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { AlertSender } from "@obs/alert";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import {
|
||||
getAlertThresholdDriftPct,
|
||||
@@ -49,13 +43,14 @@ import {
|
||||
type MongoConnectionLike,
|
||||
} from "./archive.js";
|
||||
import { computeDriftReports } from "./drift.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||
import { writeProcessAttributions } from "./judges/attribution.js";
|
||||
import { loadPrmModel } from "./judges/prm.js";
|
||||
import {
|
||||
judgeProcessRule,
|
||||
judgeProcessWithLlm,
|
||||
} from "./judges/process-judge.js";
|
||||
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||
import {
|
||||
appendCandidatesToJsonl,
|
||||
selectPromotionCandidates,
|
||||
@@ -72,6 +67,7 @@ import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
EvalProcessTraceSnapshot,
|
||||
ProcessJudgeResult,
|
||||
} from "./types.js";
|
||||
|
||||
@@ -100,13 +96,15 @@ const STOP_TIMEOUT_MS = 10_000;
|
||||
type BackgroundJobType = "shadow-eval" | "summarize";
|
||||
|
||||
interface ShadowEvalJob {
|
||||
classified: ClassifiedIntent;
|
||||
classified: unknown;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
derived: unknown;
|
||||
latencyMs?: number | undefined;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
process?: EvalProcessTraceSnapshot | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: string;
|
||||
version: number;
|
||||
}
|
||||
|
||||
interface EvalObservation {
|
||||
@@ -168,26 +166,75 @@ function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function getStringField(
|
||||
record: Record<string, unknown>,
|
||||
key: string,
|
||||
): string | null {
|
||||
const value = record[key];
|
||||
return typeof value === "string" && value.trim() ? value : null;
|
||||
}
|
||||
|
||||
function getNestedString(
|
||||
record: Record<string, unknown>,
|
||||
first: string,
|
||||
second: string,
|
||||
): string | null {
|
||||
return getStringField(asRecord(record[first]), second);
|
||||
}
|
||||
|
||||
function parseTier(value: unknown): 0 | 1 | 2 {
|
||||
const parsed = typeof value === "number" ? value : Number(value);
|
||||
return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0;
|
||||
}
|
||||
|
||||
function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] {
|
||||
return value === "skip" ||
|
||||
value === "template" ||
|
||||
value === "llm" ||
|
||||
value === "llm-with-tools"
|
||||
? value
|
||||
: "llm";
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot {
|
||||
const record = asRecord(derived);
|
||||
const intent = getStringField(record, "intent") ??
|
||||
getNestedString(record, "intent", "kind") ??
|
||||
"unknown";
|
||||
const phase = getStringField(record, "phase") ??
|
||||
getNestedString(record, "phase", "kind") ??
|
||||
"unknown";
|
||||
const abstentionScore = record["abstentionScore"];
|
||||
const suggestedMaxTokens = record["suggestedMaxTokens"];
|
||||
|
||||
return {
|
||||
...(derived.abstentionScore !== undefined
|
||||
? { abstentionScore: derived.abstentionScore }
|
||||
...(typeof abstentionScore === "number"
|
||||
? { abstentionScore }
|
||||
: {}),
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
...(derived.routingDecision
|
||||
? { routingDecision: derived.routingDecision }
|
||||
allowShortResponse: record["allowShortResponse"] === true,
|
||||
intent,
|
||||
phase,
|
||||
responseMode: parseResponseMode(record["responseMode"]),
|
||||
...(record["routingDecision"]
|
||||
? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] }
|
||||
: {}),
|
||||
shouldRespond: derived.shouldRespond,
|
||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
||||
tier: derived.tier,
|
||||
shouldRespond: record["shouldRespond"] !== false,
|
||||
suggestedMaxTokens:
|
||||
typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens)
|
||||
? suggestedMaxTokens
|
||||
: 192,
|
||||
tier: parseTier(record["tier"]),
|
||||
};
|
||||
}
|
||||
|
||||
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
|
||||
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
||||
function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined):
|
||||
| NonNullable<EvalProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
||||
| undefined {
|
||||
return process?.steps.classify?.output.abstention;
|
||||
}
|
||||
@@ -208,17 +255,17 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
message.message["input"],
|
||||
"input",
|
||||
);
|
||||
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
|
||||
const process = parseOptionalJsonField<EvalProcessTraceSnapshot>(
|
||||
message.message["process"],
|
||||
);
|
||||
|
||||
const job: ShadowEvalJob = {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
classified: parseJsonField<unknown>(
|
||||
message.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
||||
derived: parseJsonField<DerivedState>(
|
||||
derived: parseJsonField<unknown>(
|
||||
message.message["derived"],
|
||||
"derived",
|
||||
),
|
||||
@@ -228,6 +275,8 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
message.message["output"],
|
||||
"output",
|
||||
),
|
||||
skill: message.message["skill"]?.trim() || input.request.skill || "onboarding",
|
||||
version: Number(message.message["version"] ?? 1),
|
||||
};
|
||||
|
||||
if (process) {
|
||||
@@ -500,6 +549,8 @@ export class ShadowWorker {
|
||||
|
||||
private async processShadowEval(message: StreamEntry): Promise<void> {
|
||||
const job = parseShadowEvalJob(message);
|
||||
const judgeConfig = getSkillJudgeConfig(job.skill);
|
||||
const derivedSnapshot = toDerivedSnapshot(job.derived);
|
||||
const abstention = getTraceAbstention(job.process);
|
||||
const responseTextForValidation =
|
||||
job.response.text.trim()
|
||||
@@ -507,18 +558,19 @@ export class ShadowWorker {
|
||||
: abstention?.action.kind === "no-response"
|
||||
? "[NO_RESPONSE]"
|
||||
: job.response.text;
|
||||
const validation = validateOnboardingResponse(
|
||||
responseTextForValidation,
|
||||
job.derived,
|
||||
{
|
||||
allowShortResponse: job.derived.allowShortResponse,
|
||||
},
|
||||
);
|
||||
const llmJudge = await judgeOnboardingWithLlm(
|
||||
const validation = judgeConfig.validator
|
||||
? judgeConfig.validator(responseTextForValidation, job.derived)
|
||||
: {
|
||||
reasons: [],
|
||||
status: "pass" as const,
|
||||
text: responseTextForValidation,
|
||||
};
|
||||
const llmJudge = await judgeSkillWithLlm(
|
||||
this.llmClient,
|
||||
job.skill,
|
||||
toEvalCase(job, message.id),
|
||||
job.response.text,
|
||||
toDerivedSnapshot(job.derived),
|
||||
derivedSnapshot,
|
||||
);
|
||||
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
||||
const llmJudgeErrored =
|
||||
@@ -541,10 +593,12 @@ export class ShadowWorker {
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
phase: job.derived.phase.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
skill: job.skill,
|
||||
version: String(Math.max(2, job.version)),
|
||||
tier: String(derivedSnapshot.tier),
|
||||
intent: derivedSnapshot.intent,
|
||||
phase: derivedSnapshot.phase,
|
||||
responseMode: derivedSnapshot.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall: llmOverall === null ? "" : String(llmOverall),
|
||||
@@ -589,7 +643,12 @@ export class ShadowWorker {
|
||||
throw new Error("Missing process trace");
|
||||
}
|
||||
|
||||
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
|
||||
const judgeConfig = getSkillJudgeConfig(job.skill);
|
||||
const ruleResult = judgeProcessRule(
|
||||
toEvalCase(job, messageId).expected,
|
||||
job.process,
|
||||
judgeConfig.toolRegistryFactory(),
|
||||
);
|
||||
if (
|
||||
!getEvalProcessLlmJudgeEnabled() ||
|
||||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
|
||||
|
||||
+3
-4
@@ -11,6 +11,8 @@ import type {
|
||||
} from "@skills/onboarding/runtime/trace";
|
||||
import type { PrmFeatureVector } from "./judges/prm.js";
|
||||
|
||||
export type EvalProcessTraceSnapshot = ProcessTraceSnapshot;
|
||||
|
||||
export interface EvalRoomContextMessage {
|
||||
senderId: string;
|
||||
nickname: string;
|
||||
@@ -70,10 +72,7 @@ export interface RuleJudgeResult {
|
||||
}
|
||||
|
||||
export interface LlmJudgeScores {
|
||||
friendliness: number;
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
scope: number;
|
||||
[axis: string]: number;
|
||||
overall: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
import type { ConversationMessage } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import {
|
||||
loadPromptVariantManifest,
|
||||
type PromptLayerDiff,
|
||||
type PromptVariantManifest,
|
||||
} from "@eval/prompt-variant";
|
||||
import type { EvalArchiveEntry } from "@eval/archive";
|
||||
import type { SliceKey } from "@eval/aggregator";
|
||||
import {
|
||||
basicPromptSnapshot,
|
||||
getRequestText,
|
||||
getVariantManifest,
|
||||
type SkillProposeAdapter,
|
||||
} from "../../../eval/loop/skill-propose-adapter.js";
|
||||
import {
|
||||
classifyModerationIntent,
|
||||
type ModerationIntent,
|
||||
type ModerationIntentKind,
|
||||
} from "../intents.js";
|
||||
import { MODERATION_PROMPT_DATA } from "../prompt-data.js";
|
||||
|
||||
interface ModerationDerivedSnapshot {
|
||||
allowShortResponse: boolean;
|
||||
intent: string;
|
||||
phase: string;
|
||||
responseMode: "skip" | "template" | "llm" | "llm-with-tools";
|
||||
shouldRespond: boolean;
|
||||
suggestedMaxTokens: number;
|
||||
tier: 0 | 1 | 2;
|
||||
}
|
||||
|
||||
function formatHistory(history: ConversationMessage[]): string {
|
||||
if (history.length === 0) {
|
||||
return "(none)";
|
||||
}
|
||||
|
||||
return history
|
||||
.slice(-5)
|
||||
.map((message) => `${message.role}: ${message.content}`)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function formatExamples(intent: ModerationIntent): string {
|
||||
return MODERATION_PROMPT_DATA.examples[intent.kind]
|
||||
.slice(0, 2)
|
||||
.map((example) => `user: ${example.user}\nassistant: ${example.assistant}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function applyDiff(
|
||||
layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>,
|
||||
diff: PromptLayerDiff,
|
||||
): void {
|
||||
const target = diff.layer === "tool-policy" ? "policy" : diff.layer;
|
||||
if (target !== "context" &&
|
||||
target !== "examples" &&
|
||||
target !== "persona" &&
|
||||
target !== "policy" &&
|
||||
target !== "state") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (diff.op === "remove") {
|
||||
layers[target] = "";
|
||||
return;
|
||||
}
|
||||
|
||||
const text = diff.text?.trim();
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
if (diff.op === "replace") {
|
||||
layers[target] = text;
|
||||
return;
|
||||
}
|
||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||
}
|
||||
|
||||
function getManifest(
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): PromptVariantManifest | null {
|
||||
const manifest = getVariantManifest(variantId, manifests);
|
||||
if (!manifest) {
|
||||
return null;
|
||||
}
|
||||
return typeof manifest === "string"
|
||||
? loadPromptVariantManifest(manifest)
|
||||
: manifest;
|
||||
}
|
||||
|
||||
function buildMessages(
|
||||
entry: Pick<EvalArchiveEntry, "request">,
|
||||
variant?: PromptVariantManifest | null,
|
||||
): ChatMessage[] {
|
||||
const classified = classifyModerationIntent(getRequestText(entry.request));
|
||||
const intent = classified.intent;
|
||||
const layers = {
|
||||
persona: MODERATION_PROMPT_DATA.persona.join("\n"),
|
||||
policy: [
|
||||
...MODERATION_PROMPT_DATA.policy.base,
|
||||
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
|
||||
].join("\n"),
|
||||
state: ["[intent]", intent.kind].join("\n"),
|
||||
examples: formatExamples(intent),
|
||||
context: "",
|
||||
};
|
||||
|
||||
for (const diff of variant?.diff ?? []) {
|
||||
applyDiff(layers, diff);
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
layers.persona,
|
||||
"",
|
||||
"[policy]",
|
||||
layers.policy,
|
||||
"",
|
||||
layers.state,
|
||||
"",
|
||||
"[examples]",
|
||||
layers.examples,
|
||||
...(layers.context ? ["", "[context]", layers.context] : []),
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
"[history]",
|
||||
formatHistory([]),
|
||||
"",
|
||||
"[message]",
|
||||
getRequestText(entry.request),
|
||||
].join("\n"),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: true,
|
||||
intent: slice.intent,
|
||||
phase: slice.phase,
|
||||
responseMode: slice.intent === "off-topic" ? "template" : "llm",
|
||||
shouldRespond: slice.intent !== "off-topic",
|
||||
suggestedMaxTokens: 192,
|
||||
tier: slice.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function getSuggestedMaxTokens(entry: EvalArchiveEntry): number {
|
||||
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
|
||||
return typeof derived.suggestedMaxTokens === "number"
|
||||
? derived.suggestedMaxTokens
|
||||
: 192;
|
||||
}
|
||||
|
||||
function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 {
|
||||
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
|
||||
return derived.tier === 0 || derived.tier === 1 || derived.tier === 2
|
||||
? derived.tier
|
||||
: entry.tier;
|
||||
}
|
||||
|
||||
export const moderationProposeAdapter: SkillProposeAdapter = {
|
||||
skill: "moderation",
|
||||
|
||||
buildSnapshot(slice, archiveEntry) {
|
||||
const request = archiveEntry?.request ?? {
|
||||
requestId: "proposal-snapshot",
|
||||
skill: "moderation",
|
||||
chatroomId: "proposal-room",
|
||||
userId: "proposal-user",
|
||||
nickname: "문의자",
|
||||
trigger: "user-message" as const,
|
||||
userMessage: `slice=${slice.intent}/${slice.phase}`,
|
||||
};
|
||||
return basicPromptSnapshot(buildMessages({ request }));
|
||||
},
|
||||
|
||||
derivedFromSlice,
|
||||
|
||||
async generateResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string> {
|
||||
const response = await llmClient.chatCompletion(
|
||||
buildMessages(entry, getManifest(variantId, manifests)),
|
||||
{
|
||||
maxTokens: getSuggestedMaxTokens(entry),
|
||||
temperature: 0,
|
||||
tier: getTier(entry),
|
||||
},
|
||||
);
|
||||
|
||||
return response?.content.trim() ?? "";
|
||||
},
|
||||
};
|
||||
|
||||
export function normalizeModerationIntentKind(value: string): ModerationIntentKind {
|
||||
return value in MODERATION_PROMPT_DATA.examples
|
||||
? (value as ModerationIntentKind)
|
||||
: "off-topic";
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { MODERATION_INTENT_TABLE } from "./intents.js";
|
||||
import { createModerationToolRegistry } from "./tools.js";
|
||||
|
||||
export function getIntentKinds(): string[] {
|
||||
return Object.keys(MODERATION_INTENT_TABLE);
|
||||
}
|
||||
|
||||
export function getToolNames(): string[] {
|
||||
return createModerationToolRegistry()
|
||||
.list()
|
||||
.map((tool) => tool.name);
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||
import type { EvalArchiveEntry } from "@eval/archive";
|
||||
import type { SliceKey } from "@eval/aggregator";
|
||||
import {
|
||||
getVariantManifest,
|
||||
type SkillProposeAdapter,
|
||||
} from "../../../eval/loop/skill-propose-adapter.js";
|
||||
import {
|
||||
PromptBuilder,
|
||||
} from "../prompts/builder.js";
|
||||
import type { DerivedState, Intent, OnboardingPhase } from "../state/onboarding-state.js";
|
||||
import type { Blackboard, PromptContext } from "../types.js";
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function derivedFromSlice(slice: SliceKey): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: { kind: slice.intent } as Intent,
|
||||
phase: { kind: slice.phase } as OnboardingPhase,
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 384,
|
||||
tier: slice.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function defaultPromptContext(userMessage?: string): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage:
|
||||
userMessage?.trim() ||
|
||||
"현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
function getDerived(entry: EvalArchiveEntry, slice?: SliceKey): DerivedState {
|
||||
const derived = entry.derived as Partial<DerivedState>;
|
||||
if (derived.intent && derived.phase) {
|
||||
return derived as DerivedState;
|
||||
}
|
||||
return derivedFromSlice(
|
||||
slice ?? {
|
||||
skill: "onboarding",
|
||||
tier: entry.tier,
|
||||
intent: "unknown",
|
||||
phase: "unknown",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export const onboardingProposeAdapter: SkillProposeAdapter = {
|
||||
skill: "onboarding",
|
||||
|
||||
buildSnapshot(slice, archiveEntry) {
|
||||
return PromptBuilder.forDerived(
|
||||
archiveEntry ? getDerived(archiveEntry, slice) : derivedFromSlice(slice),
|
||||
null,
|
||||
defaultPromptContext(archiveEntry?.request.userMessage),
|
||||
).buildSystemPrompt();
|
||||
},
|
||||
|
||||
derivedFromSlice,
|
||||
|
||||
async generateResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string> {
|
||||
let builder = PromptBuilder.forDerived(
|
||||
getDerived(entry),
|
||||
null,
|
||||
defaultPromptContext(entry.request.userMessage),
|
||||
);
|
||||
const manifest = getVariantManifest(variantId, manifests);
|
||||
if (manifest) {
|
||||
builder = builder.applyVariant(manifest);
|
||||
}
|
||||
|
||||
const derived = getDerived(entry);
|
||||
const response = await llmClient.chatCompletion(builder.build(), {
|
||||
maxTokens: derived.suggestedMaxTokens,
|
||||
temperature: 0,
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
return response?.content.trim() ?? "";
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
import { ONBOARDING_INTENT_TABLE } from "./intents.js";
|
||||
import { createOnboardingToolRegistry } from "./tools.js";
|
||||
|
||||
export function getIntentKinds(): string[] {
|
||||
return Object.keys(ONBOARDING_INTENT_TABLE);
|
||||
}
|
||||
|
||||
export function getToolNames(): string[] {
|
||||
return createOnboardingToolRegistry()
|
||||
.list()
|
||||
.map((tool) => tool.name);
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import {
|
||||
loadPromptVariantManifest,
|
||||
type PromptLayerDiff,
|
||||
type PromptLayerName,
|
||||
type PromptVariantManifest,
|
||||
} from "@eval/prompt-variant";
|
||||
import { SkillRegistry } from "@skills/skill-loader";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import {
|
||||
@@ -19,35 +24,12 @@ import {
|
||||
import { buildToolPolicy } from "./tool-policy.js";
|
||||
import { defaultExampleRetriever } from "./example-store.js";
|
||||
|
||||
export type PromptLayerName =
|
||||
| "context"
|
||||
| "examples"
|
||||
| "persona"
|
||||
| "policy"
|
||||
| "state"
|
||||
| "tool-policy";
|
||||
|
||||
export type PromptLayerOperation = "append" | "remove" | "replace";
|
||||
|
||||
export interface PromptLayerDiff {
|
||||
layer: PromptLayerName;
|
||||
op: PromptLayerOperation;
|
||||
reason?: string | undefined;
|
||||
text?: string | undefined;
|
||||
}
|
||||
|
||||
export interface PromptVariantManifest {
|
||||
description?: string | undefined;
|
||||
diff: PromptLayerDiff[];
|
||||
expectedImprovement?: string | undefined;
|
||||
rationale?: string | undefined;
|
||||
requiredResources?: string[] | undefined;
|
||||
requiredTools?: string[] | undefined;
|
||||
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||
export {
|
||||
loadPromptVariantManifest,
|
||||
type PromptLayerDiff,
|
||||
type PromptLayerName,
|
||||
type PromptVariantManifest,
|
||||
} from "@eval/prompt-variant";
|
||||
|
||||
function getSkillRootDir(): string {
|
||||
return resolve(process.cwd(), "src", "skills");
|
||||
@@ -85,43 +67,6 @@ function applyLayerDiff(
|
||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||
}
|
||||
|
||||
export function loadPromptVariantManifest(
|
||||
variantId: string,
|
||||
): PromptVariantManifest | null {
|
||||
const normalized = variantId.trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (variantCache.has(normalized)) {
|
||||
return variantCache.get(normalized) ?? null;
|
||||
}
|
||||
|
||||
const path = resolve(
|
||||
process.cwd(),
|
||||
"src",
|
||||
"eval",
|
||||
"proposals",
|
||||
`${normalized}.json`,
|
||||
);
|
||||
if (!existsSync(path)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
||||
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
variantCache.set(normalized, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function skillIndexLayer(registry: SkillRegistry): string {
|
||||
const lines = registry.list().map(
|
||||
(manifest) =>
|
||||
|
||||
@@ -189,6 +189,7 @@ export class OnboardingSkill implements Skill {
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
skill: this.name,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
@@ -222,6 +223,7 @@ export class OnboardingSkill implements Skill {
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
skill: this.name,
|
||||
});
|
||||
|
||||
return response;
|
||||
@@ -294,6 +296,7 @@ export class OnboardingSkill implements Skill {
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
skill: this.name,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { checkOnboardingProgressTool } from "@tools/onboarding/check-onboarding-progress";
|
||||
import { getRoomContextTool } from "@tools/onboarding/get-room-context";
|
||||
import { lookupCommunityRuleTool } from "@tools/onboarding/lookup-community-rule";
|
||||
@@ -17,32 +14,6 @@ const ONBOARDING_TOOLS = [
|
||||
verifyDiscordNicknameTool,
|
||||
] as const satisfies readonly Tool<unknown, unknown>[];
|
||||
|
||||
function readSkillManifestToolNames(): string[] {
|
||||
const skillPath = resolve(process.cwd(), "src", "skills", "onboarding", "SKILL.md");
|
||||
const raw = readFileSync(skillPath, "utf8");
|
||||
const match = raw.match(/^tools:\s*\[([^\]]*)\]/mu);
|
||||
if (!match?.[1]) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return match[1]
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function assertToolManifestMatchesRegistry(): void {
|
||||
const manifestNames = readSkillManifestToolNames().sort();
|
||||
const registryNames = ONBOARDING_TOOLS.map((tool) => tool.name).sort();
|
||||
if (manifestNames.join("\n") !== registryNames.join("\n")) {
|
||||
throw new Error(
|
||||
`Onboarding SKILL.md tools do not match registry: manifest=${manifestNames.join(",")} registry=${registryNames.join(",")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
assertToolManifestMatchesRegistry();
|
||||
|
||||
export function createOnboardingToolRegistry(): ToolRegistry {
|
||||
const registry = new ToolRegistry();
|
||||
for (const tool of ONBOARDING_TOOLS) {
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
|
||||
import * as moderationBinding from "./moderation/manifest-binding.js";
|
||||
import * as onboardingBinding from "./onboarding/manifest-binding.js";
|
||||
|
||||
interface ManifestBinding {
|
||||
getIntentKinds(): string[];
|
||||
getToolNames(): string[];
|
||||
}
|
||||
|
||||
const BINDINGS: Record<string, ManifestBinding> = {
|
||||
moderation: moderationBinding,
|
||||
onboarding: onboardingBinding,
|
||||
};
|
||||
|
||||
function sortUnique(values: readonly string[]): string[] {
|
||||
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function format(values: readonly string[]): string {
|
||||
return values.length === 0 ? "(none)" : values.join(", ");
|
||||
}
|
||||
|
||||
function diff(left: readonly string[], right: readonly string[]): string[] {
|
||||
const rightSet = new Set(right);
|
||||
return left.filter((item) => !rightSet.has(item));
|
||||
}
|
||||
|
||||
function validateManifest(
|
||||
manifest: SkillManifest,
|
||||
binding: ManifestBinding | undefined,
|
||||
): string[] {
|
||||
if (!binding) {
|
||||
return [`${manifest.name}: missing manifest binding`];
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
const manifestIntents = sortUnique(manifest.intents);
|
||||
const codeIntents = sortUnique(binding.getIntentKinds());
|
||||
const unknownIntents = diff(manifestIntents, codeIntents);
|
||||
if (unknownIntents.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: SKILL.md intents not found in code: ${format(unknownIntents)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const manifestTools = sortUnique(manifest.tools);
|
||||
const codeTools = sortUnique(binding.getToolNames());
|
||||
const missingTools = diff(manifestTools, codeTools);
|
||||
const undocumentedTools = diff(codeTools, manifestTools);
|
||||
if (missingTools.length > 0 || undocumentedTools.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: SKILL.md tools mismatch; missingInRegistry=${format(missingTools)} missingInManifest=${format(undocumentedTools)}`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const [key, path] of Object.entries(manifest.resources)) {
|
||||
if (!existsSync(path)) {
|
||||
errors.push(`${manifest.name}: resource not found ${key} -> ${path}`);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||
const byPriority = new Map<number, string[]>();
|
||||
for (const manifest of manifests) {
|
||||
const current = byPriority.get(manifest.priority) ?? [];
|
||||
current.push(manifest.name);
|
||||
byPriority.set(manifest.priority, current);
|
||||
}
|
||||
|
||||
return [...byPriority.entries()].flatMap(([priority, names]) =>
|
||||
names.length > 1
|
||||
? [`priority=${priority} is shared by skills: ${format(names.sort())}`]
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
|
||||
const registry = SkillRegistry.load(rootDir);
|
||||
const manifests = registry.list();
|
||||
return [
|
||||
...manifests.flatMap((manifest) =>
|
||||
validateManifest(manifest, BINDINGS[manifest.name]),
|
||||
),
|
||||
...validatePriorityConflicts(manifests),
|
||||
];
|
||||
}
|
||||
|
||||
const invokedPath = process.argv[1] ? resolve(process.argv[1]) : "";
|
||||
if (invokedPath === resolve(fileURLToPath(import.meta.url))) {
|
||||
const errors = validateSkillManifests();
|
||||
if (errors.length > 0) {
|
||||
for (const error of errors) {
|
||||
console.error(`[skills] ${error}`);
|
||||
}
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
console.info("[skills] manifests ok");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user