diff --git a/package.json b/package.json index 4394043..cee07c5 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", - "lint": "eslint .", + "lint": "eslint . && npm run lint:skills", + "lint:skills": "node --import tsx src/skills/validate-manifests.ts", "lint:fix": "eslint . --fix", "format": "prettier --write .", "eval": "node --import tsx src/eval/cli.ts run --set=onboarding/cases", @@ -23,7 +24,9 @@ "eval:train-prm": "node --import tsx src/eval/cli.ts train-prm", "eval:train-router": "node --import tsx src/eval/cli.ts train-router", "eval:propose": "node --import tsx src/eval/cli.ts propose", + "eval:propose:moderation": "node --import tsx src/eval/cli.ts propose --skill=moderation", "eval:tournament": "node --import tsx src/eval/cli.ts tournament", + "eval:tournament:moderation": "node --import tsx src/eval/cli.ts tournament --skill=moderation", "eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results", "start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"", "start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"", diff --git a/src/eval/aggregator.ts b/src/eval/aggregator.ts index eef2cd2..d02ab8a 100644 --- a/src/eval/aggregator.ts +++ b/src/eval/aggregator.ts @@ -9,6 +9,7 @@ export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor"; const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:"; export interface SliceKey { + skill?: string | undefined; tier: 0 | 1 | 2; intent: string; phase: string; @@ -111,6 +112,14 @@ function getNestedString( return typeof value === "string" && value.trim() ? value : null; } +function getStringField( + record: Record, + key: string, +): string | null { + const value = record[key]; + return typeof value === "string" && value.trim() ? value : null; +} + function extractLlmOverall(message: Record): number | null { const direct = parseFiniteNumber(message["llmOverall"]); if (direct !== null) { @@ -177,9 +186,13 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null { const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]); const intent = entry.message["intent"]?.trim() || + getStringField(derived, "intent") || getNestedString(derived, "intent", "kind"); const phase = - entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind"); + entry.message["phase"]?.trim() || + getStringField(derived, "phase") || + getNestedString(derived, "phase", "kind"); + const skill = entry.message["skill"]?.trim(); if (tier === null || !intent || !phase) { return null; @@ -194,6 +207,7 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null { llmOverall: extractLlmOverall(entry.message), processJudgeScore: extractProcessJudgeScore(entry.message), slice: { + ...(skill ? { skill } : {}), tier, intent, phase, @@ -341,17 +355,28 @@ function average(sum: number, count: number): number | null { function sliceKeyFromString(value: string): SliceKey | null { const parts = value.split(":"); - const [tierRaw, intent, phase] = parts; + let skill: string | undefined; + let offset = 0; + let tierRaw = parts[0]; + if (parseTier(tierRaw) === null && parts.length >= 4) { + skill = parts[0]; + offset = 1; + tierRaw = parts[offset]; + } + const intent = parts[offset + 1]; + const phase = parts[offset + 2]; const tier = parseTier(tierRaw); if (tier === null || !intent || !phase) { return null; } - const routingDecision = parts.length >= 6 ? parts[3] : undefined; - const abstentionAction = parts.length >= 6 ? parts[4] : parts[3]; - const abstentionReason = parts.length >= 6 ? parts[5] : parts[4]; + const tail = parts.slice(offset + 3); + const routingDecision = tail.length >= 3 ? tail[0] : undefined; + const abstentionAction = tail.length >= 3 ? tail[1] : tail[0]; + const abstentionReason = tail.length >= 3 ? tail[2] : tail[1]; return { + ...(skill ? { skill } : {}), tier, intent, phase, @@ -376,7 +401,12 @@ function sliceKeyFromRedisKey(key: string): SliceKey | null { } export function sliceKeyToString(key: SliceKey): string { - const base = `${key.tier}:${key.intent}:${key.phase}`; + const base = [ + ...(key.skill ? [key.skill] : []), + String(key.tier), + key.intent, + key.phase, + ].join(":"); if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) { return base; } diff --git a/src/eval/archive.ts b/src/eval/archive.ts index f54c303..a323a91 100644 --- a/src/eval/archive.ts +++ b/src/eval/archive.ts @@ -17,17 +17,19 @@ import { type SliceKey, type SliceReportRow, } from "./aggregator.js"; +import type { EvalClassifiedSnapshot, EvalDerivedSnapshot } from "./types.js"; export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor"; export const EVAL_ARCHIVE_COLLECTION = "eval_archive"; export interface EvalArchiveEntry { streamId: string; + skill?: string | undefined; evaluatedAt: Date; request: AgentRequest; response: AgentResponse; - derived: DerivedState; - classified: ClassifiedIntent; + derived: DerivedState | EvalDerivedSnapshot; + classified: ClassifiedIntent | EvalClassifiedSnapshot; validation: ValidationResult; llmJudge: LlmJudgeResult; processJudge?: ProcessJudgeResult | undefined; @@ -127,6 +129,44 @@ function getAbstention( }; } +function getStringField( + record: Record, + key: string, +): string | null { + const value = record[key]; + return typeof value === "string" && value.trim() ? value : null; +} + +function getNestedString( + record: Record, + first: string, + second: string, +): string | null { + const nested = asRecord(record[first]); + return getStringField(nested, second); +} + +export function getArchiveDerivedIntent(derived: EvalArchiveEntry["derived"]): string { + const record = asRecord(derived); + return getStringField(record, "intent") ?? + getNestedString(record, "intent", "kind") ?? + "unknown"; +} + +export function getArchiveDerivedPhase(derived: EvalArchiveEntry["derived"]): string { + const record = asRecord(derived); + return getStringField(record, "phase") ?? + getNestedString(record, "phase", "kind") ?? + "unknown"; +} + +export function getArchiveRoutingDecision( + derived: EvalArchiveEntry["derived"], +): string | null { + const record = asRecord(derived); + return getNestedString(record, "routingDecision", "kind"); +} + function parseEvaluatedAt(value: string | undefined, nowMs: number): Date { if (!value?.trim()) { return new Date(nowMs); @@ -164,13 +204,13 @@ function calculatePercentile(values: number[], percentile: number): number | nul function getSlice(entry: EvalArchiveEntry): SliceKey { const abstention = getAbstention(entry); + const routingDecision = getArchiveRoutingDecision(entry.derived); return { + skill: entry.skill || entry.request.skill || "onboarding", tier: entry.tier, - intent: entry.derived.intent.kind, - phase: entry.derived.phase.kind, - ...(entry.derived.routingDecision - ? { routingDecision: entry.derived.routingDecision.kind } - : {}), + intent: getArchiveDerivedIntent(entry.derived), + phase: getArchiveDerivedPhase(entry.derived), + ...(routingDecision ? { routingDecision } : {}), ...(abstention.action ? { abstentionAction: abstention.action } : {}), ...(abstention.reason ? { abstentionReason: abstention.reason } : {}), }; @@ -192,7 +232,7 @@ function parseEvalArchiveEntry( entry.message["input"], "input", ); - const derived = parseJsonField( + const derived = parseJsonField( entry.message["derived"], "derived", ); @@ -208,11 +248,12 @@ function parseEvalArchiveEntry( const archiveEntry: EvalArchiveEntry = { streamId: entry.id, + skill: entry.message["skill"]?.trim() || input.request.skill || "onboarding", evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs), request: input.request, response: parseJsonField(entry.message["output"], "output"), derived, - classified: parseJsonField( + classified: parseJsonField( entry.message["classified"], "classified", ), @@ -248,9 +289,12 @@ async function ensureEvalArchiveIndexes( { expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 }, ), collection.createIndex({ + skill: 1, tier: 1, "derived.intent.kind": 1, + "derived.intent": 1, "derived.phase.kind": 1, + "derived.phase": 1, evaluatedAt: -1, }), ]); diff --git a/src/eval/cli.ts b/src/eval/cli.ts index 2e27d50..5b5439a 100644 --- a/src/eval/cli.ts +++ b/src/eval/cli.ts @@ -65,6 +65,7 @@ function parseSlice(value: string | undefined): SliceKey { const tier = Number(fields["tier"]); const intent = fields["intent"]; const phase = fields["phase"]; + const skill = fields["skill"]?.trim(); if ( (tier !== 0 && tier !== 1 && tier !== 2) || @@ -77,6 +78,7 @@ function parseSlice(value: string | undefined): SliceKey { } return { + ...(skill ? { skill } : {}), tier, intent, phase, @@ -620,9 +622,14 @@ async function main(): Promise { } = await import("./loop/propose.js"); const mongo = await connectMongoForEval(); try { + const slice = parseSlice(getFlagString(flags, "slice")); + const skill = getFlagString(flags, "skill"); const input = await buildProposalInputFromArchive( mongo.connection, - parseSlice(getFlagString(flags, "slice")), + { + ...slice, + ...(skill ? { skill } : {}), + }, { force: hasFlag(flags, "force"), limit: parsePositiveInteger(getFlagString(flags, "limit")), @@ -668,13 +675,16 @@ async function main(): Promise { const mongo = await connectMongoForEval(); try { const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches")); + const skill = getFlagString(flags, "skill"); const cases = await loadTournamentCases(mongo.connection, { caseIds: parseCsv(getFlagString(flags, "case-ids")), limit: maxMatches, + skill, }); const result = await runTournament(createEvalLlmClient(), { cases, maxMatches, + skill, variantIds, }); await saveTournamentResult(mongo.connection, result); @@ -691,6 +701,7 @@ async function main(): Promise { try { const results = await queryTournamentResults(mongo.connection, { limit: parsePositiveInteger(getFlagString(flags, "limit")), + skill: getFlagString(flags, "skill"), variantId: getFlagString(flags, "variant"), }); for (const result of results) { diff --git a/src/eval/drift.ts b/src/eval/drift.ts index b52024d..8cb8280 100644 --- a/src/eval/drift.ts +++ b/src/eval/drift.ts @@ -3,7 +3,12 @@ import type { EvalArchiveEntry, MongoConnectionLike, } from "./archive.js"; -import { getEvalArchiveCollection } from "./archive.js"; +import { + getArchiveDerivedIntent, + getArchiveDerivedPhase, + getArchiveRoutingDecision, + getEvalArchiveCollection, +} from "./archive.js"; import type { SliceKey } from "./aggregator.js"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -41,6 +46,7 @@ function average(values: number[]): number | null { function sliceKeyToString(slice: SliceKey): string { return [ + slice.skill ?? "onboarding", slice.tier, slice.intent, slice.phase, @@ -51,13 +57,13 @@ function sliceKeyToString(slice: SliceKey): string { } function getSlice(entry: EvalArchiveEntry): SliceKey { + const routingDecision = getArchiveRoutingDecision(entry.derived); return { + skill: entry.skill || entry.request.skill || "onboarding", tier: entry.tier, - intent: entry.derived.intent.kind, - phase: entry.derived.phase.kind, - ...(entry.derived.routingDecision - ? { routingDecision: entry.derived.routingDecision.kind } - : {}), + intent: getArchiveDerivedIntent(entry.derived), + phase: getArchiveDerivedPhase(entry.derived), + ...(routingDecision ? { routingDecision } : {}), }; } diff --git a/src/eval/governance/gates.test.ts b/src/eval/governance/gates.test.ts index 0e53601..9ff429d 100644 --- a/src/eval/governance/gates.test.ts +++ b/src/eval/governance/gates.test.ts @@ -1,8 +1,9 @@ import { describe, expect, it } from "vitest"; +import type { PromptVariantManifest } from "@eval/prompt-variant"; import { SkillRegistry } from "@skills/skill-loader"; -import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder"; -import { createBuiltInToolRegistry } from "@tools/builtins/index"; +import { createModerationToolRegistry } from "@skills/moderation/tools"; +import { createOnboardingToolRegistry } from "@skills/onboarding/tools"; import { runG1Static, runG4Permission } from "./gates.js"; function manifest( @@ -69,7 +70,7 @@ describe("governance gates", () => { it("checks required tool and resource permissions in G4", () => { const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`); - const toolRegistry = createBuiltInToolRegistry(); + const toolRegistry = createOnboardingToolRegistry(); expect( runG4Permission( @@ -80,6 +81,7 @@ describe("governance gates", () => { } as PromptVariantManifest, toolRegistry, skillRegistry, + "onboarding", ), ).toMatchObject({ passed: true, @@ -94,10 +96,31 @@ describe("governance gates", () => { } as PromptVariantManifest, toolRegistry, skillRegistry, + "onboarding", ), ).toMatchObject({ passed: false, reason: "missing-required-capability", }); }); + + it("checks moderation resources against the target skill", () => { + const skillRegistry = SkillRegistry.load(`${process.cwd()}/src/skills`); + + expect( + runG4Permission( + { + ...manifest(), + requiredResources: ["community-rules"], + requiredTools: ["lookup-rule"], + skill: "moderation", + } as PromptVariantManifest, + createModerationToolRegistry(), + skillRegistry, + "moderation", + ), + ).toMatchObject({ + passed: true, + }); + }); }); diff --git a/src/eval/governance/gates.ts b/src/eval/governance/gates.ts index 426faa0..26df998 100644 --- a/src/eval/governance/gates.ts +++ b/src/eval/governance/gates.ts @@ -1,5 +1,4 @@ -import type { PromptVariantManifest } from "@skills/onboarding/prompts/builder"; -import type { RuntimeFlagManifest } from "@skills/onboarding/runtime/runtime-flag-manifest"; +import type { PromptVariantManifest } from "@eval/prompt-variant"; import type { SkillRegistry } from "@skills/skill-loader"; import type { ToolRegistry } from "@tools/registry"; @@ -10,6 +9,12 @@ export interface GateResult { reason?: string | undefined; } +interface RuntimeFlagManifest { + description: string; + flags: Record; + variantId: string; +} + type GovernedManifest = PromptVariantManifest | RuntimeFlagManifest; const riskyPatterns: Array<{ pattern: RegExp; reason: string }> = [ @@ -94,14 +99,15 @@ export function runG4Permission( manifest: GovernedManifest, toolRegistry: ToolRegistry, skillRegistry: SkillRegistry, + targetSkill: string, ): GateResult { const record = asRecord(manifest); const requiredTools = readStringArray(record, "requiredTools"); const requiredResources = readStringArray(record, "requiredResources"); const missingTools = requiredTools.filter((toolName) => !toolRegistry.has(toolName)); - const onboarding = skillRegistry.get("onboarding"); + const skillManifest = skillRegistry.get(targetSkill); const missingResources = requiredResources.filter( - (resource) => !onboarding?.resources[resource], + (resource) => !skillManifest?.resources[resource], ); if (missingTools.length > 0 || missingResources.length > 0) { diff --git a/src/eval/judges/attribution.ts b/src/eval/judges/attribution.ts index e731d87..b2c1ff3 100644 --- a/src/eval/judges/attribution.ts +++ b/src/eval/judges/attribution.ts @@ -2,7 +2,13 @@ import type { ProcessTraceSnapshot, ProcessTraceStepName, } from "@skills/onboarding/runtime/trace"; -import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js"; +import { + getArchiveDerivedIntent, + getArchiveDerivedPhase, + getArchiveRoutingDecision, + type EvalArchiveEntry, + type MongoConnectionLike, +} from "../archive.js"; import type { SliceKey } from "../aggregator.js"; import type { PrmModel, PrmFeatureVector } from "./prm.js"; import { @@ -136,14 +142,14 @@ export function attributeTraceWithPrm( function sliceFromEntry(entry: EvalArchiveEntry): SliceKey { const abstention = (entry.process as ProcessTraceSnapshot | undefined) ?.steps.classify?.output.abstention; + const routingDecision = getArchiveRoutingDecision(entry.derived); return { + skill: entry.skill || entry.request.skill || "onboarding", tier: entry.tier, - intent: entry.derived.intent.kind, - phase: entry.derived.phase.kind, - ...(entry.derived.routingDecision - ? { routingDecision: entry.derived.routingDecision.kind } - : {}), + intent: getArchiveDerivedIntent(entry.derived), + phase: getArchiveDerivedPhase(entry.derived), + ...(routingDecision ? { routingDecision } : {}), ...(abstention?.action.kind !== undefined && abstention.action.kind !== "proceed" ? { abstentionAction: abstention.action.kind } diff --git a/src/eval/judges/llm-judge.ts b/src/eval/judges/llm-judge.ts index 9c70c71..012acae 100644 --- a/src/eval/judges/llm-judge.ts +++ b/src/eval/judges/llm-judge.ts @@ -7,29 +7,36 @@ import type { LlmJudgeResult, LlmJudgeScores, } from "../types.js"; - -const llmJudgeSchema = z.object({ - friendliness: z.number().int().min(1).max(5), - accuracy: z.number().int().min(1).max(5), - concision: z.number().int().min(1).max(5), - scope: z.number().int().min(1).max(5), - rationale: z.string().trim().min(1).max(200), -}); +import { getSkillJudgeConfig } from "./skill-config.js"; function round(value: number): number { return Number(value.toFixed(3)); } +function createJudgeSchema(axes: readonly string[]): z.ZodObject { + return z.object({ + ...Object.fromEntries( + axes.map((axis) => [axis, z.number().int().min(1).max(5)]), + ), + rationale: z.string().trim().min(1).max(200), + }); +} + function buildJudgePrompt( + skillName: string, evalCase: EvalCaseDefinition, responseText: string, derived: EvalDerivedSnapshot, ): string { + const config = getSkillJudgeConfig(skillName); + const axes = config.axes.join(", "); + const scoreShape = Object.fromEntries(config.axes.map((axis) => [axis, 1])); return [ - "다음 온보딩 에이전트 응답을 평가해 주세요.", - "평가 축은 friendliness, accuracy, concision, scope 입니다.", + `다음 ${skillName} 에이전트 응답을 평가해 주세요.`, + `평가 축은 ${axes} 입니다.`, "각 점수는 1~5의 정수만 사용하세요.", - "scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.", + config.promptTemplate ?? + "accuracy는 기대 조건과 실제 답변의 사실성을, concision은 과도한 부가 설명 여부를 포함해 평가하세요.", "", "[request]", JSON.stringify(evalCase.input.request), @@ -48,12 +55,13 @@ function buildJudgePrompt( "[assistant_response]", responseText, "", - 'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}', + `JSON object only: ${JSON.stringify({ ...scoreShape, rationale: "..." })}`, ].join("\n"); } -export async function judgeOnboardingWithLlm( +export async function judgeSkillWithLlm( llmClient: LlmClient, + skillName: string, evalCase: EvalCaseDefinition, responseText: string, derived: EvalDerivedSnapshot, @@ -77,11 +85,11 @@ export async function judgeOnboardingWithLlm( { role: "system", content: - "당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.", + `당신은 ${skillName} 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.`, }, { role: "user", - content: buildJudgePrompt(evalCase, responseText, derived), + content: buildJudgePrompt(skillName, evalCase, responseText, derived), }, ], { @@ -100,18 +108,18 @@ export async function judgeOnboardingWithLlm( } try { - const parsed = llmJudgeSchema.parse(JSON.parse(response.content)); + const config = getSkillJudgeConfig(skillName); + const parsed = createJudgeSchema(config.axes).parse( + JSON.parse(response.content), + ); + const axisScores = Object.fromEntries( + config.axes.map((axis) => [axis, Number(parsed[axis])]), + ); const scores: LlmJudgeScores = { - friendliness: parsed.friendliness, - accuracy: parsed.accuracy, - concision: parsed.concision, - scope: parsed.scope, + ...axisScores, overall: round( - (parsed.friendliness + - parsed.accuracy + - parsed.concision + - parsed.scope) / - 4, + config.axes.reduce((sum, axis) => sum + Number(parsed[axis]), 0) / + config.axes.length, ), }; @@ -121,7 +129,7 @@ export async function judgeOnboardingWithLlm( endpoint: response.endpoint, provider: response.provider, scores, - rationale: parsed.rationale, + rationale: String(parsed["rationale"]), raw: response.content, }; } catch { @@ -132,3 +140,18 @@ export async function judgeOnboardingWithLlm( }; } } + +export async function judgeOnboardingWithLlm( + llmClient: LlmClient, + evalCase: EvalCaseDefinition, + responseText: string, + derived: EvalDerivedSnapshot, +): Promise { + return judgeSkillWithLlm( + llmClient, + "onboarding", + evalCase, + responseText, + derived, + ); +} diff --git a/src/eval/judges/process-judge.test.ts b/src/eval/judges/process-judge.test.ts index c0a7e2a..15fa12e 100644 --- a/src/eval/judges/process-judge.test.ts +++ b/src/eval/judges/process-judge.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { afterEach, describe, expect, it } from "vitest"; import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; +import { createOnboardingToolRegistry } from "@skills/onboarding/tools"; import { judgeProcessRule } from "./process-judge.js"; import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js"; @@ -86,6 +87,8 @@ function makeTrace( } describe("judgeProcessRule", () => { + const toolRegistry = createOnboardingToolRegistry(); + afterEach(() => { delete process.env["EVAL_PRM_ENABLED"]; delete process.env["EVAL_PRM_WEIGHTS_PATH"]; @@ -102,6 +105,7 @@ describe("judgeProcessRule", () => { }, }, makeTrace(), + toolRegistry, ); expect(result.score).toBe(1); @@ -122,6 +126,7 @@ describe("judgeProcessRule", () => { }, }, makeTrace(), + toolRegistry, ); expect( @@ -155,6 +160,7 @@ describe("judgeProcessRule", () => { }, ], }), + toolRegistry, ); expect( @@ -180,6 +186,7 @@ describe("judgeProcessRule", () => { }, }, }), + toolRegistry, ); expect( @@ -214,6 +221,7 @@ describe("judgeProcessRule", () => { }, }, }), + toolRegistry, ); expect( @@ -226,7 +234,7 @@ describe("judgeProcessRule", () => { }); it("skips optional expectations when a golden case has not opted in", () => { - const result = judgeProcessRule({}, makeTrace()); + const result = judgeProcessRule({}, makeTrace(), toolRegistry); expect( result.checks.find((check) => check.rule === "intent-match")?.skipped, @@ -261,6 +269,7 @@ describe("judgeProcessRule", () => { }, }, }), + toolRegistry, ); expect(result.prmFeatures).toMatchObject({ diff --git a/src/eval/judges/process-judge.ts b/src/eval/judges/process-judge.ts index 357c742..e19576b 100644 --- a/src/eval/judges/process-judge.ts +++ b/src/eval/judges/process-judge.ts @@ -2,8 +2,7 @@ import { z } from "zod"; import type { AgentRequest } from "@agent/types"; import type { LlmClient } from "@llm/llm-client"; -import { createBuiltInToolRegistry } from "@tools/builtins/index"; -import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; +import type { ToolRegistry } from "@tools/registry"; import { getEvalProcessLlmJudgeMaxTokens, getEvalPrmEnabled, @@ -12,6 +11,7 @@ import { } from "../../config.js"; import type { EvalCaseExpected, + EvalProcessTraceSnapshot, ProcessJudgeCheck, ProcessJudgeResult, } from "../types.js"; @@ -80,7 +80,7 @@ function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudge function enrichWithPrm( result: ProcessJudgeResult, - trace: ProcessTraceSnapshot, + trace: EvalProcessTraceSnapshot, ): ProcessJudgeResult { if (!getEvalPrmEnabled()) { return result; @@ -125,7 +125,7 @@ function hasAllTools( return subset && actual.size === expected.size; } -function countRecordedSteps(trace: ProcessTraceSnapshot): number { +function countRecordedSteps(trace: EvalProcessTraceSnapshot): number { const scalarSteps = [ trace.steps.classify, trace.steps.derive, @@ -138,18 +138,20 @@ function countRecordedSteps(trace: ProcessTraceSnapshot): number { return scalarSteps + (trace.steps.toolCalls?.length ?? 0); } -function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck { +function validateToolArgs( + trace: EvalProcessTraceSnapshot, + toolRegistry: ToolRegistry, +): ProcessJudgeCheck { const toolCalls = trace.steps.toolCalls ?? []; if (toolCalls.length === 0) { return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls"); } - const registry = createBuiltInToolRegistry(); const failures: string[] = []; for (const call of toolCalls) { try { - registry.get(call.output.name).argsSchema.parse(call.output.args); + toolRegistry.get(call.output.name).argsSchema.parse(call.output.args); } catch { failures.push(call.output.name); } @@ -167,7 +169,8 @@ function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck { export function judgeProcessRule( expected: EvalCaseExpected, - trace: ProcessTraceSnapshot, + trace: EvalProcessTraceSnapshot, + toolRegistry: ToolRegistry, ): ProcessJudgeResult { const checks: ProcessJudgeCheck[] = []; @@ -244,7 +247,7 @@ export function judgeProcessRule( ); } - checks.push(validateToolArgs(trace)); + checks.push(validateToolArgs(trace, toolRegistry)); const toolPlanStepCount = trace.steps.plan?.output.toolPlanStepCount; if (toolPlanStepCount !== undefined) { @@ -318,7 +321,7 @@ export function judgeProcessRule( } function buildProcessJudgePrompt( - trace: ProcessTraceSnapshot, + trace: EvalProcessTraceSnapshot, options: { finalText: string; request: AgentRequest }, ): string { return [ @@ -341,7 +344,7 @@ function buildProcessJudgePrompt( export async function judgeProcessWithLlm( llmClient: LlmClient, - trace: ProcessTraceSnapshot, + trace: EvalProcessTraceSnapshot, options: { finalText: string; request: AgentRequest }, ): Promise { if (!llmClient.hasEndpoints()) { diff --git a/src/eval/judges/rule-judge.test.ts b/src/eval/judges/rule-judge.test.ts index 677133e..b5c9667 100644 --- a/src/eval/judges/rule-judge.test.ts +++ b/src/eval/judges/rule-judge.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; -import { judgeOnboardingRuleCase } from "./rule-judge.js"; +import { judgeRuleCase } from "./rule-judge.js"; +import { getSkillJudgeConfig } from "./skill-config.js"; import type { DerivedState } from "@skills/onboarding/state/onboarding-state"; function makeDerived( @@ -18,9 +19,12 @@ function makeDerived( }; } -describe("judgeOnboardingRuleCase", () => { +describe("judgeRuleCase", () => { + const validator = getSkillJudgeConfig("onboarding").validator; + it("passes when active checks match", () => { - const result = judgeOnboardingRuleCase( + const derived = makeDerived(); + const result = judgeRuleCase( { intent: "status-question", responseMode: "llm", @@ -28,7 +32,12 @@ describe("judgeOnboardingRuleCase", () => { mustNotInclude: ["디스코드 서버"], }, "이제 방 하트 인증만 남았어요.", - makeDerived(), + { + derived, + intent: derived.intent.kind, + responseMode: derived.responseMode, + validator, + }, ); expect(result.passed).toBe(true); @@ -37,13 +46,19 @@ describe("judgeOnboardingRuleCase", () => { }); it("fails intent and mustNotInclude mismatches", () => { - const result = judgeOnboardingRuleCase( + const derived = makeDerived(); + const result = judgeRuleCase( { intent: "help-how", mustNotInclude: ["디스코드"], }, "디스코드 서버에 먼저 들어오시면 돼요.", - makeDerived(), + { + derived, + intent: derived.intent.kind, + responseMode: derived.responseMode, + validator, + }, ); expect(result.passed).toBe(false); @@ -56,12 +71,18 @@ describe("judgeOnboardingRuleCase", () => { }); it("treats rejecting validator results as a failed rule check", () => { - const result = judgeOnboardingRuleCase( + const derived = makeDerived({ + intent: { kind: "photo-question" }, + }); + const result = judgeRuleCase( {}, "디스코드 서버에 들어가시면 돼요.", - makeDerived({ - intent: { kind: "photo-question" }, - }), + { + derived, + intent: derived.intent.kind, + responseMode: derived.responseMode, + validator, + }, ); expect(result.passed).toBe(false); diff --git a/src/eval/judges/rule-judge.ts b/src/eval/judges/rule-judge.ts index 72e6e07..a37956d 100644 --- a/src/eval/judges/rule-judge.ts +++ b/src/eval/judges/rule-judge.ts @@ -1,6 +1,5 @@ -import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator"; -import type { DerivedState } from "@skills/onboarding/state/onboarding-state"; import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js"; +import type { SkillJudgeConfig } from "./skill-config.js"; function normalizeText(value: string): string { return value.trim().toLowerCase(); @@ -14,18 +13,35 @@ export function judgeRuleCase( expected: EvalCaseExpected, responseText: string, actual: { + derived?: unknown; intent: string; responseMode?: string | undefined; + validator?: SkillJudgeConfig["validator"] | undefined; }, ): RuleJudgeResult { const checks: RuleJudgeCheck[] = []; + const validation = actual.validator + ? actual.validator(responseText, actual.derived) + : null; - checks.push({ - name: "validator", - passed: true, - skipped: true, - detail: "not configured for this skill", - }); + if (validation) { + checks.push({ + name: "validator", + passed: validation.status !== "reject", + skipped: false, + detail: + validation.reasons.length > 0 + ? validation.reasons.join(", ") + : validation.status, + }); + } else { + checks.push({ + name: "validator", + passed: true, + skipped: true, + detail: "not configured for this skill", + }); + } const expectedIntent = expected.intent?.trim(); if (expectedIntent) { @@ -112,117 +128,7 @@ export function judgeRuleCase( passed: activeChecks.every((check) => check.passed), score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length), checks, - validatorStatus: "pass", - validatorReasons: [], - }; -} - -export function judgeOnboardingRuleCase( - expected: EvalCaseExpected, - responseText: string, - derived: DerivedState, -): RuleJudgeResult { - const validation = validateOnboardingResponse(responseText, derived, { - allowShortResponse: derived.allowShortResponse, - }); - const checks: RuleJudgeCheck[] = []; - - checks.push({ - name: "validator", - passed: validation.status !== "reject", - skipped: false, - detail: - validation.reasons.length > 0 - ? validation.reasons.join(", ") - : validation.status, - }); - - const expectedIntent = expected.intent?.trim(); - if (expectedIntent) { - checks.push({ - name: "intent", - passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind), - skipped: false, - detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`, - }); - } else { - checks.push({ - name: "intent", - passed: true, - skipped: true, - }); - } - - const expectedResponseMode = expected.responseMode?.trim(); - if (expectedResponseMode) { - checks.push({ - name: "responseMode", - passed: - normalizeText(expectedResponseMode) === - normalizeText(derived.responseMode), - skipped: false, - detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`, - }); - } else { - checks.push({ - name: "responseMode", - passed: true, - skipped: true, - }); - } - - const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? []; - if (mustIncludeAny.length > 0) { - const normalizedResponse = normalizeText(responseText); - const matched = mustIncludeAny.find((candidate) => - normalizedResponse.includes(normalizeText(candidate)), - ); - - checks.push({ - name: "mustIncludeAny", - passed: Boolean(matched), - skipped: false, - detail: matched - ? `matched=${matched}` - : `expected one of: ${mustIncludeAny.join(", ")}`, - }); - } else { - checks.push({ - name: "mustIncludeAny", - passed: true, - skipped: true, - }); - } - - const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? []; - if (mustNotInclude.length > 0) { - const normalizedResponse = normalizeText(responseText); - const matched = mustNotInclude.find((candidate) => - normalizedResponse.includes(normalizeText(candidate)), - ); - - checks.push({ - name: "mustNotInclude", - passed: !matched, - skipped: false, - detail: matched ? `forbidden=${matched}` : undefined, - }); - } else { - checks.push({ - name: "mustNotInclude", - passed: true, - skipped: true, - }); - } - - const activeChecks = checks.filter((check) => !check.skipped); - const passedChecks = activeChecks.filter((check) => check.passed).length; - - return { - passed: activeChecks.every((check) => check.passed), - score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length), - checks, - validatorStatus: validation.status, - validatorReasons: validation.reasons, + validatorStatus: validation?.status ?? "pass", + validatorReasons: validation?.reasons ?? [], }; } diff --git a/src/eval/judges/skill-config.ts b/src/eval/judges/skill-config.ts new file mode 100644 index 0000000..de31ea7 --- /dev/null +++ b/src/eval/judges/skill-config.ts @@ -0,0 +1,46 @@ +import type { ToolRegistry } from "@tools/registry"; +import { createModerationToolRegistry } from "@skills/moderation/tools"; +import { createOnboardingToolRegistry } from "@skills/onboarding/tools"; +import { + validateOnboardingResponse, + type ValidationResult, +} from "@skills/onboarding/runtime/validator"; +import type { DerivedState } from "@skills/onboarding/state/onboarding-state"; + +export type SkillValidationResult = ValidationResult; + +export interface SkillJudgeConfig { + axes: readonly string[]; + promptTemplate?: string | undefined; + toolRegistryFactory: () => ToolRegistry; + tournamentInstruction: string; + validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined; + weights?: Record | undefined; +} + +export const SKILL_JUDGE_CONFIGS: Record = { + moderation: { + axes: ["appropriateness", "accuracy", "helpfulness"], + toolRegistryFactory: createModerationToolRegistry, + tournamentInstruction: + "커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.", + }, + onboarding: { + axes: ["friendliness", "accuracy", "concision", "scope"], + toolRegistryFactory: createOnboardingToolRegistry, + tournamentInstruction: + "온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.", + validator: (text, derived) => + validateOnboardingResponse(text, derived as DerivedState, { + allowShortResponse: (derived as DerivedState).allowShortResponse, + }), + }, +}; + +export function getSkillJudgeConfig(skillName: string): SkillJudgeConfig { + const config = SKILL_JUDGE_CONFIGS[skillName] ?? SKILL_JUDGE_CONFIGS["onboarding"]; + if (!config) { + throw new Error("Missing onboarding judge config"); + } + return config; +} diff --git a/src/eval/loop/propose.ts b/src/eval/loop/propose.ts index bc4fed0..0f9107c 100644 --- a/src/eval/loop/propose.ts +++ b/src/eval/loop/propose.ts @@ -5,15 +5,11 @@ import { dirname, resolve } from "node:path"; import { z } from "zod"; import type { LlmClient } from "@llm/llm-client"; import { SkillRegistry } from "@skills/skill-loader"; -import { createBuiltInToolRegistry } from "@tools/builtins/index"; -import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state"; -import type { Blackboard, PromptContext } from "@skills/onboarding/types"; import { - PromptBuilder, type PromptLayerDiff, type PromptLayerName, type PromptVariantManifest, -} from "@skills/onboarding/prompts/builder"; +} from "@eval/prompt-variant"; import { getEvalTournamentMinCases, getLlmProposeMaxTokens, @@ -26,6 +22,8 @@ import { queryArchivedReviewLabels } from "../review/label-archive.js"; import type { ReviewLabel } from "../review/queue.js"; import { runG1Static, runG4Permission } from "../governance/gates.js"; import { promoteTrustTier } from "../governance/trust.js"; +import { getSkillJudgeConfig } from "../judges/skill-config.js"; +import { getSkillProposeAdapter } from "./skill-adapters.js"; export interface ProposalInput { slice: SliceKey; @@ -129,44 +127,12 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null { return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall; } -function emptyBlackboard(): Blackboard { - return { - flaggedIssues: [], - openQuestions: [], - recentNicknameAttempts: [], - toneMemory: {}, - }; -} - -function derivedFromSlice(slice: SliceKey): DerivedState { - return { - allowShortResponse: false, - intent: { kind: slice.intent } as Intent, - phase: { kind: slice.phase } as OnboardingPhase, - responseMode: "llm", - shouldRespond: true, - suggestedMaxTokens: 384, - tier: slice.tier, - }; -} - -function defaultPromptContext(): PromptContext { - return { - blackboard: emptyBlackboard(), - conversationHistory: [], - userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.", - }; -} - export function buildCurrentPromptSnapshot( slice: SliceKey, archiveEntry?: EvalArchiveEntry, ): string { - return PromptBuilder.forDerived( - archiveEntry?.derived ?? derivedFromSlice(slice), - null, - defaultPromptContext(), - ).buildSystemPrompt(); + return getSkillProposeAdapter(slice.skill ?? archiveEntry?.skill ?? "onboarding") + .buildSnapshot(slice, archiveEntry); } function summarizeArchiveEntry(entry: EvalArchiveEntry): Record { @@ -233,8 +199,9 @@ export function summarizeRootCauses( } function buildProposalPrompt(input: ProposalInput): string { + const skill = input.slice.skill ?? "onboarding"; return [ - "온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.", + `${skill} 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.`, "반드시 JSON object만 출력하세요.", "prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.", "과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.", @@ -271,7 +238,7 @@ export async function proposeVariants( { role: "system", content: - "당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.", + `당신은 ${input.slice.skill ?? "onboarding"} 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.`, }, { role: "user", @@ -296,10 +263,12 @@ export async function proposeVariants( return { ...proposal, diff, + skill: input.slice.skill ?? "onboarding", variantId: stableHash({ description: proposal.description, diff, rationale: proposal.rationale, + skill: input.slice.skill ?? "onboarding", }), }; }); @@ -325,11 +294,34 @@ export async function buildProposalInputFromArchive( }, ): Promise { const limit = options?.limit ?? 50; + const skill = slice.skill ?? "onboarding"; + const skillFilter = + skill === "onboarding" + ? { + $or: [ + { skill }, + { skill: { $exists: false } }, + ], + } + : { skill }; const entries = await getEvalArchiveCollection(mongoConnection) .find({ - tier: slice.tier, - "derived.intent.kind": slice.intent, - "derived.phase.kind": slice.phase, + $and: [ + skillFilter, + { tier: slice.tier }, + { + $or: [ + { + "derived.intent.kind": slice.intent, + "derived.phase.kind": slice.phase, + }, + { + "derived.intent": slice.intent, + "derived.phase": slice.phase, + }, + ], + }, + ], }) .sort({ evaluatedAt: -1 }) .limit(limit) @@ -373,13 +365,17 @@ export async function writePromptVariantProposal( await mkdir(dirname(outputPath), { recursive: true }); const manifest: PromptVariantProposal = { ...proposal, + skill: proposal.skill ?? "onboarding", trustTier: proposal.trustTier ?? "T1", }; const g1 = runG1Static(manifest); + const skill = manifest.skill ?? "onboarding"; + const judgeConfig = getSkillJudgeConfig(skill); const g4 = runG4Permission( manifest, - createBuiltInToolRegistry(), + judgeConfig.toolRegistryFactory(), SkillRegistry.load(resolve(process.cwd(), "src", "skills")), + skill, ); const gatedManifest: PromptVariantProposal = { ...manifest, diff --git a/src/eval/loop/skill-adapters.ts b/src/eval/loop/skill-adapters.ts new file mode 100644 index 0000000..2df4401 --- /dev/null +++ b/src/eval/loop/skill-adapters.ts @@ -0,0 +1,16 @@ +import { moderationProposeAdapter } from "@skills/moderation/eval/propose-adapter"; +import { onboardingProposeAdapter } from "@skills/onboarding/eval/propose-adapter"; +import type { SkillProposeAdapter } from "./skill-propose-adapter.js"; + +const SKILL_PROPOSE_ADAPTERS: Record = { + [moderationProposeAdapter.skill]: moderationProposeAdapter, + [onboardingProposeAdapter.skill]: onboardingProposeAdapter, +}; + +export function getSkillProposeAdapter(skill: string): SkillProposeAdapter { + const adapter = SKILL_PROPOSE_ADAPTERS[skill] ?? SKILL_PROPOSE_ADAPTERS["onboarding"]; + if (!adapter) { + throw new Error("Missing onboarding propose adapter"); + } + return adapter; +} diff --git a/src/eval/loop/skill-propose-adapter.ts b/src/eval/loop/skill-propose-adapter.ts new file mode 100644 index 0000000..b5beaab --- /dev/null +++ b/src/eval/loop/skill-propose-adapter.ts @@ -0,0 +1,38 @@ +import type { AgentRequest } from "@agent/types"; +import type { LlmClient } from "@llm/llm-client"; +import type { ChatMessage } from "@llm/types"; +import type { PromptVariantManifest } from "@eval/prompt-variant"; +import type { EvalArchiveEntry } from "../archive.js"; +import type { SliceKey } from "../aggregator.js"; + +export interface SkillProposeAdapter { + readonly skill: string; + buildSnapshot(slice: SliceKey, archiveEntry?: EvalArchiveEntry): string; + derivedFromSlice(slice: SliceKey): unknown; + generateResponse( + llmClient: LlmClient, + entry: EvalArchiveEntry, + variantId: string, + manifests?: Record, + ): Promise; +} + +export function getVariantManifest( + variantId: string, + manifests?: Record, +): PromptVariantManifest | string | null { + if (variantId === "control") { + return null; + } + return manifests?.[variantId] ?? variantId; +} + +export function basicPromptSnapshot(messages: ChatMessage[]): string { + return messages + .map((message) => `[${message.role}]\n${message.content}`) + .join("\n\n"); +} + +export function getRequestText(request: AgentRequest): string { + return request.userMessage?.trim() || ""; +} diff --git a/src/eval/loop/tournament.ts b/src/eval/loop/tournament.ts index acfb87d..1a176b7 100644 --- a/src/eval/loop/tournament.ts +++ b/src/eval/loop/tournament.ts @@ -2,11 +2,12 @@ import { createHash } from "node:crypto"; import { z } from "zod"; import type { LlmClient } from "@llm/llm-client"; -import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder"; -import type { Blackboard, PromptContext } from "@skills/onboarding/types"; +import type { PromptVariantManifest } from "@eval/prompt-variant"; import { getEvalTournamentMaxMatches } from "../../config.js"; import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js"; import { getEvalArchiveCollection } from "../archive.js"; +import { getSkillJudgeConfig } from "../judges/skill-config.js"; +import { getSkillProposeAdapter } from "./skill-adapters.js"; import { createEloRating, scoreForVerdict, @@ -30,6 +31,7 @@ export interface TournamentMatch { export interface TournamentResult { generatedAt: string; matches: TournamentMatch[]; + skill?: string | undefined; variants: EloRating[]; } @@ -58,6 +60,10 @@ const sideBySideJudgeSchema = z.object({ verdict: z.enum(["A", "B", "tie"]), }); +function entrySkill(entry: EvalArchiveEntry): string { + return entry.skill || entry.request.skill || "onboarding"; +} + function stableHash(value: unknown): string { return createHash("sha256") .update(JSON.stringify(value)) @@ -65,56 +71,18 @@ function stableHash(value: unknown): string { .slice(0, 12); } -function emptyBlackboard(): Blackboard { - return { - flaggedIssues: [], - openQuestions: [], - recentNicknameAttempts: [], - toneMemory: {}, - }; -} - -function promptContextForCase(entry: EvalArchiveEntry): PromptContext { - return { - blackboard: emptyBlackboard(), - conversationHistory: [], - userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.", - }; -} - -function getVariantManifest( - variantId: string, - manifests?: Record, -): PromptVariantManifest | string | null { - if (variantId === "control") { - return null; - } - return manifests?.[variantId] ?? variantId; -} - async function generateTournamentResponse( llmClient: LlmClient, entry: EvalArchiveEntry, variantId: string, manifests?: Record, ): Promise { - let builder = PromptBuilder.forDerived( - entry.derived, - null, - promptContextForCase(entry), + return getSkillProposeAdapter(entrySkill(entry)).generateResponse( + llmClient, + entry, + variantId, + manifests, ); - const manifest = getVariantManifest(variantId, manifests); - if (manifest) { - builder = builder.applyVariant(manifest); - } - - const response = await llmClient.chatCompletion(builder.build(), { - maxTokens: entry.derived.suggestedMaxTokens, - temperature: 0, - tier: entry.derived.tier, - }); - - return response?.content.trim() ?? ""; } function buildJudgePrompt( @@ -122,9 +90,12 @@ function buildJudgePrompt( responseA: string, responseB: string, ): string { + const skill = entrySkill(entry); + const config = getSkillJudgeConfig(skill); return [ - "아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.", - "평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.", + `아래 두 ${skill} 응답 중 더 나은 쪽을 고르세요.`, + config.tournamentInstruction, + `평가 축: ${config.axes.join(", ")}`, "A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.", "", "[request]", @@ -157,7 +128,7 @@ async function judgeTournamentMatch( { role: "system", content: - "당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.", + `당신은 ${entrySkill(entry)} 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.`, }, { role: "user", @@ -201,6 +172,7 @@ async function ensureTournamentIndexes( ): Promise { await Promise.all([ collection.createIndex({ generatedAt: -1 }), + collection.createIndex({ skill: 1, generatedAt: -1 }), collection.createIndex({ "variants.id": 1, generatedAt: -1 }), ]); } @@ -225,6 +197,7 @@ export async function runTournament( cases: EvalArchiveEntry[]; maxMatches?: number | undefined; nowMs?: number | undefined; + skill?: string | undefined; variantIds: string[]; variantManifests?: Record | undefined; }, @@ -303,9 +276,11 @@ export async function runTournament( } } + const skill = options.skill ?? options.cases[0]?.skill; return { generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(), matches, + ...(skill ? { skill } : {}), variants: [...ratings.values()].sort((left, right) => right.elo - left.elo), }; } @@ -315,22 +290,35 @@ export async function loadTournamentCases( options: { caseIds?: string[] | undefined; limit?: number | undefined; + skill?: string | undefined; }, ): Promise { - const filter = - options.caseIds && options.caseIds.length > 0 - ? { - streamId: { - $in: options.caseIds, - }, - } - : {}; + const filter: Record = {}; + if (options.caseIds && options.caseIds.length > 0) { + filter.streamId = { + $in: options.caseIds, + }; + } + if (options.skill) { + if (options.skill === "onboarding") { + filter.$or = [ + { skill: options.skill }, + { skill: { $exists: false } }, + ]; + } else { + filter.skill = options.skill; + } + } - return getEvalArchiveCollection(mongoConnection) + const entries = await getEvalArchiveCollection(mongoConnection) .find(filter) .sort({ evaluatedAt: -1 }) .limit(options.limit ?? getEvalTournamentMaxMatches()) .toArray(); + return entries.map((entry) => ({ + ...entry, + skill: entry.skill || entry.request.skill || "onboarding", + })); } export async function saveTournamentResult( @@ -346,14 +334,24 @@ export async function queryTournamentResults( mongoConnection: MongoConnectionLike, options?: { limit?: number | undefined; + skill?: string | undefined; variantId?: string | undefined; }, ): Promise { - const filter = options?.variantId - ? { - "variants.id": options.variantId, - } - : {}; + const filter: Record = {}; + if (options?.variantId) { + filter["variants.id"] = options.variantId; + } + if (options?.skill) { + if (options.skill === "onboarding") { + filter.$or = [ + { skill: options.skill }, + { skill: { $exists: false } }, + ]; + } else { + filter.skill = options.skill; + } + } return getTournamentCollection(mongoConnection) .find(filter) diff --git a/src/eval/promote.ts b/src/eval/promote.ts index 7047a21..fa55d61 100644 --- a/src/eval/promote.ts +++ b/src/eval/promote.ts @@ -16,7 +16,11 @@ import { } from "./anonymize.js"; import type { EvalCaseDefinition } from "./types.js"; import type { EvalArchiveEntry, MongoConnectionLike } from "./archive.js"; -import { getEvalArchiveCollection } from "./archive.js"; +import { + getArchiveDerivedIntent, + getArchiveDerivedPhase, + getEvalArchiveCollection, +} from "./archive.js"; import { queryArchivedReviewLabels } from "./review/label-archive.js"; import type { ReviewLabel } from "./review/queue.js"; @@ -59,7 +63,7 @@ function getLlmOverall(entry: EvalArchiveEntry): number | null { } function sliceKey(entry: EvalArchiveEntry): string { - return `${entry.tier}:${entry.derived.intent.kind}:${entry.derived.phase.kind}`; + return `${entry.skill || entry.request.skill || "onboarding"}:${entry.tier}:${getArchiveDerivedIntent(entry.derived)}:${getArchiveDerivedPhase(entry.derived)}`; } function sortByLowestScore( @@ -286,8 +290,8 @@ function candidateToEvalCase(candidate: PromotionCandidate): EvalCaseDefinition candidate.labelRationale === undefined ? null : `humanRationale=${candidate.labelRationale}`, - `derivedIntent=${candidate.derived.intent.kind}`, - `derivedPhase=${candidate.derived.phase.kind}`, + `derivedIntent=${getArchiveDerivedIntent(candidate.derived)}`, + `derivedPhase=${getArchiveDerivedPhase(candidate.derived)}`, `assistantResponse=${candidate.response.text}`, candidate.betterResponse === undefined ? null diff --git a/src/eval/prompt-variant.ts b/src/eval/prompt-variant.ts new file mode 100644 index 0000000..94549ed --- /dev/null +++ b/src/eval/prompt-variant.ts @@ -0,0 +1,70 @@ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +export type PromptLayerName = + | "context" + | "examples" + | "persona" + | "policy" + | "state" + | "tool-policy"; + +export type PromptLayerOperation = "append" | "remove" | "replace"; + +export interface PromptLayerDiff { + layer: PromptLayerName; + op: PromptLayerOperation; + reason?: string | undefined; + text?: string | undefined; +} + +export interface PromptVariantManifest { + description?: string | undefined; + diff: PromptLayerDiff[]; + expectedImprovement?: string | undefined; + rationale?: string | undefined; + requiredResources?: string[] | undefined; + requiredTools?: string[] | undefined; + skill?: string | undefined; + trustTier?: "T1" | "T2" | "T3" | "T4" | undefined; + variantId: string; +} + +const variantCache = new Map(); + +export function loadPromptVariantManifest( + variantId: string, +): PromptVariantManifest | null { + const normalized = variantId.trim(); + if (!normalized) { + return null; + } + if (variantCache.has(normalized)) { + return variantCache.get(normalized) ?? null; + } + + const path = resolve( + process.cwd(), + "src", + "eval", + "proposals", + `${normalized}.json`, + ); + if (!existsSync(path)) { + variantCache.set(normalized, null); + return null; + } + + try { + const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest; + if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) { + variantCache.set(normalized, null); + return null; + } + variantCache.set(normalized, parsed); + return parsed; + } catch { + variantCache.set(normalized, null); + return null; + } +} diff --git a/src/eval/review/calibration.ts b/src/eval/review/calibration.ts index 6888994..a9fd63f 100644 --- a/src/eval/review/calibration.ts +++ b/src/eval/review/calibration.ts @@ -81,11 +81,12 @@ function toCalibrationPairs( pairs.push({ evalStreamId: label.evalStreamId, humanScore: label.humanScore, - llmAccuracy: archive.llmJudge.scores.accuracy, - llmConcision: archive.llmJudge.scores.concision, - llmFriendliness: archive.llmJudge.scores.friendliness, + llmAccuracy: archive.llmJudge.scores.accuracy ?? archive.llmJudge.scores.overall, + llmConcision: archive.llmJudge.scores.concision ?? archive.llmJudge.scores.overall, + llmFriendliness: + archive.llmJudge.scores.friendliness ?? archive.llmJudge.scores.overall, llmOverall: archive.llmJudge.scores.overall, - llmScope: archive.llmJudge.scores.scope, + llmScope: archive.llmJudge.scores.scope ?? archive.llmJudge.scores.overall, }); } diff --git a/src/eval/router-training.ts b/src/eval/router-training.ts index 7a08044..c97beaa 100644 --- a/src/eval/router-training.ts +++ b/src/eval/router-training.ts @@ -12,6 +12,26 @@ import { } from "@skills/onboarding/runtime/router"; import type { EvalArchiveEntry } from "./archive.js"; +function isOnboardingArchiveEntry( + entry: EvalArchiveEntry, +): entry is EvalArchiveEntry & { + classified: ClassifiedIntent; + derived: DerivedState; +} { + return ( + (entry.skill === undefined || + entry.skill === "onboarding" || + entry.request.skill === "onboarding") && + typeof entry.derived === "object" && + entry.derived !== null && + typeof (entry.derived as { intent?: unknown }).intent === "object" && + typeof (entry.derived as { phase?: unknown }).phase === "object" && + typeof entry.classified === "object" && + entry.classified !== null && + typeof (entry.classified as { intent?: unknown }).intent === "object" + ); +} + function getLlmOverall(entry: EvalArchiveEntry): number | null { return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall; } @@ -21,6 +41,9 @@ function requestFromEntry(entry: EvalArchiveEntry): AgentRequest { } function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null { + if (!isOnboardingArchiveEntry(entry)) { + return null; + } const staticDecision = routerDecisionKind( decidePreGenerationRouting({ classified: entry.classified, @@ -57,11 +80,11 @@ function deriveLabel(entry: EvalArchiveEntry): RoutingDecisionKind | null { } function toClassified(entry: EvalArchiveEntry): ClassifiedIntent { - return entry.classified; + return entry.classified as ClassifiedIntent; } function toDerived(entry: EvalArchiveEntry): DerivedState { - return entry.derived; + return entry.derived as DerivedState; } export function buildRouterTrainingSamples( diff --git a/src/eval/runner.ts b/src/eval/runner.ts index df8c400..c073515 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -30,12 +30,10 @@ import type { } from "@skills/onboarding/state/onboarding-state"; import { noopShadowEvalPublisher } from "./shadow-bus.js"; import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js"; -import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; +import { judgeSkillWithLlm } from "./judges/llm-judge.js"; import { judgeProcessRule } from "./judges/process-judge.js"; -import { - judgeOnboardingRuleCase, - judgeRuleCase, -} from "./judges/rule-judge.js"; +import { judgeRuleCase } from "./judges/rule-judge.js"; +import { getSkillJudgeConfig } from "./judges/skill-config.js"; import type { EvalBaselineFile, EvalCaseDefinition, @@ -161,11 +159,18 @@ function round(value: number): number { return Number(value.toFixed(3)); } -function average(values: number[]): number | null { - if (values.length === 0) { +function average(values: Array): number | null { + const finiteValues = values.filter( + (value): value is number => + typeof value === "number" && Number.isFinite(value), + ); + if (finiteValues.length === 0) { return null; } - return round(values.reduce((sum, value) => sum + value, 0) / values.length); + return round( + finiteValues.reduce((sum, value) => sum + value, 0) / + finiteValues.length, + ); } function averageOrZero(values: number[]): number { @@ -741,8 +746,16 @@ async function evaluateCase( skill, llmClient, enableLlmJudge, + effectiveSkill, ) - : evaluateGenericSkillCase(evalCase, input, skill), + : evaluateGenericSkillCase( + evalCase, + input, + skill, + llmClient, + enableLlmJudge, + effectiveSkill, + ), ); } @@ -752,9 +765,11 @@ async function evaluateOnboardingCase( skill: Skill, llmClient: LlmClient, enableLlmJudge: boolean, + skillName: string, ): Promise { const classifier = new OnboardingIntentClassifier(llmClient); const processTrace = new ProcessTraceCollector(); + const judgeConfig = getSkillJudgeConfig(skillName); return runWithProcessTrace(processTrace, async () => { const classified = await classifier.classify( @@ -775,14 +790,16 @@ async function evaluateOnboardingCase( ); const latencyMs = Date.now() - startedAt; const derivedSnapshot = toDerivedSnapshot(derived); - const ruleJudge = judgeOnboardingRuleCase( - evalCase.expected, - response.text, + const ruleJudge = judgeRuleCase(evalCase.expected, response.text, { derived, - ); + intent: derived.intent.kind, + responseMode: derived.responseMode, + validator: judgeConfig.validator, + }); const llmJudge = enableLlmJudge - ? await judgeOnboardingWithLlm( + ? await judgeSkillWithLlm( llmClient, + skillName, evalCase, response.text, derivedSnapshot, @@ -795,6 +812,7 @@ async function evaluateOnboardingCase( const processJudge = judgeProcessRule( evalCase.expected, processTraceSnapshot, + judgeConfig.toolRegistryFactory(), ); return { @@ -834,7 +852,12 @@ async function evaluateOnboardingCase( error: errorMessage, }, }, - ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived), + ruleJudge: judgeRuleCase(evalCase.expected, "", { + derived, + intent: derived.intent.kind, + responseMode: derived.responseMode, + validator: judgeConfig.validator, + }), llmJudge: { skipped: true, reason: "skill-error", @@ -842,6 +865,7 @@ async function evaluateOnboardingCase( processJudge: judgeProcessRule( evalCase.expected, processTraceSnapshot, + judgeConfig.toolRegistryFactory(), ), processTrace: processTraceSnapshot, error: errorMessage, @@ -854,8 +878,12 @@ async function evaluateGenericSkillCase( evalCase: EvalCaseDefinition, input: EvalCaseInput, skill: Skill, + llmClient: LlmClient, + enableLlmJudge: boolean, + skillName: string, ): Promise { const classified = classifyModerationIntent(input.request.userMessage ?? ""); + const judgeConfig = getSkillJudgeConfig(skillName); const derived: EvalDerivedSnapshot = { allowShortResponse: true, intent: classified.intent.kind, @@ -887,13 +915,23 @@ async function evaluateGenericSkillCase( request: input.request, response, ruleJudge: judgeRuleCase(evalCase.expected, response.text, { + derived, intent: classified.intent.kind, responseMode: derived.responseMode, + validator: judgeConfig.validator, }), - llmJudge: { - skipped: true, - reason: "unsupported-skill", - }, + llmJudge: enableLlmJudge + ? await judgeSkillWithLlm( + llmClient, + skillName, + evalCase, + response.text, + derived, + ) + : { + skipped: true, + reason: "disabled", + }, }; } catch (error) { const latencyMs = Date.now() - startedAt; @@ -920,8 +958,10 @@ async function evaluateGenericSkillCase( }, }, ruleJudge: judgeRuleCase(evalCase.expected, "", { + derived, intent: classified.intent.kind, responseMode: derived.responseMode, + validator: judgeConfig.validator, }), llmJudge: { skipped: true, diff --git a/src/eval/shadow-bus.test.ts b/src/eval/shadow-bus.test.ts index 60006f7..6096fd5 100644 --- a/src/eval/shadow-bus.test.ts +++ b/src/eval/shadow-bus.test.ts @@ -61,6 +61,7 @@ describe("ShadowEvalBus", () => { derived, request, response, + skill: "onboarding", }); expect(appendStream).toHaveBeenCalledTimes(1); @@ -68,7 +69,9 @@ describe("ShadowEvalBus", () => { expect(appendStream.mock.calls[0]?.[1]).toMatchObject({ type: "shadow-eval", requestId: "req-1", + skill: "onboarding", tier: "0", + version: "2", }); randomSpy.mockRestore(); @@ -86,6 +89,7 @@ describe("ShadowEvalBus", () => { ...response, text: "", }, + skill: "onboarding", }); expect(appendStream).not.toHaveBeenCalled(); diff --git a/src/eval/shadow-bus.ts b/src/eval/shadow-bus.ts index 0b8a464..2b1188d 100644 --- a/src/eval/shadow-bus.ts +++ b/src/eval/shadow-bus.ts @@ -1,11 +1,6 @@ import type { AgentRequest, AgentResponse } from "@agent/types"; import type { GarnetClient } from "@common/garnet"; import { getRequestLogger } from "@obs/context"; -import type { - ClassifiedIntent, - DerivedState, -} from "@skills/onboarding/state/onboarding-state"; -import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import { getBackgroundStreamMaxLen, getShadowEvalSamplingRate, @@ -14,13 +9,15 @@ import { const BACKGROUND_STREAM_KEY = "lolympus-agent:background"; export interface ShadowEvalEnvelope { - classified: ClassifiedIntent; + classified: unknown; createdAt: string; - derived: DerivedState; + derived: unknown; latencyMs?: number | undefined; - process?: ProcessTraceSnapshot | undefined; + process?: unknown; request: AgentRequest; response: AgentResponse; + skill: string; + version?: 2 | undefined; } export interface ShadowEvalPublisher { @@ -38,10 +35,46 @@ export function shouldSampleShadowEval( return randomValue < getShadowEvalSamplingRate(tier); } -function getTraceAbstention(process: ProcessTraceSnapshot | undefined): - | NonNullable["output"]["abstention"] - | undefined { - return process?.steps.classify?.output.abstention; +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null + ? (value as Record) + : {}; +} + +function parseTier(value: unknown): 0 | 1 | 2 | null { + const parsed = typeof value === "number" ? value : Number(value); + if (parsed === 0 || parsed === 1 || parsed === 2) { + return parsed; + } + return null; +} + +function getDerivedTier(derived: unknown): 0 | 1 | 2 | null { + return parseTier(asRecord(derived)["tier"]); +} + +function getTraceAbstention(process: unknown): Record | null { + const steps = asRecord(asRecord(process)["steps"]); + const classify = asRecord(steps["classify"]); + const output = asRecord(classify["output"]); + const abstention = asRecord(output["abstention"]); + return Object.keys(abstention).length > 0 ? abstention : null; +} + +function getAbstentionActionKind(abstention: Record | null): string | null { + const action = asRecord(abstention?.["action"]); + const kind = action["kind"]; + return typeof kind === "string" && kind.trim() ? kind : null; +} + +function getAbstentionReason(abstention: Record | null): string | null { + const reason = abstention?.["reason"]; + return typeof reason === "string" && reason.trim() ? reason : null; +} + +function getAbstentionScore(abstention: Record | null): number | null { + const score = abstention?.["score"]; + return typeof score === "number" && Number.isFinite(score) ? score : null; } export class ShadowEvalBus implements ShadowEvalPublisher { @@ -51,11 +84,13 @@ export class ShadowEvalBus implements ShadowEvalPublisher { publish(envelope: ShadowEvalEnvelope): void { const abstention = getTraceAbstention(envelope.process); - if (!envelope.response.text.trim() && abstention?.action.kind !== "no-response") { + const actionKind = getAbstentionActionKind(abstention); + if (!envelope.response.text.trim() && actionKind !== "no-response") { return; } - if (!shouldSampleShadowEval(envelope.derived.tier)) { + const tier = getDerivedTier(envelope.derived); + if (tier === null || !shouldSampleShadowEval(tier)) { return; } @@ -66,7 +101,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher { requestId: envelope.request.requestId, chatroomId: envelope.request.chatroomId, userId: envelope.request.userId, - tier: String(envelope.derived.tier), + skill: envelope.skill, + version: String(envelope.version ?? 2), + tier: String(tier), latencyMs: envelope.latencyMs !== undefined ? String(envelope.latencyMs) : "", input: JSON.stringify({ @@ -79,9 +116,9 @@ export class ShadowEvalBus implements ShadowEvalPublisher { }; if (abstention) { - fields.abstentionAction = abstention.action.kind; - fields.abstentionReason = abstention.reason ?? ""; - fields.abstentionScore = String(abstention.score); + fields.abstentionAction = actionKind ?? ""; + fields.abstentionReason = getAbstentionReason(abstention) ?? ""; + fields.abstentionScore = String(getAbstentionScore(abstention) ?? ""); } if (envelope.process) { diff --git a/src/eval/shadow-worker.ts b/src/eval/shadow-worker.ts index 83d1c31..470dfd2 100644 --- a/src/eval/shadow-worker.ts +++ b/src/eval/shadow-worker.ts @@ -12,12 +12,6 @@ import { summarizeConversation } from "@conv/summarizer"; import type { LlmClient } from "@llm/llm-client"; import { getLogger } from "@obs/logger"; import type { AlertSender } from "@obs/alert"; -import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator"; -import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; -import type { - ClassifiedIntent, - DerivedState, -} from "@skills/onboarding/state/onboarding-state"; import type { ConversationStore } from "@conv/conversation-store"; import { getAlertThresholdDriftPct, @@ -49,13 +43,14 @@ import { type MongoConnectionLike, } from "./archive.js"; import { computeDriftReports } from "./drift.js"; -import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; +import { judgeSkillWithLlm } from "./judges/llm-judge.js"; import { writeProcessAttributions } from "./judges/attribution.js"; import { loadPrmModel } from "./judges/prm.js"; import { judgeProcessRule, judgeProcessWithLlm, } from "./judges/process-judge.js"; +import { getSkillJudgeConfig } from "./judges/skill-config.js"; import { appendCandidatesToJsonl, selectPromotionCandidates, @@ -72,6 +67,7 @@ import type { EvalBaselineFile, EvalCaseDefinition, EvalDerivedSnapshot, + EvalProcessTraceSnapshot, ProcessJudgeResult, } from "./types.js"; @@ -100,13 +96,15 @@ const STOP_TIMEOUT_MS = 10_000; type BackgroundJobType = "shadow-eval" | "summarize"; interface ShadowEvalJob { - classified: ClassifiedIntent; + classified: unknown; createdAt: string; - derived: DerivedState; + derived: unknown; latencyMs?: number | undefined; - process?: ProcessTraceSnapshot | undefined; + process?: EvalProcessTraceSnapshot | undefined; request: AgentRequest; response: AgentResponse; + skill: string; + version: number; } interface EvalObservation { @@ -168,26 +166,75 @@ function parseOptionalJsonField(value: string | undefined): T | undefined { return JSON.parse(value) as T; } -function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot { +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null + ? (value as Record) + : {}; +} + +function getStringField( + record: Record, + key: string, +): string | null { + const value = record[key]; + return typeof value === "string" && value.trim() ? value : null; +} + +function getNestedString( + record: Record, + first: string, + second: string, +): string | null { + return getStringField(asRecord(record[first]), second); +} + +function parseTier(value: unknown): 0 | 1 | 2 { + const parsed = typeof value === "number" ? value : Number(value); + return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0; +} + +function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] { + return value === "skip" || + value === "template" || + value === "llm" || + value === "llm-with-tools" + ? value + : "llm"; +} + +function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot { + const record = asRecord(derived); + const intent = getStringField(record, "intent") ?? + getNestedString(record, "intent", "kind") ?? + "unknown"; + const phase = getStringField(record, "phase") ?? + getNestedString(record, "phase", "kind") ?? + "unknown"; + const abstentionScore = record["abstentionScore"]; + const suggestedMaxTokens = record["suggestedMaxTokens"]; + return { - ...(derived.abstentionScore !== undefined - ? { abstentionScore: derived.abstentionScore } + ...(typeof abstentionScore === "number" + ? { abstentionScore } : {}), - allowShortResponse: derived.allowShortResponse, - intent: derived.intent.kind, - phase: derived.phase.kind, - responseMode: derived.responseMode, - ...(derived.routingDecision - ? { routingDecision: derived.routingDecision } + allowShortResponse: record["allowShortResponse"] === true, + intent, + phase, + responseMode: parseResponseMode(record["responseMode"]), + ...(record["routingDecision"] + ? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] } : {}), - shouldRespond: derived.shouldRespond, - suggestedMaxTokens: derived.suggestedMaxTokens, - tier: derived.tier, + shouldRespond: record["shouldRespond"] !== false, + suggestedMaxTokens: + typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens) + ? suggestedMaxTokens + : 192, + tier: parseTier(record["tier"]), }; } -function getTraceAbstention(process: ProcessTraceSnapshot | undefined): - | NonNullable["output"]["abstention"] +function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined): + | NonNullable["output"]["abstention"] | undefined { return process?.steps.classify?.output.abstention; } @@ -208,17 +255,17 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob { message.message["input"], "input", ); - const process = parseOptionalJsonField( + const process = parseOptionalJsonField( message.message["process"], ); const job: ShadowEvalJob = { - classified: parseJsonField( + classified: parseJsonField( message.message["classified"], "classified", ), createdAt: message.message["createdAt"] ?? new Date().toISOString(), - derived: parseJsonField( + derived: parseJsonField( message.message["derived"], "derived", ), @@ -228,6 +275,8 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob { message.message["output"], "output", ), + skill: message.message["skill"]?.trim() || input.request.skill || "onboarding", + version: Number(message.message["version"] ?? 1), }; if (process) { @@ -500,6 +549,8 @@ export class ShadowWorker { private async processShadowEval(message: StreamEntry): Promise { const job = parseShadowEvalJob(message); + const judgeConfig = getSkillJudgeConfig(job.skill); + const derivedSnapshot = toDerivedSnapshot(job.derived); const abstention = getTraceAbstention(job.process); const responseTextForValidation = job.response.text.trim() @@ -507,18 +558,19 @@ export class ShadowWorker { : abstention?.action.kind === "no-response" ? "[NO_RESPONSE]" : job.response.text; - const validation = validateOnboardingResponse( - responseTextForValidation, - job.derived, - { - allowShortResponse: job.derived.allowShortResponse, - }, - ); - const llmJudge = await judgeOnboardingWithLlm( + const validation = judgeConfig.validator + ? judgeConfig.validator(responseTextForValidation, job.derived) + : { + reasons: [], + status: "pass" as const, + text: responseTextForValidation, + }; + const llmJudge = await judgeSkillWithLlm( this.llmClient, + job.skill, toEvalCase(job, message.id), job.response.text, - toDerivedSnapshot(job.derived), + derivedSnapshot, ); const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall; const llmJudgeErrored = @@ -541,10 +593,12 @@ export class ShadowWorker { requestId: job.request.requestId, chatroomId: job.request.chatroomId, userId: job.request.userId, - tier: String(job.derived.tier), - intent: job.derived.intent.kind, - phase: job.derived.phase.kind, - responseMode: job.derived.responseMode, + skill: job.skill, + version: String(Math.max(2, job.version)), + tier: String(derivedSnapshot.tier), + intent: derivedSnapshot.intent, + phase: derivedSnapshot.phase, + responseMode: derivedSnapshot.responseMode, validatorStatus: validation.status, latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "", llmOverall: llmOverall === null ? "" : String(llmOverall), @@ -589,7 +643,12 @@ export class ShadowWorker { throw new Error("Missing process trace"); } - const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process); + const judgeConfig = getSkillJudgeConfig(job.skill); + const ruleResult = judgeProcessRule( + toEvalCase(job, messageId).expected, + job.process, + judgeConfig.toolRegistryFactory(), + ); if ( !getEvalProcessLlmJudgeEnabled() || Math.random() >= getEvalProcessLlmJudgeSamplingRate() diff --git a/src/eval/types.ts b/src/eval/types.ts index 66db997..a1b2456 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -11,6 +11,8 @@ import type { } from "@skills/onboarding/runtime/trace"; import type { PrmFeatureVector } from "./judges/prm.js"; +export type EvalProcessTraceSnapshot = ProcessTraceSnapshot; + export interface EvalRoomContextMessage { senderId: string; nickname: string; @@ -70,10 +72,7 @@ export interface RuleJudgeResult { } export interface LlmJudgeScores { - friendliness: number; - accuracy: number; - concision: number; - scope: number; + [axis: string]: number; overall: number; } diff --git a/src/skills/moderation/eval/propose-adapter.ts b/src/skills/moderation/eval/propose-adapter.ts new file mode 100644 index 0000000..140e7e4 --- /dev/null +++ b/src/skills/moderation/eval/propose-adapter.ts @@ -0,0 +1,211 @@ +import type { ConversationMessage } from "@agent/types"; +import type { LlmClient } from "@llm/llm-client"; +import type { ChatMessage } from "@llm/types"; +import { + loadPromptVariantManifest, + type PromptLayerDiff, + type PromptVariantManifest, +} from "@eval/prompt-variant"; +import type { EvalArchiveEntry } from "@eval/archive"; +import type { SliceKey } from "@eval/aggregator"; +import { + basicPromptSnapshot, + getRequestText, + getVariantManifest, + type SkillProposeAdapter, +} from "../../../eval/loop/skill-propose-adapter.js"; +import { + classifyModerationIntent, + type ModerationIntent, + type ModerationIntentKind, +} from "../intents.js"; +import { MODERATION_PROMPT_DATA } from "../prompt-data.js"; + +interface ModerationDerivedSnapshot { + allowShortResponse: boolean; + intent: string; + phase: string; + responseMode: "skip" | "template" | "llm" | "llm-with-tools"; + shouldRespond: boolean; + suggestedMaxTokens: number; + tier: 0 | 1 | 2; +} + +function formatHistory(history: ConversationMessage[]): string { + if (history.length === 0) { + return "(none)"; + } + + return history + .slice(-5) + .map((message) => `${message.role}: ${message.content}`) + .join("\n"); +} + +function formatExamples(intent: ModerationIntent): string { + return MODERATION_PROMPT_DATA.examples[intent.kind] + .slice(0, 2) + .map((example) => `user: ${example.user}\nassistant: ${example.assistant}`) + .join("\n\n"); +} + +function applyDiff( + layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>, + diff: PromptLayerDiff, +): void { + const target = diff.layer === "tool-policy" ? "policy" : diff.layer; + if (target !== "context" && + target !== "examples" && + target !== "persona" && + target !== "policy" && + target !== "state") { + return; + } + + if (diff.op === "remove") { + layers[target] = ""; + return; + } + + const text = diff.text?.trim(); + if (!text) { + return; + } + if (diff.op === "replace") { + layers[target] = text; + return; + } + layers[target] = layers[target] ? `${layers[target]}\n${text}` : text; +} + +function getManifest( + variantId: string, + manifests?: Record, +): PromptVariantManifest | null { + const manifest = getVariantManifest(variantId, manifests); + if (!manifest) { + return null; + } + return typeof manifest === "string" + ? loadPromptVariantManifest(manifest) + : manifest; +} + +function buildMessages( + entry: Pick, + variant?: PromptVariantManifest | null, +): ChatMessage[] { + const classified = classifyModerationIntent(getRequestText(entry.request)); + const intent = classified.intent; + const layers = { + persona: MODERATION_PROMPT_DATA.persona.join("\n"), + policy: [ + ...MODERATION_PROMPT_DATA.policy.base, + ...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind], + ].join("\n"), + state: ["[intent]", intent.kind].join("\n"), + examples: formatExamples(intent), + context: "", + }; + + for (const diff of variant?.diff ?? []) { + applyDiff(layers, diff); + } + + return [ + { + role: "system", + content: [ + layers.persona, + "", + "[policy]", + layers.policy, + "", + layers.state, + "", + "[examples]", + layers.examples, + ...(layers.context ? ["", "[context]", layers.context] : []), + ].join("\n"), + }, + { + role: "user", + content: [ + "[history]", + formatHistory([]), + "", + "[message]", + getRequestText(entry.request), + ].join("\n"), + }, + ]; +} + +function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot { + return { + allowShortResponse: true, + intent: slice.intent, + phase: slice.phase, + responseMode: slice.intent === "off-topic" ? "template" : "llm", + shouldRespond: slice.intent !== "off-topic", + suggestedMaxTokens: 192, + tier: slice.tier, + }; +} + +function getSuggestedMaxTokens(entry: EvalArchiveEntry): number { + const derived = entry.derived as Partial; + return typeof derived.suggestedMaxTokens === "number" + ? derived.suggestedMaxTokens + : 192; +} + +function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 { + const derived = entry.derived as Partial; + return derived.tier === 0 || derived.tier === 1 || derived.tier === 2 + ? derived.tier + : entry.tier; +} + +export const moderationProposeAdapter: SkillProposeAdapter = { + skill: "moderation", + + buildSnapshot(slice, archiveEntry) { + const request = archiveEntry?.request ?? { + requestId: "proposal-snapshot", + skill: "moderation", + chatroomId: "proposal-room", + userId: "proposal-user", + nickname: "문의자", + trigger: "user-message" as const, + userMessage: `slice=${slice.intent}/${slice.phase}`, + }; + return basicPromptSnapshot(buildMessages({ request })); + }, + + derivedFromSlice, + + async generateResponse( + llmClient: LlmClient, + entry: EvalArchiveEntry, + variantId: string, + manifests?: Record, + ): Promise { + const response = await llmClient.chatCompletion( + buildMessages(entry, getManifest(variantId, manifests)), + { + maxTokens: getSuggestedMaxTokens(entry), + temperature: 0, + tier: getTier(entry), + }, + ); + + return response?.content.trim() ?? ""; + }, +}; + +export function normalizeModerationIntentKind(value: string): ModerationIntentKind { + return value in MODERATION_PROMPT_DATA.examples + ? (value as ModerationIntentKind) + : "off-topic"; +} diff --git a/src/skills/moderation/manifest-binding.ts b/src/skills/moderation/manifest-binding.ts new file mode 100644 index 0000000..0958241 --- /dev/null +++ b/src/skills/moderation/manifest-binding.ts @@ -0,0 +1,12 @@ +import { MODERATION_INTENT_TABLE } from "./intents.js"; +import { createModerationToolRegistry } from "./tools.js"; + +export function getIntentKinds(): string[] { + return Object.keys(MODERATION_INTENT_TABLE); +} + +export function getToolNames(): string[] { + return createModerationToolRegistry() + .list() + .map((tool) => tool.name); +} diff --git a/src/skills/onboarding/eval/propose-adapter.ts b/src/skills/onboarding/eval/propose-adapter.ts new file mode 100644 index 0000000..112d5c5 --- /dev/null +++ b/src/skills/onboarding/eval/propose-adapter.ts @@ -0,0 +1,99 @@ +import type { LlmClient } from "@llm/llm-client"; +import type { PromptVariantManifest } from "@eval/prompt-variant"; +import type { EvalArchiveEntry } from "@eval/archive"; +import type { SliceKey } from "@eval/aggregator"; +import { + getVariantManifest, + type SkillProposeAdapter, +} from "../../../eval/loop/skill-propose-adapter.js"; +import { + PromptBuilder, +} from "../prompts/builder.js"; +import type { DerivedState, Intent, OnboardingPhase } from "../state/onboarding-state.js"; +import type { Blackboard, PromptContext } from "../types.js"; + +function emptyBlackboard(): Blackboard { + return { + flaggedIssues: [], + openQuestions: [], + recentNicknameAttempts: [], + toneMemory: {}, + }; +} + +function derivedFromSlice(slice: SliceKey): DerivedState { + return { + allowShortResponse: false, + intent: { kind: slice.intent } as Intent, + phase: { kind: slice.phase } as OnboardingPhase, + responseMode: "llm", + shouldRespond: true, + suggestedMaxTokens: 384, + tier: slice.tier, + }; +} + +function defaultPromptContext(userMessage?: string): PromptContext { + return { + blackboard: emptyBlackboard(), + conversationHistory: [], + userMessage: + userMessage?.trim() || + "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.", + }; +} + +function getDerived(entry: EvalArchiveEntry, slice?: SliceKey): DerivedState { + const derived = entry.derived as Partial; + if (derived.intent && derived.phase) { + return derived as DerivedState; + } + return derivedFromSlice( + slice ?? { + skill: "onboarding", + tier: entry.tier, + intent: "unknown", + phase: "unknown", + }, + ); +} + +export const onboardingProposeAdapter: SkillProposeAdapter = { + skill: "onboarding", + + buildSnapshot(slice, archiveEntry) { + return PromptBuilder.forDerived( + archiveEntry ? getDerived(archiveEntry, slice) : derivedFromSlice(slice), + null, + defaultPromptContext(archiveEntry?.request.userMessage), + ).buildSystemPrompt(); + }, + + derivedFromSlice, + + async generateResponse( + llmClient: LlmClient, + entry: EvalArchiveEntry, + variantId: string, + manifests?: Record, + ): Promise { + let builder = PromptBuilder.forDerived( + getDerived(entry), + null, + defaultPromptContext(entry.request.userMessage), + ); + const manifest = getVariantManifest(variantId, manifests); + if (manifest) { + builder = builder.applyVariant(manifest); + } + + const derived = getDerived(entry); + const response = await llmClient.chatCompletion(builder.build(), { + maxTokens: derived.suggestedMaxTokens, + temperature: 0, + tier: derived.tier, + }); + + return response?.content.trim() ?? ""; + }, +}; diff --git a/src/skills/onboarding/manifest-binding.ts b/src/skills/onboarding/manifest-binding.ts new file mode 100644 index 0000000..0dc9453 --- /dev/null +++ b/src/skills/onboarding/manifest-binding.ts @@ -0,0 +1,12 @@ +import { ONBOARDING_INTENT_TABLE } from "./intents.js"; +import { createOnboardingToolRegistry } from "./tools.js"; + +export function getIntentKinds(): string[] { + return Object.keys(ONBOARDING_INTENT_TABLE); +} + +export function getToolNames(): string[] { + return createOnboardingToolRegistry() + .list() + .map((tool) => tool.name); +} diff --git a/src/skills/onboarding/prompts/builder.ts b/src/skills/onboarding/prompts/builder.ts index 2ab55c7..701c846 100644 --- a/src/skills/onboarding/prompts/builder.ts +++ b/src/skills/onboarding/prompts/builder.ts @@ -1,6 +1,11 @@ -import { existsSync, readFileSync } from "node:fs"; import { resolve } from "node:path"; +import { + loadPromptVariantManifest, + type PromptLayerDiff, + type PromptLayerName, + type PromptVariantManifest, +} from "@eval/prompt-variant"; import { SkillRegistry } from "@skills/skill-loader"; import type { ChatMessage } from "@llm/types"; import { @@ -19,35 +24,12 @@ import { import { buildToolPolicy } from "./tool-policy.js"; import { defaultExampleRetriever } from "./example-store.js"; -export type PromptLayerName = - | "context" - | "examples" - | "persona" - | "policy" - | "state" - | "tool-policy"; - -export type PromptLayerOperation = "append" | "remove" | "replace"; - -export interface PromptLayerDiff { - layer: PromptLayerName; - op: PromptLayerOperation; - reason?: string | undefined; - text?: string | undefined; -} - -export interface PromptVariantManifest { - description?: string | undefined; - diff: PromptLayerDiff[]; - expectedImprovement?: string | undefined; - rationale?: string | undefined; - requiredResources?: string[] | undefined; - requiredTools?: string[] | undefined; - trustTier?: "T1" | "T2" | "T3" | "T4" | undefined; - variantId: string; -} - -const variantCache = new Map(); +export { + loadPromptVariantManifest, + type PromptLayerDiff, + type PromptLayerName, + type PromptVariantManifest, +} from "@eval/prompt-variant"; function getSkillRootDir(): string { return resolve(process.cwd(), "src", "skills"); @@ -85,43 +67,6 @@ function applyLayerDiff( layers[target] = layers[target] ? `${layers[target]}\n${text}` : text; } -export function loadPromptVariantManifest( - variantId: string, -): PromptVariantManifest | null { - const normalized = variantId.trim(); - if (!normalized) { - return null; - } - if (variantCache.has(normalized)) { - return variantCache.get(normalized) ?? null; - } - - const path = resolve( - process.cwd(), - "src", - "eval", - "proposals", - `${normalized}.json`, - ); - if (!existsSync(path)) { - variantCache.set(normalized, null); - return null; - } - - try { - const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest; - if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) { - variantCache.set(normalized, null); - return null; - } - variantCache.set(normalized, parsed); - return parsed; - } catch { - variantCache.set(normalized, null); - return null; - } -} - function skillIndexLayer(registry: SkillRegistry): string { const lines = registry.list().map( (manifest) => diff --git a/src/skills/onboarding/skill.ts b/src/skills/onboarding/skill.ts index 63f9713..b4bfb03 100644 --- a/src/skills/onboarding/skill.ts +++ b/src/skills/onboarding/skill.ts @@ -189,6 +189,7 @@ export class OnboardingSkill implements Skill { process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, + skill: this.name, }); return response; } @@ -222,6 +223,7 @@ export class OnboardingSkill implements Skill { process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, + skill: this.name, }); return response; @@ -294,6 +296,7 @@ export class OnboardingSkill implements Skill { process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, + skill: this.name, }); } diff --git a/src/skills/onboarding/tools.ts b/src/skills/onboarding/tools.ts index dff84e4..7ca20e5 100644 --- a/src/skills/onboarding/tools.ts +++ b/src/skills/onboarding/tools.ts @@ -1,6 +1,3 @@ -import { readFileSync } from "node:fs"; -import { resolve } from "node:path"; - import { checkOnboardingProgressTool } from "@tools/onboarding/check-onboarding-progress"; import { getRoomContextTool } from "@tools/onboarding/get-room-context"; import { lookupCommunityRuleTool } from "@tools/onboarding/lookup-community-rule"; @@ -17,32 +14,6 @@ const ONBOARDING_TOOLS = [ verifyDiscordNicknameTool, ] as const satisfies readonly Tool[]; -function readSkillManifestToolNames(): string[] { - const skillPath = resolve(process.cwd(), "src", "skills", "onboarding", "SKILL.md"); - const raw = readFileSync(skillPath, "utf8"); - const match = raw.match(/^tools:\s*\[([^\]]*)\]/mu); - if (!match?.[1]) { - return []; - } - - return match[1] - .split(",") - .map((item) => item.trim()) - .filter(Boolean); -} - -function assertToolManifestMatchesRegistry(): void { - const manifestNames = readSkillManifestToolNames().sort(); - const registryNames = ONBOARDING_TOOLS.map((tool) => tool.name).sort(); - if (manifestNames.join("\n") !== registryNames.join("\n")) { - throw new Error( - `Onboarding SKILL.md tools do not match registry: manifest=${manifestNames.join(",")} registry=${registryNames.join(",")}`, - ); - } -} - -assertToolManifestMatchesRegistry(); - export function createOnboardingToolRegistry(): ToolRegistry { const registry = new ToolRegistry(); for (const tool of ONBOARDING_TOOLS) { diff --git a/src/skills/validate-manifests.ts b/src/skills/validate-manifests.ts new file mode 100644 index 0000000..5a87f44 --- /dev/null +++ b/src/skills/validate-manifests.ts @@ -0,0 +1,106 @@ +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { SkillRegistry, type SkillManifest } from "./skill-loader.js"; +import * as moderationBinding from "./moderation/manifest-binding.js"; +import * as onboardingBinding from "./onboarding/manifest-binding.js"; + +interface ManifestBinding { + getIntentKinds(): string[]; + getToolNames(): string[]; +} + +const BINDINGS: Record = { + moderation: moderationBinding, + onboarding: onboardingBinding, +}; + +function sortUnique(values: readonly string[]): string[] { + return [...new Set(values)].sort((left, right) => left.localeCompare(right)); +} + +function format(values: readonly string[]): string { + return values.length === 0 ? "(none)" : values.join(", "); +} + +function diff(left: readonly string[], right: readonly string[]): string[] { + const rightSet = new Set(right); + return left.filter((item) => !rightSet.has(item)); +} + +function validateManifest( + manifest: SkillManifest, + binding: ManifestBinding | undefined, +): string[] { + if (!binding) { + return [`${manifest.name}: missing manifest binding`]; + } + + const errors: string[] = []; + const manifestIntents = sortUnique(manifest.intents); + const codeIntents = sortUnique(binding.getIntentKinds()); + const unknownIntents = diff(manifestIntents, codeIntents); + if (unknownIntents.length > 0) { + errors.push( + `${manifest.name}: SKILL.md intents not found in code: ${format(unknownIntents)}`, + ); + } + + const manifestTools = sortUnique(manifest.tools); + const codeTools = sortUnique(binding.getToolNames()); + const missingTools = diff(manifestTools, codeTools); + const undocumentedTools = diff(codeTools, manifestTools); + if (missingTools.length > 0 || undocumentedTools.length > 0) { + errors.push( + `${manifest.name}: SKILL.md tools mismatch; missingInRegistry=${format(missingTools)} missingInManifest=${format(undocumentedTools)}`, + ); + } + + for (const [key, path] of Object.entries(manifest.resources)) { + if (!existsSync(path)) { + errors.push(`${manifest.name}: resource not found ${key} -> ${path}`); + } + } + + return errors; +} + +function validatePriorityConflicts(manifests: SkillManifest[]): string[] { + const byPriority = new Map(); + for (const manifest of manifests) { + const current = byPriority.get(manifest.priority) ?? []; + current.push(manifest.name); + byPriority.set(manifest.priority, current); + } + + return [...byPriority.entries()].flatMap(([priority, names]) => + names.length > 1 + ? [`priority=${priority} is shared by skills: ${format(names.sort())}`] + : [], + ); +} + +export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] { + const registry = SkillRegistry.load(rootDir); + const manifests = registry.list(); + return [ + ...manifests.flatMap((manifest) => + validateManifest(manifest, BINDINGS[manifest.name]), + ), + ...validatePriorityConflicts(manifests), + ]; +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === resolve(fileURLToPath(import.meta.url))) { + const errors = validateSkillManifests(); + if (errors.length > 0) { + for (const error of errors) { + console.error(`[skills] ${error}`); + } + process.exitCode = 1; + } else { + console.info("[skills] manifests ok"); + } +}