Add skill governance and runtime routing
This commit is contained in:
1 parent
6f09408423
commit
41dcfcdef7
71 files changed
+7057
-102
No files matched your search
@@ -4,6 +4,8 @@ import { dirname, resolve } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { SkillRegistry } from "@skills/skill-loader";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
||||
import {
|
||||
@@ -22,14 +24,24 @@ import { getEvalArchiveCollection } from "../archive.js";
|
||||
import type { SliceKey } from "../aggregator.js";
|
||||
import { queryArchivedReviewLabels } from "../review/label-archive.js";
|
||||
import type { ReviewLabel } from "../review/queue.js";
|
||||
import { runG1Static, runG4Permission } from "../governance/gates.js";
|
||||
import { promoteTrustTier } from "../governance/trust.js";
|
||||
|
||||
export interface ProposalInput {
|
||||
slice: SliceKey;
|
||||
recentLabels: ReviewLabel[];
|
||||
rootCauses?: RootCauseReport[] | undefined;
|
||||
worstCases: EvalArchiveEntry[];
|
||||
currentPromptSnapshot: string;
|
||||
}
|
||||
|
||||
export interface RootCauseReport {
|
||||
avgScoreLoss: number;
|
||||
count: number;
|
||||
examples: string[];
|
||||
rootCauseStep: string;
|
||||
}
|
||||
|
||||
export interface PromptVariantProposal extends PromptVariantManifest {
|
||||
description: string;
|
||||
expectedImprovement: string;
|
||||
@@ -160,6 +172,56 @@ function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown>
|
||||
};
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
export function summarizeRootCauses(
|
||||
entries: EvalArchiveEntry[],
|
||||
): RootCauseReport[] {
|
||||
const groups = new Map<
|
||||
string,
|
||||
{ examples: Array<{ loss: number; streamId: string }>; lossSum: number }
|
||||
>();
|
||||
|
||||
for (const entry of entries) {
|
||||
const rootCauseStep = entry.processJudge?.rootCauseStep;
|
||||
const rootCauseScoreLoss = entry.processJudge?.rootCauseScoreLoss ?? 0;
|
||||
if (!rootCauseStep || rootCauseScoreLoss <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const group = groups.get(rootCauseStep) ?? {
|
||||
examples: [],
|
||||
lossSum: 0,
|
||||
};
|
||||
group.examples.push({
|
||||
loss: rootCauseScoreLoss,
|
||||
streamId: entry.streamId,
|
||||
});
|
||||
group.lossSum += rootCauseScoreLoss;
|
||||
groups.set(rootCauseStep, group);
|
||||
}
|
||||
|
||||
return [...groups.entries()]
|
||||
.map(([rootCauseStep, group]) => ({
|
||||
avgScoreLoss: round(group.lossSum / group.examples.length),
|
||||
count: group.examples.length,
|
||||
examples: group.examples
|
||||
.sort((left, right) => right.loss - left.loss)
|
||||
.slice(0, 3)
|
||||
.map((example) => example.streamId),
|
||||
rootCauseStep,
|
||||
}))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.count - left.count ||
|
||||
right.avgScoreLoss - left.avgScoreLoss ||
|
||||
left.rootCauseStep.localeCompare(right.rootCauseStep),
|
||||
)
|
||||
.slice(0, 5);
|
||||
}
|
||||
|
||||
function buildProposalPrompt(input: ProposalInput): string {
|
||||
return [
|
||||
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
|
||||
@@ -173,6 +235,9 @@ function buildProposalPrompt(input: ProposalInput): string {
|
||||
"[recent_human_labels]",
|
||||
JSON.stringify(input.recentLabels.slice(0, 20)),
|
||||
"",
|
||||
"[root_causes]",
|
||||
JSON.stringify(input.rootCauses ?? []),
|
||||
"",
|
||||
"[worst_cases]",
|
||||
JSON.stringify(input.worstCases.slice(0, 5).map(summarizeArchiveEntry)),
|
||||
"",
|
||||
@@ -284,6 +349,7 @@ export async function buildProposalInputFromArchive(
|
||||
return {
|
||||
slice,
|
||||
recentLabels,
|
||||
rootCauses: summarizeRootCauses(entries),
|
||||
worstCases: worstCases.length > 0 ? worstCases : entries.slice(0, 5),
|
||||
currentPromptSnapshot: buildCurrentPromptSnapshot(slice, entries[0]),
|
||||
};
|
||||
@@ -295,6 +361,23 @@ export async function writePromptVariantProposal(
|
||||
): Promise<string> {
|
||||
const outputPath = resolve(process.cwd(), outputDir, `${proposal.variantId}.json`);
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(`${outputPath}`, `${JSON.stringify(proposal, null, 2)}\n`, "utf8");
|
||||
const manifest: PromptVariantProposal = {
|
||||
...proposal,
|
||||
trustTier: proposal.trustTier ?? "T1",
|
||||
};
|
||||
const g1 = runG1Static(manifest);
|
||||
const g4 = runG4Permission(
|
||||
manifest,
|
||||
createBuiltInToolRegistry(),
|
||||
SkillRegistry.load(resolve(process.cwd(), "src", "skills")),
|
||||
);
|
||||
const gatedManifest: PromptVariantProposal = {
|
||||
...manifest,
|
||||
trustTier:
|
||||
g1.passed && g4.passed
|
||||
? promoteTrustTier(manifest.trustTier, "T2")
|
||||
: manifest.trustTier,
|
||||
};
|
||||
await writeFile(`${outputPath}`, `${JSON.stringify(gatedManifest, null, 2)}\n`, "utf8");
|
||||
return outputPath;
|
||||
}
|
||||
Reference in new issue
Block a user