Add bounded skill handoff graph
This commit is contained in:
1 parent
54bc295ead
commit
c14adac162
42 files changed
+2243
-31
No files matched your search
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { parseControlTokens } from "./control-token-parser.js";
|
||||
|
||||
describe("parseControlTokens", () => {
|
||||
it("extracts handoff control tokens and strips them from text", () => {
|
||||
const parsed = parseControlTokens(
|
||||
"잠시 확인할게요. [HANDOFF:moderation:policy-risk]",
|
||||
);
|
||||
|
||||
expect(parsed.stripped).toBe("잠시 확인할게요.");
|
||||
expect(parsed.tokens).toEqual([
|
||||
{
|
||||
kind: "handoff",
|
||||
raw: "[HANDOFF:moderation:policy-risk]",
|
||||
reason: "policy-risk",
|
||||
targetAgent: "moderation",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts no-response tokens", () => {
|
||||
const parsed = parseControlTokens("[NO_RESPONSE]");
|
||||
|
||||
expect(parsed.stripped).toBe("");
|
||||
expect(parsed.tokens).toEqual([
|
||||
{
|
||||
kind: "no-response",
|
||||
raw: "[NO_RESPONSE]",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps invalid handoff reasons visible", () => {
|
||||
const parsed = parseControlTokens("[HANDOFF:moderation:not-a-reason]");
|
||||
|
||||
expect(parsed.stripped).toBe("[HANDOFF:moderation:not-a-reason]");
|
||||
expect(parsed.tokens).toEqual([]);
|
||||
});
|
||||
|
||||
it("accepts all handoff reason codes", () => {
|
||||
const reasons = [
|
||||
"out-of-scope",
|
||||
"missing-tool",
|
||||
"low-confidence",
|
||||
"validation-failed",
|
||||
"same-agent-revisit",
|
||||
"max-hop",
|
||||
"high-risk-action",
|
||||
"policy-risk",
|
||||
];
|
||||
|
||||
for (const reason of reasons) {
|
||||
const parsed = parseControlTokens(`[HANDOFF:supervisor:${reason}]`);
|
||||
expect(parsed.tokens[0]).toMatchObject({
|
||||
kind: "handoff",
|
||||
reason,
|
||||
targetAgent: "supervisor",
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
|
||||
const HANDOFF_REASONS = new Set<HandoffReason>([
|
||||
"out-of-scope",
|
||||
"missing-tool",
|
||||
"low-confidence",
|
||||
"validation-failed",
|
||||
"same-agent-revisit",
|
||||
"max-hop",
|
||||
"high-risk-action",
|
||||
"policy-risk",
|
||||
]);
|
||||
|
||||
export type ControlToken =
|
||||
| { kind: "no-response"; raw: string }
|
||||
| {
|
||||
kind: "handoff";
|
||||
raw: string;
|
||||
reason: HandoffReason;
|
||||
targetAgent: string;
|
||||
};
|
||||
|
||||
export function parseControlTokens(text: string): {
|
||||
stripped: string;
|
||||
tokens: ControlToken[];
|
||||
} {
|
||||
const tokens: ControlToken[] = [];
|
||||
let stripped = text;
|
||||
|
||||
stripped = stripped.replace(/\[NO_RESPONSE\]/gu, (raw) => {
|
||||
tokens.push({ kind: "no-response", raw });
|
||||
return "";
|
||||
});
|
||||
|
||||
stripped = stripped.replace(
|
||||
/\[HANDOFF:([a-z0-9_-]+):([a-z-]+)\]/giu,
|
||||
(raw, target: string, reason: string) => {
|
||||
if (!HANDOFF_REASONS.has(reason as HandoffReason)) {
|
||||
return raw;
|
||||
}
|
||||
tokens.push({
|
||||
kind: "handoff",
|
||||
raw,
|
||||
reason: reason as HandoffReason,
|
||||
targetAgent: target,
|
||||
});
|
||||
return "";
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
stripped: stripped.trim(),
|
||||
tokens,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import { decide } from "./trigger.js";
|
||||
|
||||
describe("escalation trigger", () => {
|
||||
it("turns explicit handoff reasons into structured requests", () => {
|
||||
const reasons: HandoffReason[] = [
|
||||
"out-of-scope",
|
||||
"missing-tool",
|
||||
"low-confidence",
|
||||
"validation-failed",
|
||||
"same-agent-revisit",
|
||||
"max-hop",
|
||||
"high-risk-action",
|
||||
"policy-risk",
|
||||
];
|
||||
|
||||
for (const reason of reasons) {
|
||||
const decision = decide({
|
||||
classified: { confidence: 0.8 },
|
||||
reason,
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "review",
|
||||
});
|
||||
|
||||
expect(decision).toMatchObject({
|
||||
reason,
|
||||
shouldHandoff: true,
|
||||
handoff: {
|
||||
reason,
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "review",
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("escalates exhausted budgets and validation rejects", () => {
|
||||
expect(
|
||||
decide({
|
||||
budget: {
|
||||
estimatedCostUsd: 0,
|
||||
handoffs: 4,
|
||||
maxCostUsd: 0.5,
|
||||
maxHandoffs: 4,
|
||||
maxSameAgentVisits: 2,
|
||||
maxToolCalls: 20,
|
||||
toolCalls: 0,
|
||||
visits: {},
|
||||
},
|
||||
}).handoff?.reason,
|
||||
).toBe("max-hop");
|
||||
|
||||
expect(decide({ validation: { status: "reject" } }).handoff?.reason).toBe(
|
||||
"validation-failed",
|
||||
);
|
||||
});
|
||||
|
||||
it("escalates low-confidence and uncertain high-risk responses", () => {
|
||||
expect(
|
||||
decide({ classified: { confidence: 0.2 } }).handoff?.reason,
|
||||
).toBe("low-confidence");
|
||||
|
||||
expect(
|
||||
decide({
|
||||
capabilityCard: {
|
||||
canHandle: ["policy"],
|
||||
cannotHandle: [],
|
||||
domain: "moderation",
|
||||
handoffTargets: ["supervisor"],
|
||||
outputContract: { format: "text", maxChars: "500" },
|
||||
riskTier: "high",
|
||||
},
|
||||
classified: { confidence: 0.6 },
|
||||
}).handoff?.reason,
|
||||
).toBe("high-risk-action");
|
||||
});
|
||||
|
||||
it("does not escalate ordinary confident responses", () => {
|
||||
expect(
|
||||
decide({
|
||||
capabilityCard: {
|
||||
canHandle: ["help"],
|
||||
cannotHandle: [],
|
||||
domain: "onboarding",
|
||||
handoffTargets: ["supervisor"],
|
||||
outputContract: { format: "text", maxChars: "500" },
|
||||
riskTier: "medium",
|
||||
},
|
||||
classified: { confidence: 0.9 },
|
||||
}),
|
||||
).toEqual({ shouldHandoff: false });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import type { HandoffReason, HandoffRequest } from "@agent/types";
|
||||
import type { BudgetSnapshot } from "@agent/budget";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
|
||||
export interface EscalationDecision {
|
||||
handoff?: HandoffRequest | undefined;
|
||||
reason?: HandoffReason | undefined;
|
||||
shouldHandoff: boolean;
|
||||
}
|
||||
|
||||
export function decide(input: {
|
||||
budget?: BudgetSnapshot | undefined;
|
||||
capabilityCard?: SkillCapabilityCard | undefined;
|
||||
classified?: { confidence?: number | undefined } | undefined;
|
||||
reason?: HandoffReason | undefined;
|
||||
targetAgent?: string | undefined;
|
||||
taskSpec?: string | undefined;
|
||||
validation?: { status?: string | undefined } | undefined;
|
||||
}): EscalationDecision {
|
||||
const confidence = input.classified?.confidence;
|
||||
let reason: HandoffReason | undefined = input.reason;
|
||||
|
||||
if (!reason) {
|
||||
if (input.budget && input.budget.handoffs >= input.budget.maxHandoffs) {
|
||||
reason = "max-hop";
|
||||
} else if (input.validation?.status === "reject") {
|
||||
reason = "validation-failed";
|
||||
} else if (typeof confidence === "number" && confidence < 0.35) {
|
||||
reason = "low-confidence";
|
||||
} else if (
|
||||
input.capabilityCard?.riskTier === "high" &&
|
||||
typeof confidence === "number" &&
|
||||
confidence < 0.7
|
||||
) {
|
||||
reason = "high-risk-action";
|
||||
}
|
||||
}
|
||||
|
||||
if (!reason) {
|
||||
return { shouldHandoff: false };
|
||||
}
|
||||
|
||||
return {
|
||||
handoff: {
|
||||
confidence: confidence ?? 0.5,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason,
|
||||
risk: input.capabilityCard?.riskTier ?? "medium",
|
||||
targetAgent: input.targetAgent ?? "supervisor",
|
||||
taskSpec: input.taskSpec ?? "Escalate this request to the appropriate skill.",
|
||||
},
|
||||
reason,
|
||||
shouldHandoff: true,
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,15 @@ name: moderation
|
||||
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
|
||||
version: 1
|
||||
priority: 50
|
||||
domain: moderation
|
||||
canHandle: [rules, violation-report, appeal, sanctions, policy-risk]
|
||||
cannotHandle: [onboarding-status, nickname-format, photo-verification, discord-onboarding]
|
||||
handoffTargets: [onboarding, supervisor]
|
||||
riskTier: high
|
||||
outputContract:
|
||||
format: text
|
||||
maxChars: "500"
|
||||
requiredFields: text
|
||||
intents: [greeting, rule-question, violation-report, appeal, off-topic]
|
||||
triggers: [user-message]
|
||||
tools: [lookup-rule]
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
||||
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
||||
@@ -22,6 +23,24 @@ type RuleTopic =
|
||||
| "sanctions"
|
||||
| "spam";
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: ["rules", "violation-report", "appeal", "sanctions", "policy-risk"],
|
||||
cannotHandle: [
|
||||
"onboarding-status",
|
||||
"nickname-format",
|
||||
"photo-verification",
|
||||
"discord-onboarding",
|
||||
],
|
||||
domain: "moderation",
|
||||
handoffTargets: ["onboarding", "supervisor"],
|
||||
outputContract: {
|
||||
format: "text",
|
||||
maxChars: "500",
|
||||
requiredFields: "text",
|
||||
},
|
||||
riskTier: "high",
|
||||
};
|
||||
|
||||
function formatHistory(history: ConversationMessage[]): string {
|
||||
if (history.length === 0) {
|
||||
return "(none)";
|
||||
@@ -156,6 +175,10 @@ export class ModerationSkill implements Skill {
|
||||
return request.skill === this.name;
|
||||
}
|
||||
|
||||
capabilityCard(): SkillCapabilityCard {
|
||||
return CAPABILITY_CARD;
|
||||
}
|
||||
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
|
||||
@@ -3,6 +3,15 @@ name: onboarding
|
||||
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
|
||||
version: 1
|
||||
priority: 70
|
||||
domain: onboarding
|
||||
canHandle: [new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed, onboarding-help]
|
||||
cannotHandle: [moderation-report, sanctions, appeal, policy-enforcement]
|
||||
handoffTargets: [moderation, supervisor]
|
||||
riskTier: medium
|
||||
outputContract:
|
||||
format: text
|
||||
maxChars: "500"
|
||||
requiredFields: text
|
||||
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
|
||||
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
|
||||
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
|
||||
|
||||
@@ -79,4 +79,14 @@ describe("validateOnboardingResponse", () => {
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
});
|
||||
|
||||
it("passes handoff control tokens", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"[HANDOFF:moderation:policy-risk]",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
expect(result.reasons).toContain("handoff");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { getMaxResponseLength } from "../../../config.js";
|
||||
import { parseControlTokens } from "../../control-token-parser.js";
|
||||
import { VALID_TIER_SET } from "../constants.js";
|
||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
@@ -131,15 +132,12 @@ export function validateOnboardingResponse(
|
||||
derived: DerivedState,
|
||||
options?: { allowShortResponse?: boolean | undefined },
|
||||
): ValidationResult {
|
||||
const text = postProcessResponse(draft);
|
||||
const parsedTokens = parseControlTokens(postProcessResponse(draft));
|
||||
const text = parsedTokens.stripped;
|
||||
const rejectReasons: string[] = [];
|
||||
const softWarnReasons: string[] = [];
|
||||
|
||||
if (!text) {
|
||||
rejectReasons.push("empty");
|
||||
}
|
||||
|
||||
if (text.includes("[NO_RESPONSE]")) {
|
||||
if (parsedTokens.tokens.some((token) => token.kind === "no-response")) {
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text: "[NO_RESPONSE]",
|
||||
@@ -147,6 +145,18 @@ export function validateOnboardingResponse(
|
||||
});
|
||||
}
|
||||
|
||||
if (parsedTokens.tokens.some((token) => token.kind === "handoff")) {
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text,
|
||||
reasons: ["handoff"],
|
||||
});
|
||||
}
|
||||
|
||||
if (!text) {
|
||||
rejectReasons.push("empty");
|
||||
}
|
||||
|
||||
if (!options?.allowShortResponse && text.length < 10) {
|
||||
rejectReasons.push("minLength");
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { getContext, getRequestLogger } from '@obs/context';
|
||||
import type { LlmClient } from '@llm/llm-client';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
import type { Skill } from '@skills/skill';
|
||||
import type { SkillCapabilityCard } from '@skills/skill-loader';
|
||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||
import { createOnboardingToolRegistry } from './tools.js';
|
||||
@@ -68,6 +69,32 @@ import type {
|
||||
StoredOnboardingSession,
|
||||
} from './types.js';
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: [
|
||||
'new-member',
|
||||
'nickname-invalid',
|
||||
'photo-failed',
|
||||
'progress-update',
|
||||
'timeout-warning',
|
||||
'completed',
|
||||
'onboarding-help',
|
||||
],
|
||||
cannotHandle: [
|
||||
'moderation-report',
|
||||
'sanctions',
|
||||
'appeal',
|
||||
'policy-enforcement',
|
||||
],
|
||||
domain: 'onboarding',
|
||||
handoffTargets: ['moderation', 'supervisor'],
|
||||
outputContract: {
|
||||
format: 'text',
|
||||
maxChars: '500',
|
||||
requiredFields: 'text',
|
||||
},
|
||||
riskTier: 'medium',
|
||||
};
|
||||
|
||||
export class OnboardingSkill implements Skill {
|
||||
readonly name = 'onboarding';
|
||||
readonly priority = 70;
|
||||
@@ -99,6 +126,10 @@ export class OnboardingSkill implements Skill {
|
||||
return request.skill === 'onboarding';
|
||||
}
|
||||
|
||||
capabilityCard(): SkillCapabilityCard {
|
||||
return CAPABILITY_CARD;
|
||||
}
|
||||
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
|
||||
@@ -21,6 +21,15 @@ function makeTempSkillRoot(): string {
|
||||
"priority: 65",
|
||||
"intents: [hello, help]",
|
||||
"triggers: [user-message]",
|
||||
"domain: demo-domain",
|
||||
"canHandle: [demo-task]",
|
||||
"cannotHandle: [other-task]",
|
||||
"handoffTargets: [supervisor]",
|
||||
"riskTier: medium",
|
||||
"outputContract:",
|
||||
" format: text",
|
||||
" maxChars: \"500\"",
|
||||
" requiredFields: text",
|
||||
"tools: [demo-tool]",
|
||||
"resources:",
|
||||
" guide: ./resources/guide.md",
|
||||
@@ -41,8 +50,18 @@ describe("SkillRegistry", () => {
|
||||
expect(manifest).toMatchObject({
|
||||
name: "demo",
|
||||
description: "Demo skill.",
|
||||
canHandle: ["demo-task"],
|
||||
cannotHandle: ["other-task"],
|
||||
domain: "demo-domain",
|
||||
handoffTargets: ["supervisor"],
|
||||
intents: ["hello", "help"],
|
||||
outputContract: {
|
||||
format: "text",
|
||||
maxChars: "500",
|
||||
requiredFields: "text",
|
||||
},
|
||||
priority: 65,
|
||||
riskTier: "medium",
|
||||
tools: ["demo-tool"],
|
||||
trustTier: "T2",
|
||||
version: 1,
|
||||
|
||||
@@ -3,14 +3,36 @@ import { dirname, resolve } from "node:path";
|
||||
import { getLogger } from "@obs/logger";
|
||||
|
||||
export type TrustTier = "T1" | "T2" | "T3" | "T4";
|
||||
export type SkillRiskTier = "low" | "medium" | "high";
|
||||
|
||||
export interface SkillOutputContract {
|
||||
format?: string | undefined;
|
||||
maxChars?: string | undefined;
|
||||
requiredFields?: string | undefined;
|
||||
}
|
||||
|
||||
export interface SkillCapabilityCard {
|
||||
canHandle: string[];
|
||||
cannotHandle: string[];
|
||||
domain: string;
|
||||
handoffTargets: string[];
|
||||
outputContract: SkillOutputContract;
|
||||
riskTier: SkillRiskTier;
|
||||
}
|
||||
|
||||
export interface SkillManifest {
|
||||
bodyPath: string;
|
||||
canHandle: string[];
|
||||
cannotHandle: string[];
|
||||
description: string;
|
||||
domain: string;
|
||||
handoffTargets: string[];
|
||||
intents: string[];
|
||||
name: string;
|
||||
outputContract: SkillOutputContract;
|
||||
priority: number;
|
||||
resources: Record<string, string>;
|
||||
riskTier: SkillRiskTier;
|
||||
tools: string[];
|
||||
triggers: string[];
|
||||
trustTier: TrustTier;
|
||||
@@ -121,10 +143,32 @@ function readResources(
|
||||
);
|
||||
}
|
||||
|
||||
function readStringMap(
|
||||
fields: Record<string, unknown>,
|
||||
key: string,
|
||||
): Record<string, string> {
|
||||
const raw = fields[key];
|
||||
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(raw as Record<string, unknown>).flatMap(([mapKey, value]) =>
|
||||
typeof value === "string" && value.trim()
|
||||
? [[mapKey, value.trim().replace(/^["']|["']$/gu, "")]]
|
||||
: [],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function isTrustTier(value: unknown): value is TrustTier {
|
||||
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
|
||||
}
|
||||
|
||||
function isRiskTier(value: unknown): value is SkillRiskTier {
|
||||
return value === "low" || value === "medium" || value === "high";
|
||||
}
|
||||
|
||||
function isPriority(value: unknown): value is number {
|
||||
return (
|
||||
typeof value === "number" &&
|
||||
@@ -147,6 +191,8 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
||||
const priority = parsed.fields["priority"] ?? 50;
|
||||
const version = parsed.fields["version"];
|
||||
const trustTier = parsed.fields["trustTier"];
|
||||
const domain = parsed.fields["domain"];
|
||||
const riskTier = parsed.fields["riskTier"] ?? "low";
|
||||
|
||||
if (
|
||||
typeof name !== "string" ||
|
||||
@@ -155,20 +201,28 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
||||
!description.trim() ||
|
||||
!isPriority(priority) ||
|
||||
typeof version !== "number" ||
|
||||
!isTrustTier(trustTier)
|
||||
!isTrustTier(trustTier) ||
|
||||
!isRiskTier(riskTier)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const outputContract = readStringMap(parsed.fields, "outputContract");
|
||||
|
||||
return {
|
||||
body: parsed.body,
|
||||
manifest: {
|
||||
bodyPath: path,
|
||||
canHandle: readStringArray(parsed.fields, "canHandle"),
|
||||
cannotHandle: readStringArray(parsed.fields, "cannotHandle"),
|
||||
description,
|
||||
domain: typeof domain === "string" && domain.trim() ? domain : "general",
|
||||
handoffTargets: readStringArray(parsed.fields, "handoffTargets"),
|
||||
intents: readStringArray(parsed.fields, "intents"),
|
||||
name,
|
||||
outputContract,
|
||||
priority,
|
||||
resources: readResources(parsed.fields, skillDir),
|
||||
riskTier,
|
||||
tools: readStringArray(parsed.fields, "tools"),
|
||||
triggers: readStringArray(parsed.fields, "triggers"),
|
||||
trustTier,
|
||||
|
||||
@@ -3,10 +3,12 @@ import type {
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import type { SkillCapabilityCard } from "./skill-loader.js";
|
||||
|
||||
export interface Skill {
|
||||
readonly name: string;
|
||||
readonly priority: number;
|
||||
capabilityCard?(): SkillCapabilityCard;
|
||||
canHandle(request: AgentRequest): boolean;
|
||||
handle(
|
||||
request: AgentRequest,
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
---
|
||||
name: supervisor
|
||||
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
|
||||
version: 1
|
||||
priority: 10
|
||||
domain: meta
|
||||
canHandle: [*]
|
||||
cannotHandle: []
|
||||
handoffTargets: [onboarding, moderation]
|
||||
riskTier: medium
|
||||
outputContract:
|
||||
format: text
|
||||
maxChars: "500"
|
||||
requiredFields: text
|
||||
intents: []
|
||||
triggers: [handoff, user-message]
|
||||
tools: []
|
||||
resources:
|
||||
trustTier: T4
|
||||
---
|
||||
|
||||
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||
@@ -0,0 +1,9 @@
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillDeps } from "@skills/skill-factory";
|
||||
import { SupervisorSkill } from "./skill.js";
|
||||
|
||||
export { SupervisorSkill } from "./skill.js";
|
||||
|
||||
export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export function getIntentKinds(): string[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
export function getToolNames(): string[] {
|
||||
return [];
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
|
||||
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
|
||||
|
||||
export async function appendSupervisorReview(
|
||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
|
||||
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
|
||||
const text = [
|
||||
packet?.userGoal,
|
||||
packet?.summary,
|
||||
packet?.handoffReason,
|
||||
].join(" ");
|
||||
|
||||
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) {
|
||||
return "moderation";
|
||||
}
|
||||
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) {
|
||||
return "onboarding";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import { SupervisorSkill } from "./skill.js";
|
||||
|
||||
class FakeGarnet {
|
||||
readonly streams: Array<{ key: string; fields: Record<string, unknown> }> = [];
|
||||
|
||||
appendStream(key: string, fields: Record<string, unknown>): Promise<string> {
|
||||
this.streams.push({ key, fields });
|
||||
return Promise.resolve("1-0");
|
||||
}
|
||||
|
||||
setStringIfAbsent(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
}
|
||||
|
||||
class FakeNats {
|
||||
readonly published: AlertPayload[] = [];
|
||||
|
||||
publish<T = unknown>(_subject: string, payload: T): Promise<void> {
|
||||
this.published.push(payload as AlertPayload);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
const baseRequest: AgentRequest = {
|
||||
chatroomId: "room-1",
|
||||
nickname: "tester",
|
||||
requestId: "request-1",
|
||||
skill: "supervisor",
|
||||
trigger: "handoff",
|
||||
userId: "user-1",
|
||||
};
|
||||
|
||||
const sourceResponse: AgentResponse = {
|
||||
confidence: 0.3,
|
||||
requestId: "request-1",
|
||||
source: "template",
|
||||
text: "needs review",
|
||||
};
|
||||
|
||||
describe("SupervisorSkill", () => {
|
||||
it("queues review and publishes alert for handoffs", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const skill = new SupervisorSkill(garnet, nats);
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
confidence: 0.3,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "max-hop",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "가입 상태 확인",
|
||||
},
|
||||
request: baseRequest,
|
||||
response: sourceResponse,
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
});
|
||||
|
||||
const response = await skill.handle(
|
||||
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||
[],
|
||||
);
|
||||
|
||||
expect(response.text).toContain("운영자");
|
||||
expect(response.metadata).toMatchObject({
|
||||
supervisorReason: "max-hop",
|
||||
suggestedTarget: "onboarding",
|
||||
});
|
||||
expect(garnet.streams[0]).toMatchObject({
|
||||
key: "lolympus-agent:review-queue",
|
||||
fields: {
|
||||
reason: "max-hop",
|
||||
requestId: "request-1",
|
||||
target: "onboarding",
|
||||
type: "supervisor-handoff",
|
||||
},
|
||||
});
|
||||
expect(nats.published[0]).toMatchObject({
|
||||
kind: "supervisor-handoff",
|
||||
metric: "agent.supervisor_handoff",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import { AlertService } from "@obs/alert";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import { appendSupervisorReview } from "./review-queue.js";
|
||||
import { routeSupervisorTarget } from "./routing-rules.js";
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: ["*"],
|
||||
cannotHandle: [],
|
||||
domain: "meta",
|
||||
handoffTargets: ["onboarding", "moderation"],
|
||||
outputContract: {
|
||||
format: "text",
|
||||
maxChars: "500",
|
||||
requiredFields: "text",
|
||||
},
|
||||
riskTier: "medium",
|
||||
};
|
||||
|
||||
export class SupervisorSkill implements Skill {
|
||||
readonly name = "supervisor";
|
||||
readonly priority = 10;
|
||||
private readonly alertService: AlertService;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
) {
|
||||
this.alertService = new AlertService(garnetClient, natsClient);
|
||||
}
|
||||
|
||||
canHandle(request: AgentRequest): boolean {
|
||||
return request.skill === this.name;
|
||||
}
|
||||
|
||||
capabilityCard(): SkillCapabilityCard {
|
||||
return CAPABILITY_CARD;
|
||||
}
|
||||
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
_conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const packet = readHandoffPacket(request);
|
||||
const target = routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
chatroomId: request.chatroomId,
|
||||
reason,
|
||||
requestId: request.requestId,
|
||||
target: target ?? "human-review",
|
||||
userId: request.userId,
|
||||
});
|
||||
metrics.humanOverrideTotal.inc({ kind: reason });
|
||||
|
||||
await this.alertService.sendAlert(
|
||||
{
|
||||
kind: "supervisor-handoff",
|
||||
metric: "agent.supervisor_handoff",
|
||||
severity: "warn",
|
||||
suggestedAction: "Review the supervisor handoff queue.",
|
||||
threshold: 0,
|
||||
timestamp: new Date().toISOString(),
|
||||
value: 1,
|
||||
window: "request",
|
||||
},
|
||||
{
|
||||
dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
supervisorReason: reason,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { SkillRegistry } from "./skill-loader.js";
|
||||
import { TransitionMatrix } from "./transition-matrix.js";
|
||||
|
||||
function writeSkill(
|
||||
root: string,
|
||||
name: string,
|
||||
handoffTargets: string[],
|
||||
): void {
|
||||
const skillDir = join(root, name);
|
||||
mkdirSync(skillDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(skillDir, "SKILL.md"),
|
||||
[
|
||||
"---",
|
||||
`name: ${name}`,
|
||||
`description: ${name} skill.`,
|
||||
"version: 1",
|
||||
"priority: 50",
|
||||
"domain: test",
|
||||
"canHandle: [test]",
|
||||
"cannotHandle: []",
|
||||
`handoffTargets: [${handoffTargets.join(", ")}]`,
|
||||
"riskTier: low",
|
||||
"outputContract:",
|
||||
" format: text",
|
||||
" maxChars: \"500\"",
|
||||
"trustTier: T2",
|
||||
"---",
|
||||
"",
|
||||
"body",
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
describe("TransitionMatrix", () => {
|
||||
it("allows declared handoffs and blocks self loops", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-"));
|
||||
writeSkill(root, "onboarding", ["moderation"]);
|
||||
writeSkill(root, "moderation", []);
|
||||
|
||||
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
|
||||
|
||||
expect(matrix.isAllowed("onboarding", "moderation")).toBe(true);
|
||||
expect(matrix.isAllowed("onboarding", "supervisor")).toBe(true);
|
||||
expect(matrix.isAllowed("onboarding", "onboarding")).toBe(false);
|
||||
expect(matrix.isAllowed("moderation", "onboarding")).toBe(false);
|
||||
});
|
||||
|
||||
it("reports unknown handoff targets", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-bad-"));
|
||||
writeSkill(root, "onboarding", ["missing"]);
|
||||
|
||||
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
|
||||
|
||||
expect(matrix.validateGraph(["onboarding"])).toContain(
|
||||
"onboarding: unknown handoff target missing",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { SkillRegistry } from "./skill-loader.js";
|
||||
|
||||
export class TransitionMatrix {
|
||||
private constructor(private readonly transitions: Map<string, Set<string>>) {}
|
||||
|
||||
static fromRegistry(registry: SkillRegistry): TransitionMatrix {
|
||||
const transitions = new Map<string, Set<string>>();
|
||||
for (const manifest of registry.list()) {
|
||||
const targets =
|
||||
manifest.name === "supervisor"
|
||||
? manifest.handoffTargets
|
||||
: [...manifest.handoffTargets, "supervisor"];
|
||||
transitions.set(
|
||||
manifest.name,
|
||||
new Set(targets),
|
||||
);
|
||||
}
|
||||
return new TransitionMatrix(transitions);
|
||||
}
|
||||
|
||||
isAllowed(from: string, to: string): boolean {
|
||||
if (from === to) {
|
||||
return false;
|
||||
}
|
||||
return this.transitions.get(from)?.has(to) ?? true;
|
||||
}
|
||||
|
||||
targetsOf(from: string): string[] {
|
||||
return [...(this.transitions.get(from) ?? new Set<string>())];
|
||||
}
|
||||
|
||||
validateGraph(knownSkills?: readonly string[]): string[] {
|
||||
const known = new Set([...(knownSkills ?? this.transitions.keys()), "supervisor"]);
|
||||
const errors: string[] = [];
|
||||
for (const [from, targets] of this.transitions.entries()) {
|
||||
if (targets.has(from)) {
|
||||
errors.push(`${from}: handoffTargets must not include itself`);
|
||||
}
|
||||
for (const target of targets) {
|
||||
if (!known.has(target)) {
|
||||
errors.push(`${from}: unknown handoff target ${target}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,10 @@ import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
|
||||
import { TransitionMatrix } from "./transition-matrix.js";
|
||||
import * as moderationBinding from "./moderation/manifest-binding.js";
|
||||
import * as onboardingBinding from "./onboarding/manifest-binding.js";
|
||||
import * as supervisorBinding from "./supervisor/manifest-binding.js";
|
||||
|
||||
interface ManifestBinding {
|
||||
getIntentKinds(): string[];
|
||||
@@ -14,6 +16,7 @@ interface ManifestBinding {
|
||||
const BINDINGS: Record<string, ManifestBinding> = {
|
||||
moderation: moderationBinding,
|
||||
onboarding: onboardingBinding,
|
||||
supervisor: supervisorBinding,
|
||||
};
|
||||
|
||||
function sortUnique(values: readonly string[]): string[] {
|
||||
@@ -32,12 +35,15 @@ function diff(left: readonly string[], right: readonly string[]): string[] {
|
||||
function validateManifest(
|
||||
manifest: SkillManifest,
|
||||
binding: ManifestBinding | undefined,
|
||||
knownSkills: readonly string[],
|
||||
): string[] {
|
||||
if (!binding) {
|
||||
return [`${manifest.name}: missing manifest binding`];
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
const capabilityErrors = validateCapabilityCard(manifest, knownSkills);
|
||||
errors.push(...capabilityErrors);
|
||||
const manifestIntents = sortUnique(manifest.intents);
|
||||
const codeIntents = sortUnique(binding.getIntentKinds());
|
||||
const unknownIntents = diff(manifestIntents, codeIntents);
|
||||
@@ -66,6 +72,45 @@ function validateManifest(
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateCapabilityCard(
|
||||
manifest: SkillManifest,
|
||||
knownSkills: readonly string[],
|
||||
): string[] {
|
||||
const errors: string[] = [];
|
||||
const canHandle = new Set(manifest.canHandle);
|
||||
const overlap = manifest.cannotHandle.filter((item) => canHandle.has(item));
|
||||
if (overlap.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: canHandle and cannotHandle overlap: ${format(overlap)}`,
|
||||
);
|
||||
}
|
||||
const riskTier: string = manifest.riskTier;
|
||||
if (riskTier !== "low" && riskTier !== "medium" && riskTier !== "high") {
|
||||
errors.push(`${manifest.name}: invalid riskTier ${riskTier}`);
|
||||
}
|
||||
const knownTargets = new Set([...knownSkills, "supervisor"]);
|
||||
const missingTargets = manifest.handoffTargets.filter(
|
||||
(target) => !knownTargets.has(target),
|
||||
);
|
||||
if (missingTargets.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: unknown handoffTargets: ${format(missingTargets)}`,
|
||||
);
|
||||
}
|
||||
if (manifest.handoffTargets.includes(manifest.name)) {
|
||||
errors.push(`${manifest.name}: handoffTargets must not include itself`);
|
||||
}
|
||||
if (!manifest.domain.trim()) {
|
||||
errors.push(`${manifest.name}: domain is required`);
|
||||
}
|
||||
if (!manifest.outputContract.format || !manifest.outputContract.maxChars) {
|
||||
errors.push(
|
||||
`${manifest.name}: outputContract.format and outputContract.maxChars are required`,
|
||||
);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||
const byPriority = new Map<number, string[]>();
|
||||
for (const manifest of manifests) {
|
||||
@@ -84,11 +129,13 @@ function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
|
||||
const registry = SkillRegistry.load(rootDir);
|
||||
const manifests = registry.list();
|
||||
const knownSkills = manifests.map((manifest) => manifest.name);
|
||||
return [
|
||||
...manifests.flatMap((manifest) =>
|
||||
validateManifest(manifest, BINDINGS[manifest.name]),
|
||||
validateManifest(manifest, BINDINGS[manifest.name], knownSkills),
|
||||
),
|
||||
...validatePriorityConflicts(manifests),
|
||||
...TransitionMatrix.fromRegistry(registry).validateGraph(knownSkills),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user