Unify dispatch and add supervisor routing signals
This commit is contained in:
1 parent
f0b00e5fd2
commit
b63985694a
19 files changed
+1345
-812
No files matched your search
@@ -4,6 +4,7 @@ import type { ShadowEvalBus } from "@eval/shadow-bus";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { RoomContextService } from "@room/room-context-service";
|
||||
import type { Skill } from "./skill.js";
|
||||
import type { SkillRegistry } from "./skill-loader.js";
|
||||
|
||||
export interface SkillDeps {
|
||||
garnetClient: GarnetClient;
|
||||
@@ -11,6 +12,7 @@ export interface SkillDeps {
|
||||
natsClient: NatsClient;
|
||||
roomContextService?: RoomContextService | undefined;
|
||||
shadowEvalBus: ShadowEvalBus;
|
||||
skillRegistry?: SkillRegistry | undefined;
|
||||
}
|
||||
|
||||
export type SkillFactory = (deps: SkillDeps) => Skill;
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: supervisor
|
||||
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
|
||||
description: LLM 기반 root cause 라우팅 supervisor.
|
||||
version: 1
|
||||
priority: 10
|
||||
domain: meta
|
||||
@@ -19,4 +19,4 @@ resources:
|
||||
trustTier: T4
|
||||
---
|
||||
|
||||
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||
LLM 기반 root cause 분석과 규칙 fallback을 함께 사용하는 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
|
||||
|
||||
function formatSkill(manifest: SkillManifest): string {
|
||||
return [
|
||||
`- ${manifest.name}: ${manifest.description}`,
|
||||
`domain=${manifest.domain}`,
|
||||
`canHandle=${manifest.canHandle.join(", ") || "*"}`,
|
||||
`cannotHandle=${manifest.cannotHandle.join(", ") || "-"}`,
|
||||
`risk=${manifest.riskTier}`,
|
||||
].join(" | ");
|
||||
}
|
||||
|
||||
export function buildSupervisorSystemPrompt(
|
||||
registry: SkillRegistry,
|
||||
_packet?: HandoffPacket,
|
||||
): string {
|
||||
const skills = registry
|
||||
.list()
|
||||
.filter((manifest) => manifest.name !== "supervisor");
|
||||
return [
|
||||
"너는 multi-agent 시스템의 supervisor다.",
|
||||
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
|
||||
"",
|
||||
"[사용 가능한 specialist]",
|
||||
...skills.map(formatSkill),
|
||||
"",
|
||||
"[응답 형식]",
|
||||
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단"}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildSupervisorUserPrompt(
|
||||
packet?: HandoffPacket,
|
||||
): string {
|
||||
if (!packet) {
|
||||
return "사용자 메시지가 명확하지 않음. 어디로 보낼지 결정해줘.";
|
||||
}
|
||||
|
||||
return [
|
||||
`[escalation reason]: ${packet.handoffReason}`,
|
||||
`[user goal]: ${packet.userGoal}`,
|
||||
`[summary]: ${packet.summary}`,
|
||||
`[source chain]: ${packet.sourceChain.join(" -> ")}`,
|
||||
].join("\n");
|
||||
}
|
||||
@@ -5,5 +5,10 @@ import { SupervisorSkill } from "./skill.js";
|
||||
export { SupervisorSkill } from "./skill.js";
|
||||
|
||||
export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
|
||||
return new SupervisorSkill(
|
||||
deps.garnetClient,
|
||||
deps.natsClient,
|
||||
deps.llmClient,
|
||||
deps.skillRegistry,
|
||||
);
|
||||
}
|
||||
@@ -4,15 +4,18 @@ import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorReview,
|
||||
getSupervisorReviewStats,
|
||||
getResolutionHistory,
|
||||
listSupervisorReview,
|
||||
parseSupervisorReviewEntry,
|
||||
resolveSupervisorReview,
|
||||
SUPERVISOR_RESOLVED_HASH_PREFIX,
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
} from "./review-queue.js";
|
||||
|
||||
class FakeGarnet {
|
||||
readonly hashes = new Map<string, Record<string, string>>();
|
||||
readonly resolutionStream: StreamEntry[] = [];
|
||||
readonly stream: StreamEntry[];
|
||||
|
||||
constructor(stream: StreamEntry[]) {
|
||||
@@ -25,8 +28,12 @@ class FakeGarnet {
|
||||
end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
|
||||
const entries = this.stream.filter((entry) => {
|
||||
expect([SUPERVISOR_REVIEW_STREAM_KEY, SUPERVISOR_RESOLUTIONS_STREAM_KEY]).toContain(key);
|
||||
const source =
|
||||
key === SUPERVISOR_RESOLUTIONS_STREAM_KEY
|
||||
? this.resolutionStream
|
||||
: this.stream;
|
||||
const entries = source.filter((entry) => {
|
||||
const afterStart = start === "-" || entry.id >= start;
|
||||
const beforeEnd = end === "+" || entry.id <= end;
|
||||
return afterStart && beforeEnd;
|
||||
@@ -43,6 +50,16 @@ class FakeGarnet {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
appendStream(key: string, fields: Record<string, string>): Promise<string> {
|
||||
expect(key).toBe(SUPERVISOR_RESOLUTIONS_STREAM_KEY);
|
||||
const id = `${this.resolutionStream.length + 1}-0`;
|
||||
this.resolutionStream.push({
|
||||
id,
|
||||
message: { ...fields },
|
||||
});
|
||||
return Promise.resolve(id);
|
||||
}
|
||||
|
||||
expireKey(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
@@ -140,6 +157,35 @@ describe("supervisor review queue", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("dual-writes resolution updates to an audit stream", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
await resolveSupervisorReview(garnet, "1-0", {
|
||||
action: "manual-reply",
|
||||
note: "handled",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
await resolveSupervisorReview(garnet, "1-0", {
|
||||
action: "dismiss",
|
||||
resolvedAt: "2026-05-18T03:00:00.000Z",
|
||||
resolvedBy: "operator-2@example.com",
|
||||
});
|
||||
|
||||
await expect(getResolutionHistory(garnet, "1-0")).resolves.toEqual([
|
||||
{
|
||||
action: "dismiss",
|
||||
resolvedAt: "2026-05-18T03:00:00.000Z",
|
||||
resolvedBy: "operator-2@example.com",
|
||||
},
|
||||
{
|
||||
action: "manual-reply",
|
||||
note: "handled",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("computes reason totals and top source chains", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorResolutionsStreamMaxLen,
|
||||
getSupervisorReviewQueueMaxLen,
|
||||
getSupervisorReviewResolutionTtlSec,
|
||||
} from "../../config.js";
|
||||
@@ -9,6 +10,8 @@ import {
|
||||
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
|
||||
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
|
||||
"lolympus-agent:review-resolved:";
|
||||
export const SUPERVISOR_RESOLUTIONS_STREAM_KEY =
|
||||
"lolympus-agent:review-resolutions";
|
||||
|
||||
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
|
||||
|
||||
@@ -67,7 +70,7 @@ type SupervisorReviewListGarnetClient = Pick<
|
||||
|
||||
type SupervisorReviewResolveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"expireKey" | "setHash"
|
||||
"appendStream" | "expireKey" | "setHash"
|
||||
>;
|
||||
|
||||
const BASE_FIELDS = new Set([
|
||||
@@ -285,6 +288,36 @@ export async function resolveSupervisorReview(
|
||||
const key = resolutionKey(id);
|
||||
await garnetClient.setHash(key, fields);
|
||||
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
|
||||
await garnetClient.appendStream(
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
{
|
||||
...fields,
|
||||
reviewId: id,
|
||||
type: "resolution",
|
||||
},
|
||||
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
|
||||
);
|
||||
}
|
||||
|
||||
export async function getResolutionHistory(
|
||||
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||
id: string,
|
||||
): Promise<SupervisorReviewResolution[]> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
"-",
|
||||
"+",
|
||||
);
|
||||
|
||||
return entries
|
||||
.flatMap((entry) => {
|
||||
if (entry.message["reviewId"] !== id) {
|
||||
return [];
|
||||
}
|
||||
const resolution = parseResolution(entry.message);
|
||||
return resolution ? [resolution] : [];
|
||||
})
|
||||
.sort((left, right) => right.resolvedAt.localeCompare(left.resolvedAt));
|
||||
}
|
||||
|
||||
export async function getSupervisorReviewStats(
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const supervisorRoutingSchema = z.object({
|
||||
confidence: z.number().min(0).max(1),
|
||||
reason: z.string().max(200),
|
||||
target: z.string().min(1),
|
||||
});
|
||||
|
||||
export type SupervisorRoutingDecision = z.infer<typeof supervisorRoutingSchema>;
|
||||
@@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest";
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { SkillRegistry } from "@skills/skill-loader";
|
||||
import { SupervisorSkill } from "./skill.js";
|
||||
|
||||
class FakeGarnet {
|
||||
@@ -27,6 +29,44 @@ class FakeNats {
|
||||
}
|
||||
}
|
||||
|
||||
class FakeLlm {
|
||||
chatCompletion(): Promise<{
|
||||
content: string;
|
||||
endpoint: string;
|
||||
message: { content: string; role: "assistant" };
|
||||
provider: "llama";
|
||||
raw: object;
|
||||
}> {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
confidence: 0.9,
|
||||
reason: "광고 규칙 문의라 moderation이 적합",
|
||||
target: "moderation",
|
||||
}),
|
||||
endpoint: "memory",
|
||||
message: { content: "", role: "assistant" },
|
||||
provider: "llama",
|
||||
raw: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const registry = {
|
||||
get: (name: string) => (name === "moderation" ? { name } : null),
|
||||
list: () => [
|
||||
{
|
||||
canHandle: ["rules"],
|
||||
cannotHandle: [],
|
||||
description: "커뮤니티 규칙 처리",
|
||||
domain: "community",
|
||||
handoffTargets: [],
|
||||
name: "moderation",
|
||||
outputContract: {},
|
||||
riskTier: "medium",
|
||||
},
|
||||
],
|
||||
} as unknown as SkillRegistry;
|
||||
|
||||
const baseRequest: AgentRequest = {
|
||||
chatroomId: "room-1",
|
||||
nickname: "tester",
|
||||
@@ -70,6 +110,7 @@ describe("SupervisorSkill", () => {
|
||||
|
||||
expect(response.text).toContain("운영자");
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "rules",
|
||||
supervisorReason: "max-hop",
|
||||
suggestedTarget: "onboarding",
|
||||
});
|
||||
@@ -96,4 +137,44 @@ describe("SupervisorSkill", () => {
|
||||
metric: "agent.supervisor_handoff",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses LLM routing when the decision is confident and valid", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const skill = new SupervisorSkill(
|
||||
garnet,
|
||||
nats,
|
||||
new FakeLlm() as unknown as Pick<LlmClient, "chatCompletion">,
|
||||
registry,
|
||||
);
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
confidence: 0.7,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "광고 규칙 문의",
|
||||
},
|
||||
request: baseRequest,
|
||||
response: sourceResponse,
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
});
|
||||
|
||||
const response = await skill.handle(
|
||||
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||
[],
|
||||
);
|
||||
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "llm",
|
||||
supervisorReason: "광고 규칙 문의라 moderation이 적합",
|
||||
suggestedTarget: "moderation",
|
||||
});
|
||||
expect(garnet.streams[0]?.fields).toMatchObject({
|
||||
reason: "out-of-scope",
|
||||
target: "moderation",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -4,14 +4,29 @@ import type {
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { AlertService } from "@obs/alert";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import {
|
||||
getSupervisorLlmConfidenceThreshold,
|
||||
getSupervisorLlmEnabled,
|
||||
} from "../../config.js";
|
||||
import {
|
||||
buildSupervisorSystemPrompt,
|
||||
buildSupervisorUserPrompt,
|
||||
} from "./capability-prompt.js";
|
||||
import { appendSupervisorReview } from "./review-queue.js";
|
||||
import { routeSupervisorTarget } from "./routing-rules.js";
|
||||
import {
|
||||
supervisorRoutingSchema,
|
||||
type SupervisorRoutingDecision,
|
||||
} from "./schema.js";
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: ["*"],
|
||||
@@ -34,6 +49,8 @@ export class SupervisorSkill implements Skill {
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||
) {
|
||||
this.alertService = new AlertService(garnetClient, natsClient);
|
||||
}
|
||||
@@ -51,7 +68,11 @@ export class SupervisorSkill implements Skill {
|
||||
_conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const packet = readHandoffPacket(request);
|
||||
const target = routeSupervisorTarget(packet);
|
||||
const llmDecision = await this.routeByLlm(packet);
|
||||
const useLlmRoute =
|
||||
llmDecision !== null &&
|
||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
@@ -83,7 +104,8 @@ export class SupervisorSkill implements Skill {
|
||||
return {
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
supervisorReason: reason,
|
||||
routedBy: useLlmRoute ? "llm" : "rules",
|
||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
@@ -91,4 +113,54 @@ export class SupervisorSkill implements Skill {
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
|
||||
private async routeByLlm(
|
||||
packet?: HandoffPacket,
|
||||
): Promise<SupervisorRoutingDecision | null> {
|
||||
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
|
||||
role: "system",
|
||||
},
|
||||
{
|
||||
content: buildSupervisorUserPrompt(packet),
|
||||
role: "user",
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: 150,
|
||||
temperature: 0,
|
||||
tier: 0,
|
||||
},
|
||||
);
|
||||
if (!result?.content.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = supervisorRoutingSchema.safeParse(
|
||||
JSON.parse(result.content) as unknown,
|
||||
);
|
||||
if (!parsed.success || !this.isKnownTarget(parsed.data.target)) {
|
||||
return null;
|
||||
}
|
||||
return parsed.data;
|
||||
} catch (error) {
|
||||
getLogger().warn(
|
||||
{ err: error },
|
||||
"[Supervisor] LLM routing failed, falling back to rules",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private isKnownTarget(target: string): boolean {
|
||||
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user