Unify dispatch and add supervisor routing signals

This commit is contained in:
dahyeon committed 2026-05-18 18:42:35 -07:00
1 parent f0b00e5fd2
commit b63985694a
19 files changed
+1345 -812

No files matched your search

+2
View File
@@ -4,6 +4,7 @@ import type { ShadowEvalBus } from "@eval/shadow-bus";
import type { LlmClient } from "@llm/llm-client";
import type { RoomContextService } from "@room/room-context-service";
import type { Skill } from "./skill.js";
import type { SkillRegistry } from "./skill-loader.js";
export interface SkillDeps {
garnetClient: GarnetClient;
@@ -11,6 +12,7 @@ export interface SkillDeps {
natsClient: NatsClient;
roomContextService?: RoomContextService | undefined;
shadowEvalBus: ShadowEvalBus;
skillRegistry?: SkillRegistry | undefined;
}
export type SkillFactory = (deps: SkillDeps) => Skill;
+2 -2
View File
@@ -1,6 +1,6 @@
---
name: supervisor
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
description: LLM 기반 root cause 라우팅 supervisor.
version: 1
priority: 10
domain: meta
@@ -19,4 +19,4 @@ resources:
trustTier: T4
---
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
LLM 기반 root cause 분석과 규칙 fallback을 함께 사용하는 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
@@ -0,0 +1,46 @@
import type { HandoffPacket } from "@agent/handoff-packet";
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
function formatSkill(manifest: SkillManifest): string {
return [
`- ${manifest.name}: ${manifest.description}`,
`domain=${manifest.domain}`,
`canHandle=${manifest.canHandle.join(", ") || "*"}`,
`cannotHandle=${manifest.cannotHandle.join(", ") || "-"}`,
`risk=${manifest.riskTier}`,
].join(" | ");
}
export function buildSupervisorSystemPrompt(
registry: SkillRegistry,
_packet?: HandoffPacket,
): string {
const skills = registry
.list()
.filter((manifest) => manifest.name !== "supervisor");
return [
"너는 multi-agent 시스템의 supervisor다.",
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
"",
"[사용 가능한 specialist]",
...skills.map(formatSkill),
"",
"[응답 형식]",
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단"}',
].join("\n");
}
export function buildSupervisorUserPrompt(
packet?: HandoffPacket,
): string {
if (!packet) {
return "사용자 메시지가 명확하지 않음. 어디로 보낼지 결정해줘.";
}
return [
`[escalation reason]: ${packet.handoffReason}`,
`[user goal]: ${packet.userGoal}`,
`[summary]: ${packet.summary}`,
`[source chain]: ${packet.sourceChain.join(" -> ")}`,
].join("\n");
}
+6 -1
View File
@@ -5,5 +5,10 @@ import { SupervisorSkill } from "./skill.js";
export { SupervisorSkill } from "./skill.js";
export function createSupervisorSkill(deps: SkillDeps): Skill {
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
return new SupervisorSkill(
deps.garnetClient,
deps.natsClient,
deps.llmClient,
deps.skillRegistry,
);
}
+48 -2
View File
@@ -4,15 +4,18 @@ import type { StreamEntry } from "@common/garnet";
import {
getSupervisorReview,
getSupervisorReviewStats,
getResolutionHistory,
listSupervisorReview,
parseSupervisorReviewEntry,
resolveSupervisorReview,
SUPERVISOR_RESOLVED_HASH_PREFIX,
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
SUPERVISOR_REVIEW_STREAM_KEY,
} from "./review-queue.js";
class FakeGarnet {
readonly hashes = new Map<string, Record<string, string>>();
readonly resolutionStream: StreamEntry[] = [];
readonly stream: StreamEntry[];
constructor(stream: StreamEntry[]) {
@@ -25,8 +28,12 @@ class FakeGarnet {
end: string,
options?: { count?: number | undefined },
): Promise<StreamEntry[]> {
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
const entries = this.stream.filter((entry) => {
expect([SUPERVISOR_REVIEW_STREAM_KEY, SUPERVISOR_RESOLUTIONS_STREAM_KEY]).toContain(key);
const source =
key === SUPERVISOR_RESOLUTIONS_STREAM_KEY
? this.resolutionStream
: this.stream;
const entries = source.filter((entry) => {
const afterStart = start === "-" || entry.id >= start;
const beforeEnd = end === "+" || entry.id <= end;
return afterStart && beforeEnd;
@@ -43,6 +50,16 @@ class FakeGarnet {
return Promise.resolve();
}
appendStream(key: string, fields: Record<string, string>): Promise<string> {
expect(key).toBe(SUPERVISOR_RESOLUTIONS_STREAM_KEY);
const id = `${this.resolutionStream.length + 1}-0`;
this.resolutionStream.push({
id,
message: { ...fields },
});
return Promise.resolve(id);
}
expireKey(): Promise<boolean> {
return Promise.resolve(true);
}
@@ -140,6 +157,35 @@ describe("supervisor review queue", () => {
});
});
it("dual-writes resolution updates to an audit stream", async () => {
const garnet = new FakeGarnet(entries);
await resolveSupervisorReview(garnet, "1-0", {
action: "manual-reply",
note: "handled",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await resolveSupervisorReview(garnet, "1-0", {
action: "dismiss",
resolvedAt: "2026-05-18T03:00:00.000Z",
resolvedBy: "operator-2@example.com",
});
await expect(getResolutionHistory(garnet, "1-0")).resolves.toEqual([
{
action: "dismiss",
resolvedAt: "2026-05-18T03:00:00.000Z",
resolvedBy: "operator-2@example.com",
},
{
action: "manual-reply",
note: "handled",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
},
]);
});
it("computes reason totals and top source chains", async () => {
const garnet = new FakeGarnet(entries);
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
+34 -1
View File
@@ -2,6 +2,7 @@ import type { HandoffPacket } from "@agent/handoff-packet";
import type { HandoffReason } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet";
import {
getSupervisorResolutionsStreamMaxLen,
getSupervisorReviewQueueMaxLen,
getSupervisorReviewResolutionTtlSec,
} from "../../config.js";
@@ -9,6 +10,8 @@ import {
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
"lolympus-agent:review-resolved:";
export const SUPERVISOR_RESOLUTIONS_STREAM_KEY =
"lolympus-agent:review-resolutions";
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
@@ -67,7 +70,7 @@ type SupervisorReviewListGarnetClient = Pick<
type SupervisorReviewResolveGarnetClient = Pick<
GarnetClient,
"expireKey" | "setHash"
"appendStream" | "expireKey" | "setHash"
>;
const BASE_FIELDS = new Set([
@@ -285,6 +288,36 @@ export async function resolveSupervisorReview(
const key = resolutionKey(id);
await garnetClient.setHash(key, fields);
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
await garnetClient.appendStream(
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
{
...fields,
reviewId: id,
type: "resolution",
},
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
);
}
export async function getResolutionHistory(
garnetClient: Pick<GarnetClient, "readStreamRange">,
id: string,
): Promise<SupervisorReviewResolution[]> {
const entries = await garnetClient.readStreamRange(
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
"-",
"+",
);
return entries
.flatMap((entry) => {
if (entry.message["reviewId"] !== id) {
return [];
}
const resolution = parseResolution(entry.message);
return resolution ? [resolution] : [];
})
.sort((left, right) => right.resolvedAt.localeCompare(left.resolvedAt));
}
export async function getSupervisorReviewStats(
+9
View File
@@ -0,0 +1,9 @@
import { z } from "zod";
export const supervisorRoutingSchema = z.object({
confidence: z.number().min(0).max(1),
reason: z.string().max(200),
target: z.string().min(1),
});
export type SupervisorRoutingDecision = z.infer<typeof supervisorRoutingSchema>;
+81
View File
@@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest";
import type { AlertPayload } from "@common/nats-subjects";
import { buildHandoffPacket } from "@agent/handoff-packet";
import type { AgentRequest, AgentResponse } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import type { SkillRegistry } from "@skills/skill-loader";
import { SupervisorSkill } from "./skill.js";
class FakeGarnet {
@@ -27,6 +29,44 @@ class FakeNats {
}
}
class FakeLlm {
chatCompletion(): Promise<{
content: string;
endpoint: string;
message: { content: string; role: "assistant" };
provider: "llama";
raw: object;
}> {
return Promise.resolve({
content: JSON.stringify({
confidence: 0.9,
reason: "광고 규칙 문의라 moderation이 적합",
target: "moderation",
}),
endpoint: "memory",
message: { content: "", role: "assistant" },
provider: "llama",
raw: {},
});
}
}
const registry = {
get: (name: string) => (name === "moderation" ? { name } : null),
list: () => [
{
canHandle: ["rules"],
cannotHandle: [],
description: "커뮤니티 규칙 처리",
domain: "community",
handoffTargets: [],
name: "moderation",
outputContract: {},
riskTier: "medium",
},
],
} as unknown as SkillRegistry;
const baseRequest: AgentRequest = {
chatroomId: "room-1",
nickname: "tester",
@@ -70,6 +110,7 @@ describe("SupervisorSkill", () => {
expect(response.text).toContain("운영자");
expect(response.metadata).toMatchObject({
routedBy: "rules",
supervisorReason: "max-hop",
suggestedTarget: "onboarding",
});
@@ -96,4 +137,44 @@ describe("SupervisorSkill", () => {
metric: "agent.supervisor_handoff",
});
});
it("uses LLM routing when the decision is confident and valid", async () => {
const garnet = new FakeGarnet();
const nats = new FakeNats();
const skill = new SupervisorSkill(
garnet,
nats,
new FakeLlm() as unknown as Pick<LlmClient, "chatCompletion">,
registry,
);
const packet = buildHandoffPacket({
handoff: {
confidence: 0.7,
evidenceRefs: [],
expectedOutput: "text",
reason: "out-of-scope",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "광고 규칙 문의",
},
request: baseRequest,
response: sourceResponse,
sourceChain: ["onboarding", "supervisor"],
});
const response = await skill.handle(
{ ...baseRequest, context: { handoffPacket: packet } },
[],
);
expect(response.metadata).toMatchObject({
routedBy: "llm",
supervisorReason: "광고 규칙 문의라 moderation이 적합",
suggestedTarget: "moderation",
});
expect(garnet.streams[0]?.fields).toMatchObject({
reason: "out-of-scope",
target: "moderation",
});
});
});
+75 -3
View File
@@ -4,14 +4,29 @@ import type {
ConversationMessage,
} from "@agent/types";
import { readHandoffPacket } from "@agent/handoff-packet";
import type { HandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import type { LlmClient } from "@llm/llm-client";
import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader";
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
import { getLogger } from "@obs/logger";
import {
getSupervisorLlmConfidenceThreshold,
getSupervisorLlmEnabled,
} from "../../config.js";
import {
buildSupervisorSystemPrompt,
buildSupervisorUserPrompt,
} from "./capability-prompt.js";
import { appendSupervisorReview } from "./review-queue.js";
import { routeSupervisorTarget } from "./routing-rules.js";
import {
supervisorRoutingSchema,
type SupervisorRoutingDecision,
} from "./schema.js";
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: ["*"],
@@ -34,6 +49,8 @@ export class SupervisorSkill implements Skill {
constructor(
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
natsClient: Pick<NatsClient, "publish">,
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
private readonly skillRegistry?: SkillRegistry | undefined,
) {
this.alertService = new AlertService(garnetClient, natsClient);
}
@@ -51,7 +68,11 @@ export class SupervisorSkill implements Skill {
_conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const packet = readHandoffPacket(request);
const target = routeSupervisorTarget(packet);
const llmDecision = await this.routeByLlm(packet);
const useLlmRoute =
llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope";
await appendSupervisorReview(this.garnetClient, {
@@ -83,7 +104,8 @@ export class SupervisorSkill implements Skill {
return {
confidence: 0.6,
metadata: {
supervisorReason: reason,
routedBy: useLlmRoute ? "llm" : "rules",
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
suggestedTarget: target,
},
requestId: request.requestId,
@@ -91,4 +113,54 @@ export class SupervisorSkill implements Skill {
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
};
}
private async routeByLlm(
packet?: HandoffPacket,
): Promise<SupervisorRoutingDecision | null> {
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
return null;
}
try {
const result = await this.llmClient.chatCompletion(
[
{
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
role: "system",
},
{
content: buildSupervisorUserPrompt(packet),
role: "user",
},
],
{
jsonMode: true,
maxTokens: 150,
temperature: 0,
tier: 0,
},
);
if (!result?.content.trim()) {
return null;
}
const parsed = supervisorRoutingSchema.safeParse(
JSON.parse(result.content) as unknown,
);
if (!parsed.success || !this.isKnownTarget(parsed.data.target)) {
return null;
}
return parsed.data;
} catch (error) {
getLogger().warn(
{ err: error },
"[Supervisor] LLM routing failed, falling back to rules",
);
return null;
}
}
private isKnownTarget(target: string): boolean {
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
}
}