Add bounded skill handoff graph
This commit is contained in:
1 parent
54bc295ead
commit
c14adac162
42 files changed
+2243
-31
No files matched your search
+249
-1
@@ -1,7 +1,10 @@
|
||||
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import { selectSkillForRequest } from "./agent.js";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||
import { Agent, selectSkillForRequest } from "./agent.js";
|
||||
|
||||
function makeRequest(skill = "shared"): AgentRequest {
|
||||
return {
|
||||
@@ -32,6 +35,60 @@ function makeSkill(name: string, priority: number, canHandle = true): Skill {
|
||||
};
|
||||
}
|
||||
|
||||
class MemoryConversationStore {
|
||||
readonly messages: ConversationMessage[] = [];
|
||||
|
||||
getHistory(): Promise<ConversationMessage[]> {
|
||||
return Promise.resolve([...this.messages]);
|
||||
}
|
||||
|
||||
appendMessage(
|
||||
_chatroomId: string,
|
||||
_userId: string,
|
||||
message: ConversationMessage,
|
||||
): Promise<void> {
|
||||
this.messages.push(message);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class FakeNatsClient {
|
||||
readonly responses: AgentResponse[] = [];
|
||||
|
||||
respond(_msg: unknown, response: AgentResponse): Promise<void> {
|
||||
this.responses.push(response);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleThroughAgent(
|
||||
skills: readonly Skill[],
|
||||
request: AgentRequest,
|
||||
transitionMatrix?: TransitionMatrix,
|
||||
): Promise<{
|
||||
nats: FakeNatsClient;
|
||||
store: MemoryConversationStore;
|
||||
}> {
|
||||
const nats = new FakeNatsClient();
|
||||
const store = new MemoryConversationStore();
|
||||
const agent = new Agent(
|
||||
nats as unknown as NatsClient,
|
||||
store as unknown as ConversationStore,
|
||||
transitionMatrix,
|
||||
);
|
||||
for (const skill of skills) {
|
||||
agent.registerSkill(skill);
|
||||
}
|
||||
|
||||
await (agent as unknown as {
|
||||
handleRequest(msg: { data: Uint8Array }): Promise<void>;
|
||||
}).handleRequest({
|
||||
data: new TextEncoder().encode(JSON.stringify(request)),
|
||||
});
|
||||
|
||||
return { nats, store };
|
||||
}
|
||||
|
||||
describe("selectSkillForRequest", () => {
|
||||
it("selects the highest-priority matching skill", () => {
|
||||
const request = makeRequest();
|
||||
@@ -51,3 +108,194 @@ describe("selectSkillForRequest", () => {
|
||||
expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Agent handoff dispatch", () => {
|
||||
it("responds once with the final handoff target response", async () => {
|
||||
const source: Skill = {
|
||||
...makeSkill("source", 50),
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.8,
|
||||
metadata: {
|
||||
handoffRequest: {
|
||||
confidence: 0.8,
|
||||
evidenceRefs: ["rule-1"],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "target",
|
||||
taskSpec: "continue",
|
||||
},
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "handoff",
|
||||
}),
|
||||
};
|
||||
const target: Skill = {
|
||||
...makeSkill("target", 60),
|
||||
canHandle: (request) => request.skill === "target",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 1,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "final answer",
|
||||
}),
|
||||
};
|
||||
|
||||
const { nats, store } = await handleThroughAgent(
|
||||
[source, target],
|
||||
makeRequest("source"),
|
||||
);
|
||||
|
||||
expect(nats.responses).toHaveLength(1);
|
||||
expect(nats.responses[0]?.text).toBe("final answer");
|
||||
expect(store.messages.map((message) => message.content)).toEqual([
|
||||
"final answer",
|
||||
]);
|
||||
});
|
||||
|
||||
it("routes handoff control tokens through metadata", async () => {
|
||||
const source: Skill = {
|
||||
...makeSkill("source", 50),
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.9,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "[HANDOFF:target:missing-tool]",
|
||||
}),
|
||||
};
|
||||
const target: Skill = {
|
||||
...makeSkill("target", 60),
|
||||
canHandle: (request) => request.skill === "target",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 1,
|
||||
metadata: {
|
||||
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "handled elsewhere",
|
||||
}),
|
||||
};
|
||||
|
||||
const { nats } = await handleThroughAgent(
|
||||
[source, target],
|
||||
makeRequest("source"),
|
||||
);
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("handled elsewhere");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
});
|
||||
|
||||
it("routes runtime verifier escalations to supervisor", async () => {
|
||||
const previous = process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
|
||||
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = "true";
|
||||
try {
|
||||
const moderation: Skill = {
|
||||
...makeSkill("moderation", 50),
|
||||
canHandle: (request) => request.skill === "moderation",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.9,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "https://not-allowed.example",
|
||||
}),
|
||||
};
|
||||
const supervisor: Skill = {
|
||||
...makeSkill("supervisor", 90),
|
||||
canHandle: (request) => request.skill === "supervisor",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "sent to review",
|
||||
}),
|
||||
};
|
||||
|
||||
const { nats } = await handleThroughAgent(
|
||||
[moderation, supervisor],
|
||||
makeRequest("moderation"),
|
||||
);
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("sent to review");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
} finally {
|
||||
if (previous === undefined) {
|
||||
delete process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
|
||||
} else {
|
||||
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = previous;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("reroutes illegal transitions to supervisor", async () => {
|
||||
const source: Skill = {
|
||||
...makeSkill("source", 50),
|
||||
canHandle: (request) => request.skill === "source",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.9,
|
||||
metadata: {
|
||||
handoffRequest: {
|
||||
confidence: 0.9,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "target",
|
||||
taskSpec: "continue",
|
||||
},
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "handoff",
|
||||
}),
|
||||
};
|
||||
const target: Skill = {
|
||||
...makeSkill("target", 60),
|
||||
canHandle: (request) => request.skill === "target",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 1,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "should not run",
|
||||
}),
|
||||
};
|
||||
const supervisor: Skill = {
|
||||
...makeSkill("supervisor", 10),
|
||||
canHandle: (request) => request.skill === "supervisor",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "supervisor review",
|
||||
}),
|
||||
};
|
||||
|
||||
const matrix = {
|
||||
isAllowed: () => false,
|
||||
} as unknown as TransitionMatrix;
|
||||
const { nats } = await handleThroughAgent(
|
||||
[source, target, supervisor],
|
||||
makeRequest("source"),
|
||||
matrix,
|
||||
);
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("supervisor review");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user