Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
@@ -0,0 +1,182 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { OnboardingOrchestrator } from "./orchestrator.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
|
||||
return {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "help-how" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 256,
|
||||
allowShortResponse: false,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("OnboardingOrchestrator", () => {
|
||||
it("uses direct response path for tier 0", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
return Promise.resolve({
|
||||
content: "안내 도와드릴게요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "안내 도와드릴게요." } }] },
|
||||
message: { role: "assistant" as const, content: "안내 도와드릴게요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 0 }),
|
||||
[{ role: "user", content: "도와줘" }],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("안내 도와드릴게요.");
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]?.options?.role).toBe("responder");
|
||||
});
|
||||
|
||||
it("runs planner then responder for tier 1", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "redirect-to-step",
|
||||
targetPhase: "photo-pending",
|
||||
mustInclude: ["하트 인증만 안내"],
|
||||
mustAvoid: ["디스코드 절차"],
|
||||
suggestedTools: [],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "질문 범위만 답하기",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
|
||||
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 1 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "이제 뭐 남았지" },
|
||||
],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("하트 인증만 남았어요.");
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls[0]?.options?.role).toBe("planner");
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[PLAN]");
|
||||
});
|
||||
|
||||
it("runs critic after responder for tier 2 and applies rewrite", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "answer-direct",
|
||||
targetPhase: "photo-pending",
|
||||
mustInclude: [],
|
||||
mustAvoid: ["추가 절차 나열"],
|
||||
suggestedTools: [],
|
||||
tone: "concise",
|
||||
targetLengthChars: 60,
|
||||
rationale: "짧게 답하기",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
if (calls.length === 2) {
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: {
|
||||
choices: [
|
||||
{ message: { content: "하트 인증만 남았어요. 커버 보기로 들어가서..." } },
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant" as const,
|
||||
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
|
||||
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 2 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "이제 뭐 남았지" },
|
||||
],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("하트 인증만 남았어요.");
|
||||
expect(calls).toHaveLength(3);
|
||||
expect(calls[2]?.options?.role).toBe("critic");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user