Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+109
View File
@@ -0,0 +1,109 @@
import { describe, expect, it } from "vitest";
import { LlmClient } from "@llm/llm-client";
import type {
ChatCompletionOptions,
ChatCompletionResult,
ChatMessage,
} from "@llm/types";
import { judgeOnboardingWithLlm } from "./llm-judge.js";
import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js";
class FakeLlmClient extends LlmClient {
constructor(private readonly payload: string | null) {
super({ endpoints: [] });
}
hasEndpoints(): boolean {
return this.payload !== null;
}
chatCompletion(
_messages: ChatMessage[],
_options?: ChatCompletionOptions,
): Promise<ChatCompletionResult | null> {
if (this.payload === null) {
return Promise.resolve(null);
}
return Promise.resolve({
content: this.payload,
endpoint: "http://judge.test/v1/chat/completions",
message: {
role: "assistant",
content: this.payload,
},
provider: "cliproxy",
raw: {},
});
}
}
const evalCase: EvalCaseDefinition = {
id: "case-1",
lineNumber: 1,
input: {
request: {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "이제 뭐 남았지",
},
},
expected: {
intent: "status-question",
responseMode: "llm",
},
};
const derived: EvalDerivedSnapshot = {
allowShortResponse: true,
intent: "status-question",
phase: "photo-pending",
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 128,
tier: 0,
};
describe("judgeOnboardingWithLlm", () => {
it("parses a valid JSON judge response", async () => {
const result = await judgeOnboardingWithLlm(
new FakeLlmClient(
JSON.stringify({
friendliness: 5,
accuracy: 4,
concision: 5,
scope: 4,
rationale: "질문 범위에 맞게 짧고 정확하게 답했다.",
}),
),
evalCase,
"이제 방 하트 인증만 남았어요.",
derived,
);
expect(result.skipped).toBe(false);
if (!result.skipped) {
expect(result.scores.overall).toBe(4.5);
expect(result.provider).toBe("cliproxy");
}
});
it("skips when the judge returns invalid JSON", async () => {
const result = await judgeOnboardingWithLlm(
new FakeLlmClient('{"friendliness":"bad"}'),
evalCase,
"이제 방 하트 인증만 남았어요.",
derived,
);
expect(result.skipped).toBe(true);
if (result.skipped) {
expect(result.reason).toBe("invalid-judge-json");
}
});
});