Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
@@ -0,0 +1,109 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { LlmClient } from "@llm/llm-client";
|
||||
import type {
|
||||
ChatCompletionOptions,
|
||||
ChatCompletionResult,
|
||||
ChatMessage,
|
||||
} from "@llm/types";
|
||||
import { judgeOnboardingWithLlm } from "./llm-judge.js";
|
||||
import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js";
|
||||
|
||||
class FakeLlmClient extends LlmClient {
|
||||
constructor(private readonly payload: string | null) {
|
||||
super({ endpoints: [] });
|
||||
}
|
||||
|
||||
hasEndpoints(): boolean {
|
||||
return this.payload !== null;
|
||||
}
|
||||
|
||||
chatCompletion(
|
||||
_messages: ChatMessage[],
|
||||
_options?: ChatCompletionOptions,
|
||||
): Promise<ChatCompletionResult | null> {
|
||||
if (this.payload === null) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: this.payload,
|
||||
endpoint: "http://judge.test/v1/chat/completions",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: this.payload,
|
||||
},
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const evalCase: EvalCaseDefinition = {
|
||||
id: "case-1",
|
||||
lineNumber: 1,
|
||||
input: {
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "이제 뭐 남았지",
|
||||
},
|
||||
},
|
||||
expected: {
|
||||
intent: "status-question",
|
||||
responseMode: "llm",
|
||||
},
|
||||
};
|
||||
|
||||
const derived: EvalDerivedSnapshot = {
|
||||
allowShortResponse: true,
|
||||
intent: "status-question",
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 128,
|
||||
tier: 0,
|
||||
};
|
||||
|
||||
describe("judgeOnboardingWithLlm", () => {
|
||||
it("parses a valid JSON judge response", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient(
|
||||
JSON.stringify({
|
||||
friendliness: 5,
|
||||
accuracy: 4,
|
||||
concision: 5,
|
||||
scope: 4,
|
||||
rationale: "질문 범위에 맞게 짧고 정확하게 답했다.",
|
||||
}),
|
||||
),
|
||||
evalCase,
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
derived,
|
||||
);
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.scores.overall).toBe(4.5);
|
||||
expect(result.provider).toBe("cliproxy");
|
||||
}
|
||||
});
|
||||
|
||||
it("skips when the judge returns invalid JSON", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient('{"friendliness":"bad"}'),
|
||||
evalCase,
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
derived,
|
||||
);
|
||||
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) {
|
||||
expect(result.reason).toBe("invalid-judge-json");
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user