133 lines
3.2 KiB
TypeScript
133 lines
3.2 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import { LlmClient } from "@llm/llm-client";
|
|
import type {
|
|
ChatCompletionOptions,
|
|
ChatCompletionResult,
|
|
ChatMessage,
|
|
} from "@llm/types";
|
|
import { judgeOnboardingWithLlm } from "./llm-judge.js";
|
|
import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js";
|
|
|
|
class FakeLlmClient extends LlmClient {
|
|
constructor(private readonly payload: string | null) {
|
|
super({ endpoints: [] });
|
|
}
|
|
|
|
hasEndpoints(): boolean {
|
|
return this.payload !== null;
|
|
}
|
|
|
|
chatCompletion(
|
|
_messages: ChatMessage[],
|
|
_options?: ChatCompletionOptions,
|
|
): Promise<ChatCompletionResult | null> {
|
|
if (this.payload === null) {
|
|
return Promise.resolve(null);
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: this.payload,
|
|
endpoint: "http://judge.test/v1/chat/completions",
|
|
message: {
|
|
role: "assistant",
|
|
content: this.payload,
|
|
},
|
|
provider: "cliproxy",
|
|
raw: {},
|
|
});
|
|
}
|
|
}
|
|
|
|
const evalCase: EvalCaseDefinition = {
|
|
id: "case-1",
|
|
lineNumber: 1,
|
|
input: {
|
|
request: {
|
|
requestId: "req-1",
|
|
skill: "onboarding",
|
|
chatroomId: "room-1",
|
|
userId: "user-1",
|
|
nickname: "프리리/95/여/D",
|
|
trigger: "user-message",
|
|
userMessage: "이제 뭐 남았지",
|
|
},
|
|
},
|
|
expected: {
|
|
intent: "status-question",
|
|
responseMode: "llm",
|
|
},
|
|
};
|
|
|
|
const derived: EvalDerivedSnapshot = {
|
|
allowShortResponse: true,
|
|
intent: "status-question",
|
|
phase: "photo-pending",
|
|
responseMode: "llm",
|
|
shouldRespond: true,
|
|
suggestedMaxTokens: 128,
|
|
tier: 0,
|
|
};
|
|
|
|
describe("judgeOnboardingWithLlm", () => {
|
|
it("parses a valid JSON judge response", async () => {
|
|
const result = await judgeOnboardingWithLlm(
|
|
new FakeLlmClient(
|
|
JSON.stringify({
|
|
friendliness: 5,
|
|
accuracy: 4,
|
|
concision: 5,
|
|
scope: 4,
|
|
rationale: "질문 범위에 맞게 짧고 정확하게 답했다.",
|
|
}),
|
|
),
|
|
evalCase,
|
|
"이제 방 하트 인증만 남았어요.",
|
|
derived,
|
|
);
|
|
|
|
expect(result.skipped).toBe(false);
|
|
if (!result.skipped) {
|
|
expect(result.scores.overall).toBe(4.5);
|
|
expect(result.provider).toBe("cliproxy");
|
|
}
|
|
});
|
|
|
|
it("keeps scores when rationale is longer than the prompt asks for", async () => {
|
|
const result = await judgeOnboardingWithLlm(
|
|
new FakeLlmClient(
|
|
JSON.stringify({
|
|
friendliness: 3,
|
|
accuracy: 4,
|
|
concision: 5,
|
|
scope: 4,
|
|
rationale: "가".repeat(800),
|
|
}),
|
|
),
|
|
evalCase,
|
|
"이제 방 하트 인증만 남았어요.",
|
|
derived,
|
|
);
|
|
|
|
expect(result.skipped).toBe(false);
|
|
if (!result.skipped) {
|
|
expect(result.scores.overall).toBe(4);
|
|
expect(result.rationale).toHaveLength(500);
|
|
}
|
|
});
|
|
|
|
it("skips when the judge returns invalid JSON", async () => {
|
|
const result = await judgeOnboardingWithLlm(
|
|
new FakeLlmClient('{"friendliness":"bad"}'),
|
|
evalCase,
|
|
"이제 방 하트 인증만 남았어요.",
|
|
derived,
|
|
);
|
|
|
|
expect(result.skipped).toBe(true);
|
|
if (result.skipped) {
|
|
expect(result.reason).toBe("invalid-judge-json");
|
|
}
|
|
});
|
|
});
|