import { describe, expect, it } from "vitest"; import { LlmClient } from "@llm/llm-client"; import type { ChatCompletionOptions, ChatCompletionResult, ChatMessage, } from "@llm/types"; import { judgeOnboardingWithLlm } from "./llm-judge.js"; import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js"; class FakeLlmClient extends LlmClient { constructor(private readonly payload: string | null) { super({ endpoints: [] }); } hasEndpoints(): boolean { return this.payload !== null; } chatCompletion( _messages: ChatMessage[], _options?: ChatCompletionOptions, ): Promise { if (this.payload === null) { return Promise.resolve(null); } return Promise.resolve({ content: this.payload, endpoint: "http://judge.test/v1/chat/completions", message: { role: "assistant", content: this.payload, }, provider: "cliproxy", raw: {}, }); } } const evalCase: EvalCaseDefinition = { id: "case-1", lineNumber: 1, input: { request: { requestId: "req-1", skill: "onboarding", chatroomId: "room-1", userId: "user-1", nickname: "프리리/95/여/D", trigger: "user-message", userMessage: "이제 뭐 남았지", }, }, expected: { intent: "status-question", responseMode: "llm", }, }; const derived: EvalDerivedSnapshot = { allowShortResponse: true, intent: "status-question", phase: "photo-pending", responseMode: "llm", shouldRespond: true, suggestedMaxTokens: 128, tier: 0, }; describe("judgeOnboardingWithLlm", () => { it("parses a valid JSON judge response", async () => { const result = await judgeOnboardingWithLlm( new FakeLlmClient( JSON.stringify({ friendliness: 5, accuracy: 4, concision: 5, scope: 4, rationale: "질문 범위에 맞게 짧고 정확하게 답했다.", }), ), evalCase, "이제 방 하트 인증만 남았어요.", derived, ); expect(result.skipped).toBe(false); if (!result.skipped) { expect(result.scores.overall).toBe(4.5); expect(result.provider).toBe("cliproxy"); } }); it("keeps scores when rationale is longer than the prompt asks for", async () => { const result = await judgeOnboardingWithLlm( new FakeLlmClient( JSON.stringify({ friendliness: 3, accuracy: 4, concision: 5, scope: 4, rationale: "가".repeat(800), }), ), evalCase, "이제 방 하트 인증만 남았어요.", derived, ); expect(result.skipped).toBe(false); if (!result.skipped) { expect(result.scores.overall).toBe(4); expect(result.rationale).toHaveLength(500); } }); it("skips when the judge returns invalid JSON", async () => { const result = await judgeOnboardingWithLlm( new FakeLlmClient('{"friendliness":"bad"}'), evalCase, "이제 방 하트 인증만 남았어요.", derived, ); expect(result.skipped).toBe(true); if (result.skipped) { expect(result.reason).toBe("invalid-judge-json"); } }); });