Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+93
View File
@@ -0,0 +1,93 @@
import { describe, expect, it, vi } from "vitest";
import type { AgentRequest, AgentResponse } from "@agent/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
import {
ShadowEvalBus,
shouldSampleShadowEval,
} from "./shadow-bus.js";
const request: AgentRequest = {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "이제 뭐 남았지",
};
const response: AgentResponse = {
requestId: "req-1",
text: "이제 방 하트 인증만 남았어요.",
confidence: 0.9,
source: "llm",
};
const classified: ClassifiedIntent = {
intent: { kind: "status-question" },
confidence: 0.98,
source: "regex",
};
const derived: DerivedState = {
phase: { kind: "photo-pending" },
intent: { kind: "status-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
};
describe("shouldSampleShadowEval", () => {
it("samples when the random value is under the tier rate", () => {
expect(shouldSampleShadowEval(0, 0)).toBe(true);
});
it("does not sample when the random value exceeds the tier rate", () => {
expect(shouldSampleShadowEval(0, 1)).toBe(false);
});
});
describe("ShadowEvalBus", () => {
it("publishes a shadow-eval stream entry when sampled", () => {
const appendStream = vi.fn().mockResolvedValue("1-0");
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
new ShadowEvalBus({ appendStream }).publish({
classified,
createdAt: "2026-04-16T00:00:00.000Z",
derived,
request,
response,
});
expect(appendStream).toHaveBeenCalledTimes(1);
expect(appendStream.mock.calls[0]?.[0]).toBe("lolympus-agent:background");
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
type: "shadow-eval",
requestId: "req-1",
tier: "0",
});
randomSpy.mockRestore();
});
it("skips publishing for empty responses", () => {
const appendStream = vi.fn().mockResolvedValue("1-0");
new ShadowEvalBus({ appendStream }).publish({
classified,
createdAt: "2026-04-16T00:00:00.000Z",
derived,
request,
response: {
...response,
text: "",
},
});
expect(appendStream).not.toHaveBeenCalled();
});
});