Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
ShadowEvalBus,
|
||||
shouldSampleShadowEval,
|
||||
} from "./shadow-bus.js";
|
||||
|
||||
const request: AgentRequest = {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "이제 뭐 남았지",
|
||||
};
|
||||
|
||||
const response: AgentResponse = {
|
||||
requestId: "req-1",
|
||||
text: "이제 방 하트 인증만 남았어요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
};
|
||||
|
||||
const classified: ClassifiedIntent = {
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.98,
|
||||
source: "regex",
|
||||
};
|
||||
|
||||
const derived: DerivedState = {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
};
|
||||
|
||||
describe("shouldSampleShadowEval", () => {
|
||||
it("samples when the random value is under the tier rate", () => {
|
||||
expect(shouldSampleShadowEval(0, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not sample when the random value exceeds the tier rate", () => {
|
||||
expect(shouldSampleShadowEval(0, 1)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ShadowEvalBus", () => {
|
||||
it("publishes a shadow-eval stream entry when sampled", () => {
|
||||
const appendStream = vi.fn().mockResolvedValue("1-0");
|
||||
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
|
||||
|
||||
new ShadowEvalBus({ appendStream }).publish({
|
||||
classified,
|
||||
createdAt: "2026-04-16T00:00:00.000Z",
|
||||
derived,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
|
||||
expect(appendStream).toHaveBeenCalledTimes(1);
|
||||
expect(appendStream.mock.calls[0]?.[0]).toBe("lolympus-agent:background");
|
||||
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
|
||||
type: "shadow-eval",
|
||||
requestId: "req-1",
|
||||
tier: "0",
|
||||
});
|
||||
|
||||
randomSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("skips publishing for empty responses", () => {
|
||||
const appendStream = vi.fn().mockResolvedValue("1-0");
|
||||
|
||||
new ShadowEvalBus({ appendStream }).publish({
|
||||
classified,
|
||||
createdAt: "2026-04-16T00:00:00.000Z",
|
||||
derived,
|
||||
request,
|
||||
response: {
|
||||
...response,
|
||||
text: "",
|
||||
},
|
||||
});
|
||||
|
||||
expect(appendStream).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user