395 lines
12 KiB
TypeScript
395 lines
12 KiB
TypeScript
import { mkdtempSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
import type { AgentRequest } from "@agent/types";
|
|
import type {
|
|
ClassifiedIntent,
|
|
DerivedState,
|
|
Intent,
|
|
OnboardingPhase,
|
|
} from "../state/onboarding-state.js";
|
|
import type { OnboardingSession } from "../types.js";
|
|
import {
|
|
decideHybridRouting,
|
|
decidePreGenerationRouting,
|
|
extractRouterFeatures,
|
|
loadLearnedRouterModel,
|
|
predictRoutingDecisionKind,
|
|
saveLearnedRouterModel,
|
|
trainLearnedRouterModel,
|
|
} from "./router.js";
|
|
|
|
function intent(kind: Intent["kind"]): Intent {
|
|
return { kind };
|
|
}
|
|
|
|
function request(overrides?: Partial<AgentRequest>): AgentRequest {
|
|
return {
|
|
requestId: "req-1",
|
|
skill: "onboarding",
|
|
chatroomId: "room-1",
|
|
userId: "user-1",
|
|
nickname: "프리리/95/여/D",
|
|
trigger: "user-message",
|
|
userMessage: "도와주세요",
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function session(overrides?: Partial<OnboardingSession>): OnboardingSession {
|
|
return {
|
|
chatroomId: "room-1",
|
|
deadlineAt: "2026-05-02T00:00:00.000Z",
|
|
joinedAt: "2026-05-01T00:00:00.000Z",
|
|
kakaotalkNickname: "프리리/95/여/D",
|
|
kakaotalkNicknameAliases: [],
|
|
kakaotalkNicknameVerified: false,
|
|
lastEventAt: "2026-05-01T00:00:00.000Z",
|
|
nicknameVerified: false,
|
|
photoVerified: false,
|
|
status: "pending",
|
|
userId: "user-1",
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function derived(
|
|
phase: OnboardingPhase,
|
|
nextIntent: Intent,
|
|
overrides?: Partial<DerivedState>,
|
|
): DerivedState {
|
|
return {
|
|
allowShortResponse: false,
|
|
intent: nextIntent,
|
|
phase,
|
|
responseMode: "llm",
|
|
shouldRespond: true,
|
|
suggestedMaxTokens: 256,
|
|
tier: 0,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function classified(
|
|
kind: Intent["kind"],
|
|
confidence: number,
|
|
): ClassifiedIntent {
|
|
return {
|
|
confidence,
|
|
intent: intent(kind),
|
|
source: "regex",
|
|
};
|
|
}
|
|
|
|
describe("decidePreGenerationRouting", () => {
|
|
it.each([
|
|
{
|
|
confidence: 0.91,
|
|
expected: { kind: "abstain", reason: "off-topic-strong" },
|
|
intentKind: "off-topic-to-other" as const,
|
|
name: "abstains on high-confidence off-topic",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.9,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "off-topic-to-other" as const,
|
|
name: "does not abstain at the off-topic confidence boundary",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.3,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "off-topic-to-other" as const,
|
|
name: "keeps low-confidence off-topic on tier0",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.96,
|
|
expected: { kind: "template-only", templateKey: "greeting" },
|
|
intentKind: "greeting" as const,
|
|
name: "uses a greeting template",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.95,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "greeting" as const,
|
|
name: "does not template greeting at the confidence boundary",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.99,
|
|
expected: { kind: "template-only", templateKey: "greeting" },
|
|
intentKind: "greeting" as const,
|
|
name: "uses greeting template from a pending phase",
|
|
phase: { kind: "photo-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.91,
|
|
expected: { kind: "template-only", templateKey: "acknowledgement" },
|
|
intentKind: "acknowledgement" as const,
|
|
name: "uses an acknowledgement template",
|
|
phase: { kind: "photo-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.9,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "acknowledgement" as const,
|
|
name: "does not template acknowledgement at the confidence boundary",
|
|
phase: { kind: "photo-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "template-only", templateKey: "completed" },
|
|
intentKind: "help-how" as const,
|
|
name: "keeps completed sessions on template path",
|
|
phase: { kind: "completed" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "status-question" as const,
|
|
name: "lets completed status questions through",
|
|
phase: { kind: "completed" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "template-only", templateKey: "completed" },
|
|
intentKind: "age-question" as const,
|
|
name: "uses completed template for completed phase simple questions",
|
|
phase: { kind: "completed" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "nickname-question" as const,
|
|
name: "routes nickname help through tools",
|
|
phase: { kind: "nickname-pending", current: "프리리" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "nickname-question" as const,
|
|
name: "routes nickname-rejected phase through tools",
|
|
phase: {
|
|
kind: "nickname-rejected",
|
|
current: "프리리/95/여D",
|
|
issues: [],
|
|
} as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "photo-question" as const,
|
|
name: "routes photo help through tools",
|
|
phase: { kind: "photo-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "photo-question" as const,
|
|
name: "routes photo help through tools from both-pending",
|
|
phase: { kind: "both-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "help-how" as const,
|
|
name: "routes general step-one help through tools",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "help-where" as const,
|
|
name: "routes where-help through tools",
|
|
phase: { kind: "both-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "help-how" as const,
|
|
name: "keeps later phase general help on tier0",
|
|
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "discord-verify-question" as const,
|
|
name: "routes discord verification through tools",
|
|
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-with-tools" },
|
|
intentKind: "discord-verify-question" as const,
|
|
name: "routes discord verification through tools from fresh phase",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-direct", skipPlanner: false },
|
|
intentKind: "status-question" as const,
|
|
name: "keeps deadline-near on tier1",
|
|
phase: { kind: "deadline-near", remaining: ["photo"] } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier1-direct", skipPlanner: false },
|
|
intentKind: "age-question" as const,
|
|
name: "routes deadline-near simple questions through tier1",
|
|
phase: { kind: "deadline-near", remaining: ["nickname"] } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "age-question" as const,
|
|
name: "defaults simple questions to tier0",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "age-question" as const,
|
|
name: "keeps age questions on tier0 during photo-pending",
|
|
phase: { kind: "photo-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "status-question" as const,
|
|
name: "keeps ordinary status questions on tier0",
|
|
phase: { kind: "both-pending" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "discord-question" as const,
|
|
name: "keeps generic discord questions on tier0",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "unknown" as const,
|
|
name: "keeps unknown intent on tier0",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "chitchat" as const,
|
|
name: "keeps chitchat on tier0",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "age-question" as const,
|
|
name: "keeps simple questions on tier0 during photo rejection",
|
|
phase: { kind: "photo-rejected", reason: "unclear" } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.8,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "nickname-question" as const,
|
|
name: "keeps nickname question on tier0 after step one",
|
|
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
|
},
|
|
{
|
|
confidence: 0.89,
|
|
expected: { kind: "tier0-only" },
|
|
intentKind: "acknowledgement" as const,
|
|
name: "keeps lower-confidence acknowledgement on tier0",
|
|
phase: { kind: "fresh" } as OnboardingPhase,
|
|
},
|
|
])("$name", ({ confidence, expected, intentKind, phase }) => {
|
|
const nextIntent = intent(intentKind);
|
|
expect(
|
|
decidePreGenerationRouting({
|
|
classified: classified(intentKind, confidence),
|
|
derived: derived(phase, nextIntent),
|
|
request: request(),
|
|
session: session(),
|
|
}),
|
|
).toEqual(expected);
|
|
});
|
|
|
|
it("trains, saves, loads, and applies a learned router model", () => {
|
|
const lowConfidenceInput = {
|
|
classified: classified("age-question", 0.25),
|
|
derived: derived({ kind: "fresh" }, intent("age-question")),
|
|
request: request({ userMessage: "나이 제한이 있나요?" }),
|
|
session: session(),
|
|
};
|
|
const highConfidenceInput = {
|
|
classified: classified("age-question", 0.95),
|
|
derived: derived({ kind: "fresh" }, intent("age-question")),
|
|
request: request({ userMessage: "나이 제한이 있나요?" }),
|
|
session: session(),
|
|
};
|
|
const model = trainLearnedRouterModel(
|
|
[
|
|
{
|
|
features: extractRouterFeatures(lowConfidenceInput),
|
|
label: "tier1-direct",
|
|
},
|
|
{
|
|
features: extractRouterFeatures(highConfidenceInput),
|
|
label: "tier0-only",
|
|
},
|
|
],
|
|
{
|
|
epochs: 200,
|
|
learningRate: 0.5,
|
|
trainedAt: "2026-05-01T00:00:00.000Z",
|
|
},
|
|
);
|
|
const path = join(mkdtempSync(join(tmpdir(), "router-")), "weights.json");
|
|
|
|
saveLearnedRouterModel(path, model);
|
|
const loaded = loadLearnedRouterModel(path);
|
|
|
|
expect(loaded?.trainedAt).toBe("2026-05-01T00:00:00.000Z");
|
|
expect(
|
|
predictRoutingDecisionKind(model, extractRouterFeatures(lowConfidenceInput)),
|
|
).toBe("tier1-direct");
|
|
expect(
|
|
predictRoutingDecisionKind(model, extractRouterFeatures(highConfidenceInput)),
|
|
).toBe("tier0-only");
|
|
expect(
|
|
decideHybridRouting(lowConfidenceInput, { learnedModel: model }),
|
|
).toEqual({
|
|
kind: "tier1-direct",
|
|
skipPlanner: false,
|
|
});
|
|
});
|
|
|
|
it("keeps static abstain and template guards in hybrid routing", () => {
|
|
const offTopicInput = {
|
|
classified: classified("off-topic-to-other", 0.99),
|
|
derived: derived({ kind: "fresh" }, intent("off-topic-to-other")),
|
|
request: request({ userMessage: "다른 주제로 얘기하자" }),
|
|
session: session(),
|
|
};
|
|
const model = trainLearnedRouterModel(
|
|
[
|
|
{
|
|
features: extractRouterFeatures(offTopicInput),
|
|
label: "tier2-full",
|
|
},
|
|
],
|
|
{ epochs: 10 },
|
|
);
|
|
|
|
expect(decideHybridRouting(offTopicInput, { learnedModel: model })).toEqual({
|
|
kind: "abstain",
|
|
reason: "off-topic-strong",
|
|
});
|
|
});
|
|
});
|