This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent da76af3826
commit e4f7a5df48
72 files changed
+7077 -113

No files matched your search

@@ -0,0 +1,394 @@
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import type { AgentRequest } from "@agent/types";
import type {
ClassifiedIntent,
DerivedState,
Intent,
OnboardingPhase,
} from "../state/onboarding-state.js";
import type { OnboardingSession } from "../types.js";
import {
decideHybridRouting,
decidePreGenerationRouting,
extractRouterFeatures,
loadLearnedRouterModel,
predictRoutingDecisionKind,
saveLearnedRouterModel,
trainLearnedRouterModel,
} from "./router.js";
function intent(kind: Intent["kind"]): Intent {
return { kind };
}
function request(overrides?: Partial<AgentRequest>): AgentRequest {
return {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "도와주세요",
...overrides,
};
}
function session(overrides?: Partial<OnboardingSession>): OnboardingSession {
return {
chatroomId: "room-1",
deadlineAt: "2026-05-02T00:00:00.000Z",
joinedAt: "2026-05-01T00:00:00.000Z",
kakaotalkNickname: "프리리/95/여/D",
kakaotalkNicknameAliases: [],
kakaotalkNicknameVerified: false,
lastEventAt: "2026-05-01T00:00:00.000Z",
nicknameVerified: false,
photoVerified: false,
status: "pending",
userId: "user-1",
...overrides,
};
}
function derived(
phase: OnboardingPhase,
nextIntent: Intent,
overrides?: Partial<DerivedState>,
): DerivedState {
return {
allowShortResponse: false,
intent: nextIntent,
phase,
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 256,
tier: 0,
...overrides,
};
}
function classified(
kind: Intent["kind"],
confidence: number,
): ClassifiedIntent {
return {
confidence,
intent: intent(kind),
source: "regex",
};
}
describe("decidePreGenerationRouting", () => {
it.each([
{
confidence: 0.91,
expected: { kind: "abstain", reason: "off-topic-strong" },
intentKind: "off-topic-to-other" as const,
name: "abstains on high-confidence off-topic",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.9,
expected: { kind: "tier0-only" },
intentKind: "off-topic-to-other" as const,
name: "does not abstain at the off-topic confidence boundary",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.3,
expected: { kind: "tier0-only" },
intentKind: "off-topic-to-other" as const,
name: "keeps low-confidence off-topic on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.96,
expected: { kind: "template-only", templateKey: "greeting" },
intentKind: "greeting" as const,
name: "uses a greeting template",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.95,
expected: { kind: "tier0-only" },
intentKind: "greeting" as const,
name: "does not template greeting at the confidence boundary",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.99,
expected: { kind: "template-only", templateKey: "greeting" },
intentKind: "greeting" as const,
name: "uses greeting template from a pending phase",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.91,
expected: { kind: "template-only", templateKey: "acknowledgement" },
intentKind: "acknowledgement" as const,
name: "uses an acknowledgement template",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.9,
expected: { kind: "tier0-only" },
intentKind: "acknowledgement" as const,
name: "does not template acknowledgement at the confidence boundary",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "template-only", templateKey: "completed" },
intentKind: "help-how" as const,
name: "keeps completed sessions on template path",
phase: { kind: "completed" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "status-question" as const,
name: "lets completed status questions through",
phase: { kind: "completed" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "template-only", templateKey: "completed" },
intentKind: "age-question" as const,
name: "uses completed template for completed phase simple questions",
phase: { kind: "completed" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "nickname-question" as const,
name: "routes nickname help through tools",
phase: { kind: "nickname-pending", current: "프리리" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "nickname-question" as const,
name: "routes nickname-rejected phase through tools",
phase: {
kind: "nickname-rejected",
current: "프리리/95/여D",
issues: [],
} as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "photo-question" as const,
name: "routes photo help through tools",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "photo-question" as const,
name: "routes photo help through tools from both-pending",
phase: { kind: "both-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "help-how" as const,
name: "routes general step-one help through tools",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "help-where" as const,
name: "routes where-help through tools",
phase: { kind: "both-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "help-how" as const,
name: "keeps later phase general help on tier0",
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "discord-verify-question" as const,
name: "routes discord verification through tools",
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "discord-verify-question" as const,
name: "routes discord verification through tools from fresh phase",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-direct", skipPlanner: false },
intentKind: "status-question" as const,
name: "keeps deadline-near on tier1",
phase: { kind: "deadline-near", remaining: ["photo"] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-direct", skipPlanner: false },
intentKind: "age-question" as const,
name: "routes deadline-near simple questions through tier1",
phase: { kind: "deadline-near", remaining: ["nickname"] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "age-question" as const,
name: "defaults simple questions to tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "age-question" as const,
name: "keeps age questions on tier0 during photo-pending",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "status-question" as const,
name: "keeps ordinary status questions on tier0",
phase: { kind: "both-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "discord-question" as const,
name: "keeps generic discord questions on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "unknown" as const,
name: "keeps unknown intent on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "chitchat" as const,
name: "keeps chitchat on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "age-question" as const,
name: "keeps simple questions on tier0 during photo rejection",
phase: { kind: "photo-rejected", reason: "unclear" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "nickname-question" as const,
name: "keeps nickname question on tier0 after step one",
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
},
{
confidence: 0.89,
expected: { kind: "tier0-only" },
intentKind: "acknowledgement" as const,
name: "keeps lower-confidence acknowledgement on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
])("$name", ({ confidence, expected, intentKind, phase }) => {
const nextIntent = intent(intentKind);
expect(
decidePreGenerationRouting({
classified: classified(intentKind, confidence),
derived: derived(phase, nextIntent),
request: request(),
session: session(),
}),
).toEqual(expected);
});
it("trains, saves, loads, and applies a learned router model", () => {
const lowConfidenceInput = {
classified: classified("age-question", 0.25),
derived: derived({ kind: "fresh" }, intent("age-question")),
request: request({ userMessage: "나이 제한이 있나요?" }),
session: session(),
};
const highConfidenceInput = {
classified: classified("age-question", 0.95),
derived: derived({ kind: "fresh" }, intent("age-question")),
request: request({ userMessage: "나이 제한이 있나요?" }),
session: session(),
};
const model = trainLearnedRouterModel(
[
{
features: extractRouterFeatures(lowConfidenceInput),
label: "tier1-direct",
},
{
features: extractRouterFeatures(highConfidenceInput),
label: "tier0-only",
},
],
{
epochs: 200,
learningRate: 0.5,
trainedAt: "2026-05-01T00:00:00.000Z",
},
);
const path = join(mkdtempSync(join(tmpdir(), "router-")), "weights.json");
saveLearnedRouterModel(path, model);
const loaded = loadLearnedRouterModel(path);
expect(loaded?.trainedAt).toBe("2026-05-01T00:00:00.000Z");
expect(
predictRoutingDecisionKind(model, extractRouterFeatures(lowConfidenceInput)),
).toBe("tier1-direct");
expect(
predictRoutingDecisionKind(model, extractRouterFeatures(highConfidenceInput)),
).toBe("tier0-only");
expect(
decideHybridRouting(lowConfidenceInput, { learnedModel: model }),
).toEqual({
kind: "tier1-direct",
skipPlanner: false,
});
});
it("keeps static abstain and template guards in hybrid routing", () => {
const offTopicInput = {
classified: classified("off-topic-to-other", 0.99),
derived: derived({ kind: "fresh" }, intent("off-topic-to-other")),
request: request({ userMessage: "다른 주제로 얘기하자" }),
session: session(),
};
const model = trainLearnedRouterModel(
[
{
features: extractRouterFeatures(offTopicInput),
label: "tier2-full",
},
],
{ epochs: 10 },
);
expect(decideHybridRouting(offTopicInput, { learnedModel: model })).toEqual({
kind: "abstain",
reason: "off-topic-strong",
});
});
});