.
This commit is contained in:
1 parent
da76af3826
commit
e4f7a5df48
72 files changed
+7077
-113
No files matched your search
@@ -0,0 +1,394 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
Intent,
|
||||
OnboardingPhase,
|
||||
} from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession } from "../types.js";
|
||||
import {
|
||||
decideHybridRouting,
|
||||
decidePreGenerationRouting,
|
||||
extractRouterFeatures,
|
||||
loadLearnedRouterModel,
|
||||
predictRoutingDecisionKind,
|
||||
saveLearnedRouterModel,
|
||||
trainLearnedRouterModel,
|
||||
} from "./router.js";
|
||||
|
||||
function intent(kind: Intent["kind"]): Intent {
|
||||
return { kind };
|
||||
}
|
||||
|
||||
function request(overrides?: Partial<AgentRequest>): AgentRequest {
|
||||
return {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "도와주세요",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function session(overrides?: Partial<OnboardingSession>): OnboardingSession {
|
||||
return {
|
||||
chatroomId: "room-1",
|
||||
deadlineAt: "2026-05-02T00:00:00.000Z",
|
||||
joinedAt: "2026-05-01T00:00:00.000Z",
|
||||
kakaotalkNickname: "프리리/95/여/D",
|
||||
kakaotalkNicknameAliases: [],
|
||||
kakaotalkNicknameVerified: false,
|
||||
lastEventAt: "2026-05-01T00:00:00.000Z",
|
||||
nicknameVerified: false,
|
||||
photoVerified: false,
|
||||
status: "pending",
|
||||
userId: "user-1",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function derived(
|
||||
phase: OnboardingPhase,
|
||||
nextIntent: Intent,
|
||||
overrides?: Partial<DerivedState>,
|
||||
): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: nextIntent,
|
||||
phase,
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 256,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function classified(
|
||||
kind: Intent["kind"],
|
||||
confidence: number,
|
||||
): ClassifiedIntent {
|
||||
return {
|
||||
confidence,
|
||||
intent: intent(kind),
|
||||
source: "regex",
|
||||
};
|
||||
}
|
||||
|
||||
describe("decidePreGenerationRouting", () => {
|
||||
it.each([
|
||||
{
|
||||
confidence: 0.91,
|
||||
expected: { kind: "abstain", reason: "off-topic-strong" },
|
||||
intentKind: "off-topic-to-other" as const,
|
||||
name: "abstains on high-confidence off-topic",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.9,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "off-topic-to-other" as const,
|
||||
name: "does not abstain at the off-topic confidence boundary",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.3,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "off-topic-to-other" as const,
|
||||
name: "keeps low-confidence off-topic on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.96,
|
||||
expected: { kind: "template-only", templateKey: "greeting" },
|
||||
intentKind: "greeting" as const,
|
||||
name: "uses a greeting template",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.95,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "greeting" as const,
|
||||
name: "does not template greeting at the confidence boundary",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.99,
|
||||
expected: { kind: "template-only", templateKey: "greeting" },
|
||||
intentKind: "greeting" as const,
|
||||
name: "uses greeting template from a pending phase",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.91,
|
||||
expected: { kind: "template-only", templateKey: "acknowledgement" },
|
||||
intentKind: "acknowledgement" as const,
|
||||
name: "uses an acknowledgement template",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.9,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "acknowledgement" as const,
|
||||
name: "does not template acknowledgement at the confidence boundary",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "template-only", templateKey: "completed" },
|
||||
intentKind: "help-how" as const,
|
||||
name: "keeps completed sessions on template path",
|
||||
phase: { kind: "completed" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "status-question" as const,
|
||||
name: "lets completed status questions through",
|
||||
phase: { kind: "completed" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "template-only", templateKey: "completed" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "uses completed template for completed phase simple questions",
|
||||
phase: { kind: "completed" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "nickname-question" as const,
|
||||
name: "routes nickname help through tools",
|
||||
phase: { kind: "nickname-pending", current: "프리리" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "nickname-question" as const,
|
||||
name: "routes nickname-rejected phase through tools",
|
||||
phase: {
|
||||
kind: "nickname-rejected",
|
||||
current: "프리리/95/여D",
|
||||
issues: [],
|
||||
} as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "photo-question" as const,
|
||||
name: "routes photo help through tools",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "photo-question" as const,
|
||||
name: "routes photo help through tools from both-pending",
|
||||
phase: { kind: "both-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "help-how" as const,
|
||||
name: "routes general step-one help through tools",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "help-where" as const,
|
||||
name: "routes where-help through tools",
|
||||
phase: { kind: "both-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "help-how" as const,
|
||||
name: "keeps later phase general help on tier0",
|
||||
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "discord-verify-question" as const,
|
||||
name: "routes discord verification through tools",
|
||||
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "discord-verify-question" as const,
|
||||
name: "routes discord verification through tools from fresh phase",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-direct", skipPlanner: false },
|
||||
intentKind: "status-question" as const,
|
||||
name: "keeps deadline-near on tier1",
|
||||
phase: { kind: "deadline-near", remaining: ["photo"] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-direct", skipPlanner: false },
|
||||
intentKind: "age-question" as const,
|
||||
name: "routes deadline-near simple questions through tier1",
|
||||
phase: { kind: "deadline-near", remaining: ["nickname"] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "defaults simple questions to tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "keeps age questions on tier0 during photo-pending",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "status-question" as const,
|
||||
name: "keeps ordinary status questions on tier0",
|
||||
phase: { kind: "both-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "discord-question" as const,
|
||||
name: "keeps generic discord questions on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "unknown" as const,
|
||||
name: "keeps unknown intent on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "chitchat" as const,
|
||||
name: "keeps chitchat on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "keeps simple questions on tier0 during photo rejection",
|
||||
phase: { kind: "photo-rejected", reason: "unclear" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "nickname-question" as const,
|
||||
name: "keeps nickname question on tier0 after step one",
|
||||
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.89,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "acknowledgement" as const,
|
||||
name: "keeps lower-confidence acknowledgement on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
])("$name", ({ confidence, expected, intentKind, phase }) => {
|
||||
const nextIntent = intent(intentKind);
|
||||
expect(
|
||||
decidePreGenerationRouting({
|
||||
classified: classified(intentKind, confidence),
|
||||
derived: derived(phase, nextIntent),
|
||||
request: request(),
|
||||
session: session(),
|
||||
}),
|
||||
).toEqual(expected);
|
||||
});
|
||||
|
||||
it("trains, saves, loads, and applies a learned router model", () => {
|
||||
const lowConfidenceInput = {
|
||||
classified: classified("age-question", 0.25),
|
||||
derived: derived({ kind: "fresh" }, intent("age-question")),
|
||||
request: request({ userMessage: "나이 제한이 있나요?" }),
|
||||
session: session(),
|
||||
};
|
||||
const highConfidenceInput = {
|
||||
classified: classified("age-question", 0.95),
|
||||
derived: derived({ kind: "fresh" }, intent("age-question")),
|
||||
request: request({ userMessage: "나이 제한이 있나요?" }),
|
||||
session: session(),
|
||||
};
|
||||
const model = trainLearnedRouterModel(
|
||||
[
|
||||
{
|
||||
features: extractRouterFeatures(lowConfidenceInput),
|
||||
label: "tier1-direct",
|
||||
},
|
||||
{
|
||||
features: extractRouterFeatures(highConfidenceInput),
|
||||
label: "tier0-only",
|
||||
},
|
||||
],
|
||||
{
|
||||
epochs: 200,
|
||||
learningRate: 0.5,
|
||||
trainedAt: "2026-05-01T00:00:00.000Z",
|
||||
},
|
||||
);
|
||||
const path = join(mkdtempSync(join(tmpdir(), "router-")), "weights.json");
|
||||
|
||||
saveLearnedRouterModel(path, model);
|
||||
const loaded = loadLearnedRouterModel(path);
|
||||
|
||||
expect(loaded?.trainedAt).toBe("2026-05-01T00:00:00.000Z");
|
||||
expect(
|
||||
predictRoutingDecisionKind(model, extractRouterFeatures(lowConfidenceInput)),
|
||||
).toBe("tier1-direct");
|
||||
expect(
|
||||
predictRoutingDecisionKind(model, extractRouterFeatures(highConfidenceInput)),
|
||||
).toBe("tier0-only");
|
||||
expect(
|
||||
decideHybridRouting(lowConfidenceInput, { learnedModel: model }),
|
||||
).toEqual({
|
||||
kind: "tier1-direct",
|
||||
skipPlanner: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps static abstain and template guards in hybrid routing", () => {
|
||||
const offTopicInput = {
|
||||
classified: classified("off-topic-to-other", 0.99),
|
||||
derived: derived({ kind: "fresh" }, intent("off-topic-to-other")),
|
||||
request: request({ userMessage: "다른 주제로 얘기하자" }),
|
||||
session: session(),
|
||||
};
|
||||
const model = trainLearnedRouterModel(
|
||||
[
|
||||
{
|
||||
features: extractRouterFeatures(offTopicInput),
|
||||
label: "tier2-full",
|
||||
},
|
||||
],
|
||||
{ epochs: 10 },
|
||||
);
|
||||
|
||||
expect(decideHybridRouting(offTopicInput, { learnedModel: model })).toEqual({
|
||||
kind: "abstain",
|
||||
reason: "off-topic-strong",
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user