Refine onboarding state and nickname checks
This commit is contained in:
1 parent
5579db54da
commit
9c013f7d4f
33 files changed
+1523
-119
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { getContext } from "@obs/context";
|
||||
import { getContext, getRequestLogger } from "@obs/context";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
@@ -147,11 +147,11 @@ function buildPlannerMessages(
|
||||
approach:
|
||||
"answer-direct | ask-clarification | redirect-to-step | validate-then-explain | lookup-rule-then-answer | soft-decline",
|
||||
targetPhase:
|
||||
"fresh | nickname-pending | photo-pending | both-pending | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
|
||||
"fresh | nickname-pending | photo-pending | both-pending | step1-completed | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
|
||||
mustInclude: ["short strings"],
|
||||
mustAvoid: ["short strings"],
|
||||
suggestedTools: [
|
||||
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context",
|
||||
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context | verify-discord-nickname",
|
||||
],
|
||||
tone: "friendly | concise | encouraging | neutral | apologetic",
|
||||
targetLengthChars: 150,
|
||||
@@ -186,8 +186,9 @@ function buildResponderAddendum(plan: PlannerOutput): string {
|
||||
function pickToolsForPlan(
|
||||
plan: PlannerOutput,
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
forceTools: boolean,
|
||||
): readonly Tool<unknown, unknown>[] {
|
||||
if (plan.approach === "answer-direct" || plan.approach === "soft-decline") {
|
||||
if (!forceTools && (plan.approach === "answer-direct" || plan.approach === "soft-decline")) {
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -214,8 +215,19 @@ export class OnboardingOrchestrator {
|
||||
maxTokens: number,
|
||||
modelOverride: string,
|
||||
): Promise<string | null> {
|
||||
const logger = getRequestLogger();
|
||||
const deadlineMs = getRemainingDeadlineMs();
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
intent: derived.intent.kind,
|
||||
responderTools: responderTools.map((tool) => tool.name),
|
||||
role: "responder",
|
||||
tier: derived.tier,
|
||||
},
|
||||
"[OnboardingOrchestrator] Responder execution mode decided",
|
||||
);
|
||||
|
||||
if (responderTools.length > 0) {
|
||||
return runWithTools({
|
||||
deadlineMs,
|
||||
@@ -247,6 +259,7 @@ export class OnboardingOrchestrator {
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): Promise<PlannerOutput> {
|
||||
const logger = getRequestLogger();
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
@@ -262,6 +275,14 @@ export class OnboardingOrchestrator {
|
||||
? tryParsePlannerOutput(firstAttempt.content)
|
||||
: null;
|
||||
if (firstPlan) {
|
||||
logger.debug(
|
||||
{
|
||||
intent: derived.intent.kind,
|
||||
plan: firstPlan,
|
||||
source: "first-attempt",
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return firstPlan;
|
||||
}
|
||||
|
||||
@@ -285,7 +306,28 @@ export class OnboardingOrchestrator {
|
||||
});
|
||||
|
||||
const retryPlan = retry?.content ? tryParsePlannerOutput(retry.content) : null;
|
||||
return retryPlan ?? getDefaultPlan(derived);
|
||||
if (retryPlan) {
|
||||
logger.debug(
|
||||
{
|
||||
intent: derived.intent.kind,
|
||||
plan: retryPlan,
|
||||
source: "retry",
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return retryPlan;
|
||||
}
|
||||
|
||||
const fallbackPlan = getDefaultPlan(derived);
|
||||
logger.debug(
|
||||
{
|
||||
intent: derived.intent.kind,
|
||||
plan: fallbackPlan,
|
||||
source: "fallback",
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner fell back to default plan",
|
||||
);
|
||||
return fallbackPlan;
|
||||
}
|
||||
|
||||
private async runTier0(
|
||||
@@ -317,7 +359,8 @@ export class OnboardingOrchestrator {
|
||||
messages,
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
|
||||
const forceTools = derived.responseMode === "llm-with-tools";
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
|
||||
|
||||
return this.runDirect(
|
||||
derived,
|
||||
@@ -339,7 +382,8 @@ export class OnboardingOrchestrator {
|
||||
messages,
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
|
||||
const forceTools = derived.responseMode === "llm-with-tools";
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
|
||||
|
||||
const draft = await this.runDirect(
|
||||
derived,
|
||||
|
||||
@@ -14,6 +14,7 @@ export const PlannerTargetPhaseSchema = z.enum([
|
||||
"nickname-pending",
|
||||
"photo-pending",
|
||||
"both-pending",
|
||||
"step1-completed",
|
||||
"nickname-rejected",
|
||||
"photo-rejected",
|
||||
"deadline-near",
|
||||
@@ -26,6 +27,7 @@ export const PlannerSuggestedToolSchema = z.enum([
|
||||
"check-onboarding-progress",
|
||||
"lookup-community-rule",
|
||||
"get-room-context",
|
||||
"verify-discord-nickname",
|
||||
]);
|
||||
|
||||
export const PlannerToneSchema = z.enum([
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { withToolSpan } from "@obs/spans";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { parseToolCalls } from "@tools/parser";
|
||||
@@ -22,6 +23,7 @@ export interface ToolLoopOptions {
|
||||
export async function runWithTools(
|
||||
options: ToolLoopOptions,
|
||||
): Promise<string | null> {
|
||||
const logger = getRequestLogger();
|
||||
const maxHops = options.maxHops ?? 2;
|
||||
const messages = options.messages.map((message) => ({ ...message }));
|
||||
let allowNativeTools = true;
|
||||
@@ -39,14 +41,44 @@ export async function runWithTools(
|
||||
});
|
||||
|
||||
if (!response) {
|
||||
logger.debug(
|
||||
{
|
||||
hop,
|
||||
role: options.role,
|
||||
tier: options.tier,
|
||||
},
|
||||
"[ToolLoop] LLM returned no response",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const toolCalls = parseToolCalls(response.message);
|
||||
if (toolCalls.length === 0) {
|
||||
logger.debug(
|
||||
{
|
||||
content: response.content,
|
||||
hop,
|
||||
role: options.role,
|
||||
tier: options.tier,
|
||||
},
|
||||
"[ToolLoop] No tool call in model response",
|
||||
);
|
||||
return response.content;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
hop,
|
||||
role: options.role,
|
||||
tier: options.tier,
|
||||
toolCalls: toolCalls.map((call) => ({
|
||||
id: call.id,
|
||||
name: call.name,
|
||||
})),
|
||||
},
|
||||
"[ToolLoop] Parsed tool calls",
|
||||
);
|
||||
|
||||
if (hop === maxHops) {
|
||||
messages.push(response.message);
|
||||
messages.push({
|
||||
@@ -66,6 +98,15 @@ export async function runWithTools(
|
||||
throw new Error(`Tool not registered for call: ${call.name}`);
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
arguments: call.arguments,
|
||||
callId: call.id,
|
||||
tool: tool.name,
|
||||
},
|
||||
"[ToolLoop] Executing tool call",
|
||||
);
|
||||
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
@@ -76,6 +117,15 @@ export async function runWithTools(
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
callId: call.id,
|
||||
result,
|
||||
tool: tool.name,
|
||||
},
|
||||
"[ToolLoop] Tool call completed",
|
||||
);
|
||||
|
||||
messages.push({
|
||||
role: "tool",
|
||||
name: tool.name,
|
||||
|
||||
@@ -19,7 +19,7 @@ function makeDerived(
|
||||
}
|
||||
|
||||
describe("validateOnboardingResponse", () => {
|
||||
it("rejects discord guidance before onboarding is completed", () => {
|
||||
it("rejects discord guidance before step1 is completed", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"디스코드 서버에 들어가시면 돼요.",
|
||||
makeDerived(),
|
||||
@@ -29,6 +29,21 @@ describe("validateOnboardingResponse", () => {
|
||||
expect(result.reasons).toContain("stateConsistency");
|
||||
});
|
||||
|
||||
it("allows discord guidance after step1 is completed", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"이제 디스코드 프로필 닉네임과 프로필 메시지를 설정하시면 돼요.",
|
||||
makeDerived({
|
||||
intent: { kind: "discord-question" },
|
||||
phase: {
|
||||
kind: "step1-completed",
|
||||
remaining: ["discord-profile-nickname", "discord-profile-message"],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
});
|
||||
|
||||
it("rejects invalid nickname examples", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"홍길동/95/남/Z로 바꾸시면 돼요.",
|
||||
|
||||
@@ -96,6 +96,22 @@ function hasRepeatedPhrase(text: string): boolean {
|
||||
return [...phrases.values()].some((count) => count >= 3);
|
||||
}
|
||||
|
||||
function canAdvanceDiscordStep(derived: DerivedState): boolean {
|
||||
if (
|
||||
derived.phase.kind === "completed" ||
|
||||
derived.phase.kind === "step1-completed"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (derived.phase.kind !== "deadline-near") {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !derived.phase.remaining.includes("nickname") &&
|
||||
!derived.phase.remaining.includes("photo");
|
||||
}
|
||||
|
||||
function markValidation(result: ValidationResult): ValidationResult {
|
||||
const activeSpan = trace.getActiveSpan();
|
||||
activeSpan?.setAttributes({
|
||||
@@ -137,7 +153,7 @@ export function validateOnboardingResponse(
|
||||
|
||||
if (
|
||||
mentionsDiscord &&
|
||||
derived.phase.kind !== "completed" &&
|
||||
!canAdvanceDiscordStep(derived) &&
|
||||
advancesDiscordStep &&
|
||||
!defersDiscordStep
|
||||
) {
|
||||
|
||||
Reference in new issue
Block a user