Refine onboarding state and nickname checks

This commit is contained in:
dahyeon committed 2026-04-21 12:42:24 -07:00
1 parent 5579db54da
commit 9c013f7d4f
33 files changed
+1523 -119

No files matched your search

+51 -7
View File
@@ -1,4 +1,4 @@
import { getContext } from "@obs/context";
import { getContext, getRequestLogger } from "@obs/context";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { Tool, ToolContext } from "@tools/tool";
@@ -147,11 +147,11 @@ function buildPlannerMessages(
approach:
"answer-direct | ask-clarification | redirect-to-step | validate-then-explain | lookup-rule-then-answer | soft-decline",
targetPhase:
"fresh | nickname-pending | photo-pending | both-pending | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
"fresh | nickname-pending | photo-pending | both-pending | step1-completed | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
mustInclude: ["short strings"],
mustAvoid: ["short strings"],
suggestedTools: [
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context",
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context | verify-discord-nickname",
],
tone: "friendly | concise | encouraging | neutral | apologetic",
targetLengthChars: 150,
@@ -186,8 +186,9 @@ function buildResponderAddendum(plan: PlannerOutput): string {
function pickToolsForPlan(
plan: PlannerOutput,
tools: readonly Tool<unknown, unknown>[],
forceTools: boolean,
): readonly Tool<unknown, unknown>[] {
if (plan.approach === "answer-direct" || plan.approach === "soft-decline") {
if (!forceTools && (plan.approach === "answer-direct" || plan.approach === "soft-decline")) {
return [];
}
@@ -214,8 +215,19 @@ export class OnboardingOrchestrator {
maxTokens: number,
modelOverride: string,
): Promise<string | null> {
const logger = getRequestLogger();
const deadlineMs = getRemainingDeadlineMs();
logger.debug(
{
intent: derived.intent.kind,
responderTools: responderTools.map((tool) => tool.name),
role: "responder",
tier: derived.tier,
},
"[OnboardingOrchestrator] Responder execution mode decided",
);
if (responderTools.length > 0) {
return runWithTools({
deadlineMs,
@@ -247,6 +259,7 @@ export class OnboardingOrchestrator {
derived: DerivedState,
messages: ChatMessage[],
): Promise<PlannerOutput> {
const logger = getRequestLogger();
const plannerMessages = buildPlannerMessages(derived, messages);
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
deadlineMs: getRemainingDeadlineMs(),
@@ -262,6 +275,14 @@ export class OnboardingOrchestrator {
? tryParsePlannerOutput(firstAttempt.content)
: null;
if (firstPlan) {
logger.debug(
{
intent: derived.intent.kind,
plan: firstPlan,
source: "first-attempt",
},
"[OnboardingOrchestrator] Planner output parsed",
);
return firstPlan;
}
@@ -285,7 +306,28 @@ export class OnboardingOrchestrator {
});
const retryPlan = retry?.content ? tryParsePlannerOutput(retry.content) : null;
return retryPlan ?? getDefaultPlan(derived);
if (retryPlan) {
logger.debug(
{
intent: derived.intent.kind,
plan: retryPlan,
source: "retry",
},
"[OnboardingOrchestrator] Planner output parsed",
);
return retryPlan;
}
const fallbackPlan = getDefaultPlan(derived);
logger.debug(
{
intent: derived.intent.kind,
plan: fallbackPlan,
source: "fallback",
},
"[OnboardingOrchestrator] Planner fell back to default plan",
);
return fallbackPlan;
}
private async runTier0(
@@ -317,7 +359,8 @@ export class OnboardingOrchestrator {
messages,
buildResponderAddendum(plan),
);
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
const forceTools = derived.responseMode === "llm-with-tools";
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
return this.runDirect(
derived,
@@ -339,7 +382,8 @@ export class OnboardingOrchestrator {
messages,
buildResponderAddendum(plan),
);
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
const forceTools = derived.responseMode === "llm-with-tools";
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
const draft = await this.runDirect(
derived,
@@ -14,6 +14,7 @@ export const PlannerTargetPhaseSchema = z.enum([
"nickname-pending",
"photo-pending",
"both-pending",
"step1-completed",
"nickname-rejected",
"photo-rejected",
"deadline-near",
@@ -26,6 +27,7 @@ export const PlannerSuggestedToolSchema = z.enum([
"check-onboarding-progress",
"lookup-community-rule",
"get-room-context",
"verify-discord-nickname",
]);
export const PlannerToneSchema = z.enum([
@@ -1,4 +1,5 @@
import { withToolSpan } from "@obs/spans";
import { getRequestLogger } from "@obs/context";
import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
@@ -22,6 +23,7 @@ export interface ToolLoopOptions {
export async function runWithTools(
options: ToolLoopOptions,
): Promise<string | null> {
const logger = getRequestLogger();
const maxHops = options.maxHops ?? 2;
const messages = options.messages.map((message) => ({ ...message }));
let allowNativeTools = true;
@@ -39,14 +41,44 @@ export async function runWithTools(
});
if (!response) {
logger.debug(
{
hop,
role: options.role,
tier: options.tier,
},
"[ToolLoop] LLM returned no response",
);
return null;
}
const toolCalls = parseToolCalls(response.message);
if (toolCalls.length === 0) {
logger.debug(
{
content: response.content,
hop,
role: options.role,
tier: options.tier,
},
"[ToolLoop] No tool call in model response",
);
return response.content;
}
logger.debug(
{
hop,
role: options.role,
tier: options.tier,
toolCalls: toolCalls.map((call) => ({
id: call.id,
name: call.name,
})),
},
"[ToolLoop] Parsed tool calls",
);
if (hop === maxHops) {
messages.push(response.message);
messages.push({
@@ -66,6 +98,15 @@ export async function runWithTools(
throw new Error(`Tool not registered for call: ${call.name}`);
}
logger.debug(
{
arguments: call.arguments,
callId: call.id,
tool: tool.name,
},
"[ToolLoop] Executing tool call",
);
const validated = tool.argsSchema.parse(call.arguments);
const result = await withToolSpan(
tool.name,
@@ -76,6 +117,15 @@ export async function runWithTools(
() => tool.execute(validated, options.toolContext),
);
logger.debug(
{
callId: call.id,
result,
tool: tool.name,
},
"[ToolLoop] Tool call completed",
);
messages.push({
role: "tool",
name: tool.name,
@@ -19,7 +19,7 @@ function makeDerived(
}
describe("validateOnboardingResponse", () => {
it("rejects discord guidance before onboarding is completed", () => {
it("rejects discord guidance before step1 is completed", () => {
const result = validateOnboardingResponse(
"디스코드 서버에 들어가시면 돼요.",
makeDerived(),
@@ -29,6 +29,21 @@ describe("validateOnboardingResponse", () => {
expect(result.reasons).toContain("stateConsistency");
});
it("allows discord guidance after step1 is completed", () => {
const result = validateOnboardingResponse(
"이제 디스코드 프로필 닉네임과 프로필 메시지를 설정하시면 돼요.",
makeDerived({
intent: { kind: "discord-question" },
phase: {
kind: "step1-completed",
remaining: ["discord-profile-nickname", "discord-profile-message"],
},
}),
);
expect(result.status).toBe("pass");
});
it("rejects invalid nickname examples", () => {
const result = validateOnboardingResponse(
"홍길동/95/남/Z로 바꾸시면 돼요.",
+17 -1
View File
@@ -96,6 +96,22 @@ function hasRepeatedPhrase(text: string): boolean {
return [...phrases.values()].some((count) => count >= 3);
}
function canAdvanceDiscordStep(derived: DerivedState): boolean {
if (
derived.phase.kind === "completed" ||
derived.phase.kind === "step1-completed"
) {
return true;
}
if (derived.phase.kind !== "deadline-near") {
return false;
}
return !derived.phase.remaining.includes("nickname") &&
!derived.phase.remaining.includes("photo");
}
function markValidation(result: ValidationResult): ValidationResult {
const activeSpan = trace.getActiveSpan();
activeSpan?.setAttributes({
@@ -137,7 +153,7 @@ export function validateOnboardingResponse(
if (
mentionsDiscord &&
derived.phase.kind !== "completed" &&
!canAdvanceDiscordStep(derived) &&
advancesDiscordStep &&
!defersDiscordStep
) {