Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -1,3 +1,6 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession, PromptContext } from "../types.js";
|
||||
@@ -10,11 +13,104 @@ import {
|
||||
} from "./layers.js";
|
||||
import { buildToolPolicy } from "./tool-policy.js";
|
||||
|
||||
export type PromptLayerName =
|
||||
| "context"
|
||||
| "examples"
|
||||
| "persona"
|
||||
| "policy"
|
||||
| "state"
|
||||
| "tool-policy";
|
||||
|
||||
export type PromptLayerOperation = "append" | "remove" | "replace";
|
||||
|
||||
export interface PromptLayerDiff {
|
||||
layer: PromptLayerName;
|
||||
op: PromptLayerOperation;
|
||||
reason?: string | undefined;
|
||||
text?: string | undefined;
|
||||
}
|
||||
|
||||
export interface PromptVariantManifest {
|
||||
description?: string | undefined;
|
||||
diff: PromptLayerDiff[];
|
||||
expectedImprovement?: string | undefined;
|
||||
rationale?: string | undefined;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||
|
||||
function getLayerTarget(layer: PromptLayerName): Exclude<PromptLayerName, "tool-policy"> {
|
||||
return layer === "tool-policy" ? "policy" : layer;
|
||||
}
|
||||
|
||||
function applyLayerDiff(
|
||||
layers: Record<Exclude<PromptLayerName, "tool-policy">, string>,
|
||||
diff: PromptLayerDiff,
|
||||
): void {
|
||||
const target = getLayerTarget(diff.layer);
|
||||
|
||||
if (diff.op === "remove") {
|
||||
layers[target] = "";
|
||||
return;
|
||||
}
|
||||
|
||||
const text = diff.text?.trim();
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (diff.op === "replace") {
|
||||
layers[target] = text;
|
||||
return;
|
||||
}
|
||||
|
||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||
}
|
||||
|
||||
export function loadPromptVariantManifest(
|
||||
variantId: string,
|
||||
): PromptVariantManifest | null {
|
||||
const normalized = variantId.trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (variantCache.has(normalized)) {
|
||||
return variantCache.get(normalized) ?? null;
|
||||
}
|
||||
|
||||
const path = resolve(
|
||||
process.cwd(),
|
||||
"src",
|
||||
"eval",
|
||||
"proposals",
|
||||
`${normalized}.json`,
|
||||
);
|
||||
if (!existsSync(path)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
||||
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
variantCache.set(normalized, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class PromptBuilder {
|
||||
private constructor(
|
||||
private readonly derived: DerivedState,
|
||||
private readonly session: OnboardingSession | null,
|
||||
private readonly context: PromptContext,
|
||||
private readonly variant?: PromptVariantManifest | undefined,
|
||||
) {}
|
||||
|
||||
static forDerived(
|
||||
@@ -25,7 +121,18 @@ export class PromptBuilder {
|
||||
return new PromptBuilder(derived, session, context);
|
||||
}
|
||||
|
||||
build(): ChatMessage[] {
|
||||
applyVariant(variant: PromptVariantManifest | string): PromptBuilder {
|
||||
const manifest =
|
||||
typeof variant === "string" ? loadPromptVariantManifest(variant) : variant;
|
||||
return new PromptBuilder(
|
||||
this.derived,
|
||||
this.session,
|
||||
this.context,
|
||||
manifest ?? undefined,
|
||||
);
|
||||
}
|
||||
|
||||
buildSystemPrompt(): string {
|
||||
const promptHistory = this.context.summary
|
||||
? this.context.conversationHistory.slice(-5)
|
||||
: this.context.conversationHistory;
|
||||
@@ -34,21 +141,41 @@ export class PromptBuilder {
|
||||
conversationHistory: promptHistory,
|
||||
};
|
||||
|
||||
const systemPrompt = [
|
||||
personaLayer(),
|
||||
policyLayer(this.derived.intent, {
|
||||
const layers = {
|
||||
persona: personaLayer(),
|
||||
policy: policyLayer(this.derived.intent, {
|
||||
toolPolicy:
|
||||
this.derived.responseMode === "llm-with-tools"
|
||||
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
|
||||
: undefined,
|
||||
}),
|
||||
stateLayer(this.session, this.derived),
|
||||
examplesLayer(this.derived.intent),
|
||||
contextLayer(promptContext),
|
||||
].join("\n\n");
|
||||
state: stateLayer(this.session, this.derived),
|
||||
examples: examplesLayer(this.derived.intent),
|
||||
context: contextLayer(promptContext),
|
||||
};
|
||||
|
||||
for (const diff of this.variant?.diff ?? []) {
|
||||
applyLayerDiff(layers, diff);
|
||||
}
|
||||
|
||||
return [
|
||||
{ role: "system", content: systemPrompt },
|
||||
layers.persona,
|
||||
layers.policy,
|
||||
layers.state,
|
||||
layers.examples,
|
||||
layers.context,
|
||||
]
|
||||
.filter((layer) => layer.trim().length > 0)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
build(): ChatMessage[] {
|
||||
const promptHistory = this.context.summary
|
||||
? this.context.conversationHistory.slice(-5)
|
||||
: this.context.conversationHistory;
|
||||
|
||||
return [
|
||||
{ role: "system", content: this.buildSystemPrompt() },
|
||||
...promptHistory.map((message) => ({
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
getTier2CriticModel,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface CritiqueResult {
|
||||
changed: boolean;
|
||||
@@ -55,6 +56,7 @@ export class OnboardingCritic {
|
||||
messages: ChatMessage[],
|
||||
options?: { deadlineMs?: number | undefined },
|
||||
): Promise<CritiqueResult> {
|
||||
const traceStep = getActiveProcessTrace()?.startStep("critic");
|
||||
const response = await this.llmClient.chatCompletion(
|
||||
buildCriticMessages(draft, derived, messages),
|
||||
{
|
||||
@@ -69,12 +71,20 @@ export class OnboardingCritic {
|
||||
|
||||
const text = response?.content.trim() ?? "";
|
||||
if (!text || text.startsWith("[OK]")) {
|
||||
traceStep?.finish({
|
||||
decision: "keep",
|
||||
rationale: text || "empty critic response",
|
||||
});
|
||||
return {
|
||||
changed: false,
|
||||
text: draft,
|
||||
};
|
||||
}
|
||||
|
||||
traceStep?.finish({
|
||||
decision: "rewrite",
|
||||
rationale: text.slice(0, 200),
|
||||
});
|
||||
return {
|
||||
changed: true,
|
||||
text,
|
||||
|
||||
@@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
import type { PlannerOutput } from "./planner-schema.js";
|
||||
import { PlannerOutputSchema } from "./planner-schema.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
|
||||
export interface OrchestratorRunOptions {
|
||||
@@ -260,6 +261,17 @@ export class OnboardingOrchestrator {
|
||||
messages: ChatMessage[],
|
||||
): Promise<PlannerOutput> {
|
||||
const logger = getRequestLogger();
|
||||
const traceStep = getActiveProcessTrace()?.startStep("plan");
|
||||
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
|
||||
traceStep?.finish({
|
||||
approach: plan.approach,
|
||||
mustAvoid: plan.mustAvoid,
|
||||
mustInclude: plan.mustInclude,
|
||||
suggestedTools: plan.suggestedTools,
|
||||
targetLengthChars: plan.targetLengthChars,
|
||||
});
|
||||
return plan;
|
||||
};
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
@@ -283,7 +295,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return firstPlan;
|
||||
return finishTrace(firstPlan);
|
||||
}
|
||||
|
||||
const retryMessages = [
|
||||
@@ -315,7 +327,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return retryPlan;
|
||||
return finishTrace(retryPlan);
|
||||
}
|
||||
|
||||
const fallbackPlan = getDefaultPlan(derived);
|
||||
@@ -327,7 +339,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner fell back to default plan",
|
||||
);
|
||||
return fallbackPlan;
|
||||
return finishTrace(fallbackPlan);
|
||||
}
|
||||
|
||||
private async runTier0(
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { parseToolCalls } from "@tools/parser";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface ToolLoopOptions {
|
||||
deadlineMs?: number | undefined;
|
||||
@@ -107,15 +108,33 @@ export async function runWithTools(
|
||||
"[ToolLoop] Executing tool call",
|
||||
);
|
||||
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
||||
let result: unknown;
|
||||
try {
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
name: tool.name,
|
||||
ok: true,
|
||||
resultSummary: summarizeToolResult(result),
|
||||
});
|
||||
} catch (error) {
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
@@ -137,3 +156,11 @@ export async function runWithTools(
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function summarizeToolResult(result: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(result).slice(0, 200);
|
||||
} catch {
|
||||
return String(result).slice(0, 200);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
|
||||
export type ProcessTraceStepName =
|
||||
| "classify"
|
||||
| "critic"
|
||||
| "derive"
|
||||
| "plan"
|
||||
| "respond"
|
||||
| "tool-call"
|
||||
| "validate";
|
||||
|
||||
export interface ProcessTraceStep<T> {
|
||||
name: ProcessTraceStepName;
|
||||
startedAtMs: number;
|
||||
durationMs: number;
|
||||
output: T;
|
||||
}
|
||||
|
||||
export interface ProcessTraceSnapshot {
|
||||
steps: {
|
||||
classify?: ProcessTraceStep<{
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: "regex" | "llm";
|
||||
}> | undefined;
|
||||
derive?: ProcessTraceStep<{
|
||||
phase: string;
|
||||
responseMode: string;
|
||||
shouldRespond: boolean;
|
||||
tier: 0 | 1 | 2;
|
||||
}> | undefined;
|
||||
plan?: ProcessTraceStep<{
|
||||
approach: string;
|
||||
mustAvoid: string[];
|
||||
mustInclude: string[];
|
||||
suggestedTools: string[];
|
||||
targetLengthChars: number;
|
||||
}> | undefined;
|
||||
toolCalls?: Array<
|
||||
ProcessTraceStep<{
|
||||
args: unknown;
|
||||
name: string;
|
||||
ok: boolean;
|
||||
resultSummary: string;
|
||||
}>
|
||||
> | undefined;
|
||||
respond?: ProcessTraceStep<{
|
||||
draftText: string;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
}> | undefined;
|
||||
critic?: ProcessTraceStep<{
|
||||
decision: "keep" | "rewrite";
|
||||
rationale: string;
|
||||
}> | undefined;
|
||||
validate?: ProcessTraceStep<{
|
||||
reasons: string[];
|
||||
status: "pass" | "reject" | "soft-warn";
|
||||
}> | undefined;
|
||||
};
|
||||
}
|
||||
|
||||
type ProcessTraceOutputByName = {
|
||||
classify: NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"];
|
||||
critic: NonNullable<ProcessTraceSnapshot["steps"]["critic"]>["output"];
|
||||
derive: NonNullable<ProcessTraceSnapshot["steps"]["derive"]>["output"];
|
||||
plan: NonNullable<ProcessTraceSnapshot["steps"]["plan"]>["output"];
|
||||
respond: NonNullable<ProcessTraceSnapshot["steps"]["respond"]>["output"];
|
||||
"tool-call": NonNullable<
|
||||
NonNullable<ProcessTraceSnapshot["steps"]["toolCalls"]>[number]
|
||||
>["output"];
|
||||
validate: NonNullable<ProcessTraceSnapshot["steps"]["validate"]>["output"];
|
||||
};
|
||||
|
||||
const processTraceStorage = new AsyncLocalStorage<ProcessTraceCollector>();
|
||||
|
||||
export class ProcessTraceCollector {
|
||||
private readonly steps: ProcessTraceSnapshot["steps"] = {};
|
||||
|
||||
startStep<Name extends ProcessTraceStepName>(
|
||||
name: Name,
|
||||
): {
|
||||
finish: (output: ProcessTraceOutputByName[Name]) => void;
|
||||
} {
|
||||
const startedAtMs = Date.now();
|
||||
let finished = false;
|
||||
|
||||
return {
|
||||
finish: (output: ProcessTraceOutputByName[Name]): void => {
|
||||
if (finished) {
|
||||
return;
|
||||
}
|
||||
finished = true;
|
||||
|
||||
const step: ProcessTraceStep<ProcessTraceOutputByName[Name]> = {
|
||||
name,
|
||||
startedAtMs,
|
||||
durationMs: Math.max(0, Date.now() - startedAtMs),
|
||||
output,
|
||||
};
|
||||
|
||||
if (name === "tool-call") {
|
||||
this.steps.toolCalls = [
|
||||
...(this.steps.toolCalls ?? []),
|
||||
step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["tool-call"]
|
||||
>,
|
||||
];
|
||||
return;
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case "classify":
|
||||
this.steps.classify = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["classify"]
|
||||
>;
|
||||
return;
|
||||
case "critic":
|
||||
this.steps.critic = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["critic"]
|
||||
>;
|
||||
return;
|
||||
case "derive":
|
||||
this.steps.derive = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["derive"]
|
||||
>;
|
||||
return;
|
||||
case "plan":
|
||||
this.steps.plan = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["plan"]
|
||||
>;
|
||||
return;
|
||||
case "respond":
|
||||
this.steps.respond = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["respond"]
|
||||
>;
|
||||
return;
|
||||
case "validate":
|
||||
this.steps.validate = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["validate"]
|
||||
>;
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): ProcessTraceSnapshot {
|
||||
return {
|
||||
steps: {
|
||||
...this.steps,
|
||||
toolCalls: this.steps.toolCalls
|
||||
? [...this.steps.toolCalls]
|
||||
: undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function runWithProcessTrace<T>(
|
||||
collector: ProcessTraceCollector,
|
||||
fn: () => T,
|
||||
): T {
|
||||
return processTraceStorage.run(collector, fn);
|
||||
}
|
||||
|
||||
export function getActiveProcessTrace(): ProcessTraceCollector | undefined {
|
||||
return processTraceStorage.getStore();
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { getMaxResponseLength } from "../../../config.js";
|
||||
import { VALID_TIER_SET } from "../constants.js";
|
||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
const ALLOWED_URL_PATTERNS = [
|
||||
/discord\.gg\/lolympus/iu,
|
||||
@@ -118,6 +119,10 @@ function markValidation(result: ValidationResult): ValidationResult {
|
||||
"agent.validator.status": result.status,
|
||||
"agent.validator.warnings": result.reasons,
|
||||
});
|
||||
getActiveProcessTrace()?.startStep("validate").finish({
|
||||
status: result.status,
|
||||
reasons: result.reasons,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ export function collectNicknameAliases(
|
||||
|
||||
for (const source of sources) {
|
||||
if (Array.isArray(source)) {
|
||||
for (const value of source) {
|
||||
for (const value of source as readonly string[]) {
|
||||
pushAlias(value);
|
||||
}
|
||||
continue;
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { ChatMessage } from '@llm/types';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
import type { Skill } from '@skills/skill';
|
||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||
import { createBuiltInToolRegistry } from '@tools/builtins/index';
|
||||
import type { Tool, ToolContext } from '@tools/tool';
|
||||
import { PromptBuilder } from './prompts/builder.js';
|
||||
@@ -27,12 +28,21 @@ import {
|
||||
getRemainingStatusMessage,
|
||||
getTimeoutWarningMessage,
|
||||
} from './onboarding-templates.js';
|
||||
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from '../../config.js';
|
||||
import {
|
||||
getOnboardingSessionKeyPrefix,
|
||||
getLlmTemperature,
|
||||
getPromptVariantId,
|
||||
} from '../../config.js';
|
||||
import { OnboardingBlackboardStore } from './state/blackboard.js';
|
||||
import { deriveState } from './state/derive.js';
|
||||
import { OnboardingIntentClassifier } from './state/classifier.js';
|
||||
import type { DerivedState, Intent } from './state/onboarding-state.js';
|
||||
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
|
||||
import {
|
||||
getActiveProcessTrace,
|
||||
ProcessTraceCollector,
|
||||
runWithProcessTrace,
|
||||
} from './runtime/trace.js';
|
||||
import { validateOnboardingResponse } from './runtime/validator.js';
|
||||
import { normalizeOnboardingSession } from './session.js';
|
||||
import type { OnboardingSession, PromptContext, StoredOnboardingSession } from './types.js';
|
||||
@@ -71,19 +81,57 @@ export class OnboardingSkill implements Skill {
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
if (getActiveProcessTrace()) {
|
||||
return this.handleWithTrace(request, conversationHistory);
|
||||
}
|
||||
|
||||
const collector = new ProcessTraceCollector();
|
||||
return runWithProcessTrace(collector, () =>
|
||||
this.handleWithTrace(request, conversationHistory),
|
||||
);
|
||||
}
|
||||
|
||||
private async handleWithTrace(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const logger = getRequestLogger();
|
||||
const trace = getActiveProcessTrace();
|
||||
const session = await this.loadSession(request.chatroomId, request.userId);
|
||||
|
||||
if (request.trigger === 'user-message' && !request.userMessage?.trim()) {
|
||||
return this.makeEmptyResponse(request.requestId);
|
||||
}
|
||||
|
||||
const classifyStep = trace?.startStep('classify');
|
||||
const classified = await this.classifier.classify(request.userMessage, request.trigger);
|
||||
classifyStep?.finish({
|
||||
confidence: classified.confidence,
|
||||
intent: classified.intent.kind,
|
||||
source: classified.source,
|
||||
});
|
||||
|
||||
const deriveStep = trace?.startStep('derive');
|
||||
const derived = deriveState(session, request.trigger, classified.intent);
|
||||
deriveStep?.finish({
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
if (!derived.shouldRespond) {
|
||||
return this.makeEmptyResponse(request.requestId);
|
||||
const response = this.makeEmptyResponse(request.requestId);
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse('[NO_RESPONSE]', derived, {
|
||||
allowShortResponse: true,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
let response: AgentResponse;
|
||||
@@ -102,6 +150,15 @@ export class OnboardingSkill implements Skill {
|
||||
}
|
||||
}
|
||||
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
});
|
||||
|
||||
if (derived.intent.kind === 'discord-verify-question') {
|
||||
logger.debug(
|
||||
{
|
||||
@@ -122,6 +179,7 @@ export class OnboardingSkill implements Skill {
|
||||
createdAt: new Date().toISOString(),
|
||||
derived,
|
||||
latencyMs: this.getCurrentLatencyMs(),
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
@@ -185,7 +243,9 @@ export class OnboardingSkill implements Skill {
|
||||
userMessage: this.buildPromptUserMessage(request, derived),
|
||||
};
|
||||
|
||||
return PromptBuilder.forDerived(derived, session, context).build();
|
||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||
const promptVariantId = getPromptVariantId();
|
||||
return promptVariantId ? builder.applyVariant(promptVariantId).build() : builder.build();
|
||||
}
|
||||
|
||||
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { DEADLINE_NEAR_THRESHOLD_MIN } from "../constants.js";
|
||||
import { analyzeNicknameIssues } from "../onboarding-templates.js";
|
||||
import type { OnboardingSession } from "../types.js";
|
||||
import type { OnboardingPhase, RemainingStep } from "./onboarding-state.js";
|
||||
import type { OnboardingPhase } from "./onboarding-state.js";
|
||||
import {
|
||||
getRemainingSteps,
|
||||
isOnboardingCompleted,
|
||||
|
||||
Reference in new issue
Block a user