Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -8,6 +8,7 @@ import type { ChatMessage } from '@llm/types';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
import type { Skill } from '@skills/skill';
|
||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||
import { createBuiltInToolRegistry } from '@tools/builtins/index';
|
||||
import type { Tool, ToolContext } from '@tools/tool';
|
||||
import { PromptBuilder } from './prompts/builder.js';
|
||||
@@ -27,12 +28,21 @@ import {
|
||||
getRemainingStatusMessage,
|
||||
getTimeoutWarningMessage,
|
||||
} from './onboarding-templates.js';
|
||||
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from '../../config.js';
|
||||
import {
|
||||
getOnboardingSessionKeyPrefix,
|
||||
getLlmTemperature,
|
||||
getPromptVariantId,
|
||||
} from '../../config.js';
|
||||
import { OnboardingBlackboardStore } from './state/blackboard.js';
|
||||
import { deriveState } from './state/derive.js';
|
||||
import { OnboardingIntentClassifier } from './state/classifier.js';
|
||||
import type { DerivedState, Intent } from './state/onboarding-state.js';
|
||||
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
|
||||
import {
|
||||
getActiveProcessTrace,
|
||||
ProcessTraceCollector,
|
||||
runWithProcessTrace,
|
||||
} from './runtime/trace.js';
|
||||
import { validateOnboardingResponse } from './runtime/validator.js';
|
||||
import { normalizeOnboardingSession } from './session.js';
|
||||
import type { OnboardingSession, PromptContext, StoredOnboardingSession } from './types.js';
|
||||
@@ -71,19 +81,57 @@ export class OnboardingSkill implements Skill {
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
if (getActiveProcessTrace()) {
|
||||
return this.handleWithTrace(request, conversationHistory);
|
||||
}
|
||||
|
||||
const collector = new ProcessTraceCollector();
|
||||
return runWithProcessTrace(collector, () =>
|
||||
this.handleWithTrace(request, conversationHistory),
|
||||
);
|
||||
}
|
||||
|
||||
private async handleWithTrace(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const logger = getRequestLogger();
|
||||
const trace = getActiveProcessTrace();
|
||||
const session = await this.loadSession(request.chatroomId, request.userId);
|
||||
|
||||
if (request.trigger === 'user-message' && !request.userMessage?.trim()) {
|
||||
return this.makeEmptyResponse(request.requestId);
|
||||
}
|
||||
|
||||
const classifyStep = trace?.startStep('classify');
|
||||
const classified = await this.classifier.classify(request.userMessage, request.trigger);
|
||||
classifyStep?.finish({
|
||||
confidence: classified.confidence,
|
||||
intent: classified.intent.kind,
|
||||
source: classified.source,
|
||||
});
|
||||
|
||||
const deriveStep = trace?.startStep('derive');
|
||||
const derived = deriveState(session, request.trigger, classified.intent);
|
||||
deriveStep?.finish({
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
if (!derived.shouldRespond) {
|
||||
return this.makeEmptyResponse(request.requestId);
|
||||
const response = this.makeEmptyResponse(request.requestId);
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse('[NO_RESPONSE]', derived, {
|
||||
allowShortResponse: true,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
let response: AgentResponse;
|
||||
@@ -102,6 +150,15 @@ export class OnboardingSkill implements Skill {
|
||||
}
|
||||
}
|
||||
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
});
|
||||
|
||||
if (derived.intent.kind === 'discord-verify-question') {
|
||||
logger.debug(
|
||||
{
|
||||
@@ -122,6 +179,7 @@ export class OnboardingSkill implements Skill {
|
||||
createdAt: new Date().toISOString(),
|
||||
derived,
|
||||
latencyMs: this.getCurrentLatencyMs(),
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
@@ -185,7 +243,9 @@ export class OnboardingSkill implements Skill {
|
||||
userMessage: this.buildPromptUserMessage(request, derived),
|
||||
};
|
||||
|
||||
return PromptBuilder.forDerived(derived, session, context).build();
|
||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||
const promptVariantId = getPromptVariantId();
|
||||
return promptVariantId ? builder.applyVariant(promptVariantId).build() : builder.build();
|
||||
}
|
||||
|
||||
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
|
||||
|
||||
Reference in new issue
Block a user