Add hybrid memory retrieval and escalation history
This commit is contained in:
1 parent
1a86bded0b
commit
211de18476
38 files changed
+1602
-37
No files matched your search
@@ -10,7 +10,11 @@ import { getRequestLogger } from "@obs/context";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||
import { getRagEnabled } from "../../config.js";
|
||||
import {
|
||||
getRagEnabled,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
} from "../../config.js";
|
||||
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
||||
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
||||
import { createModerationToolRegistry } from "./tools.js";
|
||||
@@ -150,6 +154,8 @@ async function lookupReferenceText(
|
||||
try {
|
||||
const docs = await memoryService.longTerm.search(message, {
|
||||
category: "rule",
|
||||
hybrid: getRagHybridEnabled(),
|
||||
rerank: getRagRerankerEnabled(),
|
||||
skill: "moderation",
|
||||
topK: 3,
|
||||
});
|
||||
|
||||
@@ -66,6 +66,8 @@ export const ONBOARDING_PROMPT_DATA = {
|
||||
"[답변 원칙]",
|
||||
"- 마지막 메시지에서 직접 물은 내용에만 정확히 답해.",
|
||||
"- 사용자가 요청하지 않은 다음 단계, 방법, 위치, 예시는 먼저 붙이지 마.",
|
||||
"- [user_context] 블록이 있으면 그 안의 정보를 응답 톤과 우선순위에 반영해.",
|
||||
"- [long_term_memory] 블록이 있으면 사용자의 과거 패턴을 참고하되, 사용자에게 직접 언급하지 마.",
|
||||
"- 마크다운 문법, 링크, URL은 쓰지 마.",
|
||||
'- 온보딩과 무관한 잡담이거나 다른 사람에게 하는 말이면 "[NO_RESPONSE]"만 출력해.',
|
||||
],
|
||||
|
||||
@@ -144,6 +144,28 @@ function formatRetrievedKnowledge(context: PromptContext): string[] {
|
||||
];
|
||||
}
|
||||
|
||||
function formatProfileSnippets(context: PromptContext): string[] {
|
||||
const snippets = context.profileSnippets ?? [];
|
||||
if (snippets.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
'',
|
||||
'[user_context]',
|
||||
snippets.map((snippet) => `- ${snippet.block}`).join('\n'),
|
||||
];
|
||||
}
|
||||
|
||||
function formatLongTermMemory(context: PromptContext): string[] {
|
||||
const summary = context.userProfile?.longTermSummary?.text?.trim();
|
||||
if (!summary) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return ['', '[long_term_memory]', summary];
|
||||
}
|
||||
|
||||
export function personaLayer(): string {
|
||||
return ONBOARDING_PROMPT_DATA.persona;
|
||||
}
|
||||
@@ -222,6 +244,8 @@ export function contextLayer(context: PromptContext): string {
|
||||
formatUserProfile(context.userProfile),
|
||||
]
|
||||
: []),
|
||||
...formatProfileSnippets(context),
|
||||
...formatLongTermMemory(context),
|
||||
...formatRetrievedKnowledge(context),
|
||||
].join('\n');
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import type { GarnetClient } from '@common/garnet';
|
||||
import { getConversationSummaryKey } from '@conv/conversation-store';
|
||||
import type { ChatMessage } from '@llm/types';
|
||||
import type { MemoryService } from '../../../memory/memory-types.js';
|
||||
import { deriveProfilePromptSnippets } from '../../../memory/profile-update-policy.js';
|
||||
import { getRequestLogger } from '@obs/context';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
import { meetsTrustTier } from '../../../eval/governance/trust.js';
|
||||
@@ -10,6 +11,8 @@ import {
|
||||
getAgentTrustMinTier,
|
||||
getAgentUserMemoryEnabled,
|
||||
getRagEnabled,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
getPromptVariantId,
|
||||
} from '../../../config.js';
|
||||
import {
|
||||
@@ -127,6 +130,8 @@ export async function buildLlmMessages(
|
||||
query: buildRetrievalQuery(request, derived),
|
||||
options: {
|
||||
category: 'onboarding-step',
|
||||
hybrid: getRagHybridEnabled(),
|
||||
rerank: getRagRerankerEnabled(),
|
||||
skill: 'onboarding',
|
||||
topK: 2,
|
||||
},
|
||||
@@ -173,6 +178,7 @@ export async function buildLlmMessages(
|
||||
throw new Error('Failed to load onboarding blackboard');
|
||||
}
|
||||
|
||||
const userProfile = workingMemory?.userProfile ?? legacyUserProfile;
|
||||
const context: PromptContext = {
|
||||
blackboard,
|
||||
conversationHistory:
|
||||
@@ -180,10 +186,13 @@ export async function buildLlmMessages(
|
||||
? workingMemory.conversationHistory
|
||||
: conversationHistory,
|
||||
roomContext,
|
||||
profileSnippets: userProfile
|
||||
? deriveProfilePromptSnippets(userProfile, derived)
|
||||
: undefined,
|
||||
retrievedKnowledge: workingMemory?.retrievedKnowledge,
|
||||
summary: workingMemory?.summary ?? legacySummary,
|
||||
userMessage: buildPromptUserMessage(request, derived),
|
||||
userProfile: workingMemory?.userProfile ?? legacyUserProfile,
|
||||
userProfile,
|
||||
};
|
||||
|
||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||
|
||||
@@ -2,6 +2,10 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/ty
|
||||
import type { GarnetClient } from '@common/garnet';
|
||||
import type { NatsClient } from '@common/nats';
|
||||
import type { MemoryService } from '../../memory/memory-types.js';
|
||||
import {
|
||||
buildLongTermSummary,
|
||||
shouldRefreshLongTermSummary,
|
||||
} from '../../memory/long-term-summarizer.js';
|
||||
import { getContext, getRequestLogger } from '@obs/context';
|
||||
import type { LlmClient } from '@llm/llm-client';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
@@ -427,12 +431,43 @@ export class OnboardingSkill implements Skill {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
||||
const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
||||
flaggedIssues:
|
||||
derived.phase.kind === 'nickname-rejected'
|
||||
? derived.phase.issues.map((issue) => issue.message)
|
||||
: [],
|
||||
});
|
||||
if (!shouldRefreshLongTermSummary(profile)) {
|
||||
return;
|
||||
}
|
||||
|
||||
void this.refreshLongTermSummary(request, profile);
|
||||
}
|
||||
|
||||
private async refreshLongTermSummary(
|
||||
request: AgentRequest,
|
||||
profile: Awaited<ReturnType<UserMemoryStore['observeTurn']>>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const conversationSummary =
|
||||
(await this.memoryService?.session.getSummary(
|
||||
request.chatroomId,
|
||||
request.userId,
|
||||
)) ?? null;
|
||||
const longTermSummary = await buildLongTermSummary(
|
||||
this.llmClient,
|
||||
profile,
|
||||
conversationSummary,
|
||||
);
|
||||
await this.userMemoryStore.updateProfile(request.userId, {
|
||||
longTermSummary,
|
||||
});
|
||||
} catch (error) {
|
||||
getRequestLogger().warn(
|
||||
{ err: error },
|
||||
'[OnboardingSkill] Failed to refresh long-term user summary',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private makeTemplateFallbackResponse(
|
||||
|
||||
@@ -4,6 +4,8 @@ import {
|
||||
anonymizeText,
|
||||
} from "@eval/anonymize";
|
||||
import { getAgentUserMemoryTtlDays } from "../../../config.js";
|
||||
import { evaluateUserFlags } from "../../../memory/forgetting-policy.js";
|
||||
import type { LongTermSummary } from "../../../memory/long-term-summarizer.js";
|
||||
import type { LongTermMemory } from "../../../memory/memory-types.js";
|
||||
import type { DerivedState, Intent } from "./onboarding-state.js";
|
||||
|
||||
@@ -11,9 +13,11 @@ export type PreferredStyle = "concise" | "neutral" | "warm";
|
||||
|
||||
export interface UserProfile {
|
||||
completedOnboardingAt?: string | undefined;
|
||||
archivedFlaggedIssues?: string[] | undefined;
|
||||
firstSeenAt: string;
|
||||
flaggedIssues: string[];
|
||||
lastSeenAt: string;
|
||||
longTermSummary?: LongTermSummary | undefined;
|
||||
pastIntents: Array<{ count: number; kind: Intent["kind"] }>;
|
||||
preferredStyle: PreferredStyle;
|
||||
totalTurns: number;
|
||||
@@ -115,14 +119,32 @@ export class UserMemoryStore {
|
||||
derived.phase.kind === "completed"
|
||||
? current.completedOnboardingAt ?? now
|
||||
: current.completedOnboardingAt;
|
||||
const flagDecision = evaluateUserFlags(
|
||||
{
|
||||
...current,
|
||||
flaggedIssues: sanitizeIssues([
|
||||
...current.flaggedIssues,
|
||||
...(options?.flaggedIssues ?? []),
|
||||
]),
|
||||
},
|
||||
10,
|
||||
);
|
||||
const next: UserProfile = {
|
||||
...(current.archivedFlaggedIssues || flagDecision.archived.length > 0
|
||||
? {
|
||||
archivedFlaggedIssues: [
|
||||
...(current.archivedFlaggedIssues ?? []),
|
||||
...flagDecision.archived,
|
||||
].slice(-50),
|
||||
}
|
||||
: {}),
|
||||
...(completedOnboardingAt ? { completedOnboardingAt } : {}),
|
||||
firstSeenAt: current.firstSeenAt,
|
||||
flaggedIssues: sanitizeIssues([
|
||||
...current.flaggedIssues,
|
||||
...(options?.flaggedIssues ?? []),
|
||||
]),
|
||||
flaggedIssues: flagDecision.kept,
|
||||
lastSeenAt: now,
|
||||
...(current.longTermSummary
|
||||
? { longTermSummary: current.longTermSummary }
|
||||
: {}),
|
||||
pastIntents: mergeIntent(current.pastIntents, derived.intent.kind),
|
||||
preferredStyle: inferPreferredStyle(
|
||||
current.preferredStyle,
|
||||
@@ -140,4 +162,29 @@ export class UserMemoryStore {
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
async updateProfile(
|
||||
userId: string,
|
||||
patch: Partial<UserProfile>,
|
||||
): Promise<UserProfile> {
|
||||
const current = (await this.load(userId)) ?? emptyProfile(userId);
|
||||
const next: UserProfile = {
|
||||
...current,
|
||||
...patch,
|
||||
firstSeenAt: patch.firstSeenAt ?? current.firstSeenAt,
|
||||
flaggedIssues: patch.flaggedIssues ?? current.flaggedIssues,
|
||||
lastSeenAt: patch.lastSeenAt ?? nowIso(),
|
||||
pastIntents: patch.pastIntents ?? current.pastIntents,
|
||||
preferredStyle: patch.preferredStyle ?? current.preferredStyle,
|
||||
totalTurns: patch.totalTurns ?? current.totalTurns,
|
||||
userId: current.userId,
|
||||
};
|
||||
|
||||
if (this.longTermMemory) {
|
||||
await this.longTermMemory.updateUserProfile(userId, next);
|
||||
} else {
|
||||
await this.garnetClient.setJson(profileKey(userId), next, ttlSeconds());
|
||||
}
|
||||
return next;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { ConversationMessage } from "@agent/types";
|
||||
import type { ProfilePromptSnippet } from "../../memory/profile-update-policy.js";
|
||||
import type { RetrievedDocument } from "../../memory/memory-types.js";
|
||||
import type { RoomMessage } from "@room/types";
|
||||
import type { UserProfile } from "./state/user-memory.js";
|
||||
@@ -27,6 +28,7 @@ export interface Example {
|
||||
export interface PromptContext {
|
||||
blackboard: Blackboard;
|
||||
conversationHistory: ConversationMessage[];
|
||||
profileSnippets?: ProfilePromptSnippet[] | undefined;
|
||||
roomContext?: RoomMessage[] | undefined;
|
||||
retrievedKnowledge?: RetrievedDocument[] | undefined;
|
||||
summary?: string | undefined;
|
||||
|
||||
@@ -2,12 +2,14 @@ import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { ShadowEvalBus } from "@eval/shadow-bus";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { EscalationHistoryService } from "../memory/escalation-history.js";
|
||||
import type { MemoryService } from "../memory/memory-types.js";
|
||||
import type { RoomContextService } from "@room/room-context-service";
|
||||
import type { Skill } from "./skill.js";
|
||||
import type { SkillRegistry } from "./skill-loader.js";
|
||||
|
||||
export interface SkillDeps {
|
||||
escalationHistoryService?: EscalationHistoryService | undefined;
|
||||
garnetClient: GarnetClient;
|
||||
llmClient: LlmClient;
|
||||
memoryService?: MemoryService | undefined;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { UserEscalationHistory } from "../../memory/escalation-history.js";
|
||||
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
|
||||
|
||||
function formatSkill(manifest: SkillManifest): string {
|
||||
@@ -14,16 +15,35 @@ function formatSkill(manifest: SkillManifest): string {
|
||||
export function buildSupervisorSystemPrompt(
|
||||
registry: SkillRegistry,
|
||||
_packet?: HandoffPacket,
|
||||
history?: UserEscalationHistory | null,
|
||||
): string {
|
||||
const skills = registry
|
||||
.list()
|
||||
.filter((manifest) => manifest.name !== "supervisor");
|
||||
const historyBlock = history
|
||||
? [
|
||||
"",
|
||||
"[user_escalation_history]",
|
||||
`총 escalation: ${history.totalEscalations}`,
|
||||
`마지막 escalation: ${history.lastEscalationAt}`,
|
||||
`top reasons: ${history.topReasons.map((item) => `${item.reason}(${item.count})`).join(", ") || "-"}`,
|
||||
`recent resolutions: ${
|
||||
history.escalations
|
||||
.slice(-3)
|
||||
.map((item) => item.resolution?.action)
|
||||
.filter(Boolean)
|
||||
.join(", ") || "-"
|
||||
}`,
|
||||
"- 이 블록은 라우팅 참고용이다. 사용자에게 직접 언급하지 않는다.",
|
||||
]
|
||||
: [];
|
||||
return [
|
||||
"너는 multi-agent 시스템의 supervisor다.",
|
||||
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
|
||||
"",
|
||||
"[사용 가능한 specialist]",
|
||||
...skills.map(formatSkill),
|
||||
...historyBlock,
|
||||
"",
|
||||
"[응답 형식]",
|
||||
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}',
|
||||
|
||||
@@ -10,5 +10,6 @@ export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||
deps.natsClient,
|
||||
deps.llmClient,
|
||||
deps.skillRegistry,
|
||||
deps.escalationHistoryService,
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import type { EscalationHistoryService } from "../../memory/escalation-history.js";
|
||||
import {
|
||||
getSupervisorResolutionsStreamMaxLen,
|
||||
getSupervisorReviewQueueMaxLen,
|
||||
@@ -152,13 +153,13 @@ function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean
|
||||
export async function appendSupervisorReview(
|
||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await garnetClient.appendStream(
|
||||
): Promise<string> {
|
||||
return garnetClient.appendStream(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
{
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
queuedAt: fields["queuedAt"] ?? new Date().toISOString(),
|
||||
},
|
||||
{ maxLen: getSupervisorReviewQueueMaxLen() },
|
||||
);
|
||||
@@ -275,6 +276,10 @@ export async function resolveSupervisorReview(
|
||||
garnetClient: SupervisorReviewResolveGarnetClient,
|
||||
id: string,
|
||||
resolution: SupervisorReviewResolution,
|
||||
options?: {
|
||||
escalationHistoryService?: EscalationHistoryService | undefined;
|
||||
userId?: string | undefined;
|
||||
},
|
||||
): Promise<void> {
|
||||
const fields: Record<string, string> = {
|
||||
action: resolution.action,
|
||||
@@ -297,6 +302,13 @@ export async function resolveSupervisorReview(
|
||||
},
|
||||
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
|
||||
);
|
||||
if (options?.escalationHistoryService && options.userId) {
|
||||
await options.escalationHistoryService.recordResolution({
|
||||
userId: options.userId,
|
||||
reviewId: id,
|
||||
resolution,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function getResolutionHistory(
|
||||
|
||||
@@ -8,6 +8,10 @@ import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type {
|
||||
EscalationHistoryService,
|
||||
UserEscalationHistory,
|
||||
} from "../../memory/escalation-history.js";
|
||||
import { AlertService } from "@obs/alert";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
|
||||
@@ -87,6 +91,7 @@ export class SupervisorSkill implements Skill {
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||
private readonly escalationHistoryService?: EscalationHistoryService | undefined,
|
||||
) {
|
||||
this.alertService = new AlertService(
|
||||
garnetClient,
|
||||
@@ -108,7 +113,8 @@ export class SupervisorSkill implements Skill {
|
||||
_conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const packet = readHandoffPacket(request);
|
||||
const llmDecision = await this.routeByLlm(packet);
|
||||
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
||||
const llmDecision = await this.routeByLlm(packet, escalationHistory);
|
||||
const useLlmRoute =
|
||||
llmDecision !== null &&
|
||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||
@@ -117,14 +123,26 @@ export class SupervisorSkill implements Skill {
|
||||
: routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
const queuedAt = new Date().toISOString();
|
||||
const reviewId = await appendSupervisorReview(this.garnetClient, {
|
||||
chatroomId: request.chatroomId,
|
||||
packet: JSON.stringify(packet ?? null),
|
||||
queuedAt,
|
||||
reason,
|
||||
requestId: request.requestId,
|
||||
target: target ?? "human-review",
|
||||
userId: request.userId,
|
||||
});
|
||||
await this.escalationHistoryService?.recordEscalation({
|
||||
chatroomId: request.chatroomId,
|
||||
packetSummary: packet?.userGoal ?? packet?.summary ?? "",
|
||||
queuedAt,
|
||||
reason,
|
||||
reviewId,
|
||||
sourceChain: packet?.sourceChain ?? [],
|
||||
target: target ?? "human-review",
|
||||
userId: request.userId,
|
||||
});
|
||||
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
|
||||
|
||||
await this.alertService.sendAlert(
|
||||
@@ -154,6 +172,7 @@ export class SupervisorSkill implements Skill {
|
||||
routedBy: useLlmRoute ? "llm" : "rules",
|
||||
source: userMessage.source,
|
||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
@@ -164,6 +183,7 @@ export class SupervisorSkill implements Skill {
|
||||
|
||||
private async routeByLlm(
|
||||
packet?: HandoffPacket,
|
||||
escalationHistory?: UserEscalationHistory | null,
|
||||
): Promise<SupervisorRoutingDecision | null> {
|
||||
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
||||
return null;
|
||||
@@ -173,7 +193,11 @@ export class SupervisorSkill implements Skill {
|
||||
const result = await this.llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
|
||||
content: buildSupervisorSystemPrompt(
|
||||
this.skillRegistry,
|
||||
packet,
|
||||
escalationHistory,
|
||||
),
|
||||
role: "system",
|
||||
},
|
||||
{
|
||||
@@ -213,4 +237,19 @@ export class SupervisorSkill implements Skill {
|
||||
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
||||
);
|
||||
}
|
||||
|
||||
private async loadEscalationHistory(userId: string) {
|
||||
if (!this.escalationHistoryService) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return await this.escalationHistoryService.load(userId);
|
||||
} catch (error) {
|
||||
getLogger().warn(
|
||||
{ err: error },
|
||||
"[Supervisor] Failed to load escalation history",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user