Add hybrid memory retrieval and escalation history

This commit is contained in:
dahyeon committed 2026-05-18 23:04:34 -07:00
1 parent 1a86bded0b
commit 211de18476
38 files changed
+1602 -37

No files matched your search

+7 -1
View File
@@ -10,7 +10,11 @@ import { getRequestLogger } from "@obs/context";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader";
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
import { getRagEnabled } from "../../config.js";
import {
getRagEnabled,
getRagHybridEnabled,
getRagRerankerEnabled,
} from "../../config.js";
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
import { createModerationToolRegistry } from "./tools.js";
@@ -150,6 +154,8 @@ async function lookupReferenceText(
try {
const docs = await memoryService.longTerm.search(message, {
category: "rule",
hybrid: getRagHybridEnabled(),
rerank: getRagRerankerEnabled(),
skill: "moderation",
topK: 3,
});
+2
View File
@@ -66,6 +66,8 @@ export const ONBOARDING_PROMPT_DATA = {
"[답변 원칙]",
"- 마지막 메시지에서 직접 물은 내용에만 정확히 답해.",
"- 사용자가 요청하지 않은 다음 단계, 방법, 위치, 예시는 먼저 붙이지 마.",
"- [user_context] 블록이 있으면 그 안의 정보를 응답 톤과 우선순위에 반영해.",
"- [long_term_memory] 블록이 있으면 사용자의 과거 패턴을 참고하되, 사용자에게 직접 언급하지 마.",
"- 마크다운 문법, 링크, URL은 쓰지 마.",
'- 온보딩과 무관한 잡담이거나 다른 사람에게 하는 말이면 "[NO_RESPONSE]"만 출력해.',
],
+24
View File
@@ -144,6 +144,28 @@ function formatRetrievedKnowledge(context: PromptContext): string[] {
];
}
function formatProfileSnippets(context: PromptContext): string[] {
const snippets = context.profileSnippets ?? [];
if (snippets.length === 0) {
return [];
}
return [
'',
'[user_context]',
snippets.map((snippet) => `- ${snippet.block}`).join('\n'),
];
}
function formatLongTermMemory(context: PromptContext): string[] {
const summary = context.userProfile?.longTermSummary?.text?.trim();
if (!summary) {
return [];
}
return ['', '[long_term_memory]', summary];
}
export function personaLayer(): string {
return ONBOARDING_PROMPT_DATA.persona;
}
@@ -222,6 +244,8 @@ export function contextLayer(context: PromptContext): string {
formatUserProfile(context.userProfile),
]
: []),
...formatProfileSnippets(context),
...formatLongTermMemory(context),
...formatRetrievedKnowledge(context),
].join('\n');
}
@@ -3,6 +3,7 @@ import type { GarnetClient } from '@common/garnet';
import { getConversationSummaryKey } from '@conv/conversation-store';
import type { ChatMessage } from '@llm/types';
import type { MemoryService } from '../../../memory/memory-types.js';
import { deriveProfilePromptSnippets } from '../../../memory/profile-update-policy.js';
import { getRequestLogger } from '@obs/context';
import type { RoomContextService } from '@room/room-context-service';
import { meetsTrustTier } from '../../../eval/governance/trust.js';
@@ -10,6 +11,8 @@ import {
getAgentTrustMinTier,
getAgentUserMemoryEnabled,
getRagEnabled,
getRagHybridEnabled,
getRagRerankerEnabled,
getPromptVariantId,
} from '../../../config.js';
import {
@@ -127,6 +130,8 @@ export async function buildLlmMessages(
query: buildRetrievalQuery(request, derived),
options: {
category: 'onboarding-step',
hybrid: getRagHybridEnabled(),
rerank: getRagRerankerEnabled(),
skill: 'onboarding',
topK: 2,
},
@@ -173,6 +178,7 @@ export async function buildLlmMessages(
throw new Error('Failed to load onboarding blackboard');
}
const userProfile = workingMemory?.userProfile ?? legacyUserProfile;
const context: PromptContext = {
blackboard,
conversationHistory:
@@ -180,10 +186,13 @@ export async function buildLlmMessages(
? workingMemory.conversationHistory
: conversationHistory,
roomContext,
profileSnippets: userProfile
? deriveProfilePromptSnippets(userProfile, derived)
: undefined,
retrievedKnowledge: workingMemory?.retrievedKnowledge,
summary: workingMemory?.summary ?? legacySummary,
userMessage: buildPromptUserMessage(request, derived),
userProfile: workingMemory?.userProfile ?? legacyUserProfile,
userProfile,
};
const builder = PromptBuilder.forDerived(derived, session, context);
+36 -1
View File
@@ -2,6 +2,10 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/ty
import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats';
import type { MemoryService } from '../../memory/memory-types.js';
import {
buildLongTermSummary,
shouldRefreshLongTermSummary,
} from '../../memory/long-term-summarizer.js';
import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { RoomContextService } from '@room/room-context-service';
@@ -427,12 +431,43 @@ export class OnboardingSkill implements Skill {
return;
}
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, {
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: [],
});
if (!shouldRefreshLongTermSummary(profile)) {
return;
}
void this.refreshLongTermSummary(request, profile);
}
private async refreshLongTermSummary(
request: AgentRequest,
profile: Awaited<ReturnType<UserMemoryStore['observeTurn']>>,
): Promise<void> {
try {
const conversationSummary =
(await this.memoryService?.session.getSummary(
request.chatroomId,
request.userId,
)) ?? null;
const longTermSummary = await buildLongTermSummary(
this.llmClient,
profile,
conversationSummary,
);
await this.userMemoryStore.updateProfile(request.userId, {
longTermSummary,
});
} catch (error) {
getRequestLogger().warn(
{ err: error },
'[OnboardingSkill] Failed to refresh long-term user summary',
);
}
}
private makeTemplateFallbackResponse(
+51 -4
View File
@@ -4,6 +4,8 @@ import {
anonymizeText,
} from "@eval/anonymize";
import { getAgentUserMemoryTtlDays } from "../../../config.js";
import { evaluateUserFlags } from "../../../memory/forgetting-policy.js";
import type { LongTermSummary } from "../../../memory/long-term-summarizer.js";
import type { LongTermMemory } from "../../../memory/memory-types.js";
import type { DerivedState, Intent } from "./onboarding-state.js";
@@ -11,9 +13,11 @@ export type PreferredStyle = "concise" | "neutral" | "warm";
export interface UserProfile {
completedOnboardingAt?: string | undefined;
archivedFlaggedIssues?: string[] | undefined;
firstSeenAt: string;
flaggedIssues: string[];
lastSeenAt: string;
longTermSummary?: LongTermSummary | undefined;
pastIntents: Array<{ count: number; kind: Intent["kind"] }>;
preferredStyle: PreferredStyle;
totalTurns: number;
@@ -115,14 +119,32 @@ export class UserMemoryStore {
derived.phase.kind === "completed"
? current.completedOnboardingAt ?? now
: current.completedOnboardingAt;
const flagDecision = evaluateUserFlags(
{
...current,
flaggedIssues: sanitizeIssues([
...current.flaggedIssues,
...(options?.flaggedIssues ?? []),
]),
},
10,
);
const next: UserProfile = {
...(current.archivedFlaggedIssues || flagDecision.archived.length > 0
? {
archivedFlaggedIssues: [
...(current.archivedFlaggedIssues ?? []),
...flagDecision.archived,
].slice(-50),
}
: {}),
...(completedOnboardingAt ? { completedOnboardingAt } : {}),
firstSeenAt: current.firstSeenAt,
flaggedIssues: sanitizeIssues([
...current.flaggedIssues,
...(options?.flaggedIssues ?? []),
]),
flaggedIssues: flagDecision.kept,
lastSeenAt: now,
...(current.longTermSummary
? { longTermSummary: current.longTermSummary }
: {}),
pastIntents: mergeIntent(current.pastIntents, derived.intent.kind),
preferredStyle: inferPreferredStyle(
current.preferredStyle,
@@ -140,4 +162,29 @@ export class UserMemoryStore {
}
return next;
}
async updateProfile(
userId: string,
patch: Partial<UserProfile>,
): Promise<UserProfile> {
const current = (await this.load(userId)) ?? emptyProfile(userId);
const next: UserProfile = {
...current,
...patch,
firstSeenAt: patch.firstSeenAt ?? current.firstSeenAt,
flaggedIssues: patch.flaggedIssues ?? current.flaggedIssues,
lastSeenAt: patch.lastSeenAt ?? nowIso(),
pastIntents: patch.pastIntents ?? current.pastIntents,
preferredStyle: patch.preferredStyle ?? current.preferredStyle,
totalTurns: patch.totalTurns ?? current.totalTurns,
userId: current.userId,
};
if (this.longTermMemory) {
await this.longTermMemory.updateUserProfile(userId, next);
} else {
await this.garnetClient.setJson(profileKey(userId), next, ttlSeconds());
}
return next;
}
}
+2
View File
@@ -1,4 +1,5 @@
import type { ConversationMessage } from "@agent/types";
import type { ProfilePromptSnippet } from "../../memory/profile-update-policy.js";
import type { RetrievedDocument } from "../../memory/memory-types.js";
import type { RoomMessage } from "@room/types";
import type { UserProfile } from "./state/user-memory.js";
@@ -27,6 +28,7 @@ export interface Example {
export interface PromptContext {
blackboard: Blackboard;
conversationHistory: ConversationMessage[];
profileSnippets?: ProfilePromptSnippet[] | undefined;
roomContext?: RoomMessage[] | undefined;
retrievedKnowledge?: RetrievedDocument[] | undefined;
summary?: string | undefined;
+2
View File
@@ -2,12 +2,14 @@ import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import type { ShadowEvalBus } from "@eval/shadow-bus";
import type { LlmClient } from "@llm/llm-client";
import type { EscalationHistoryService } from "../memory/escalation-history.js";
import type { MemoryService } from "../memory/memory-types.js";
import type { RoomContextService } from "@room/room-context-service";
import type { Skill } from "./skill.js";
import type { SkillRegistry } from "./skill-loader.js";
export interface SkillDeps {
escalationHistoryService?: EscalationHistoryService | undefined;
garnetClient: GarnetClient;
llmClient: LlmClient;
memoryService?: MemoryService | undefined;
@@ -1,4 +1,5 @@
import type { HandoffPacket } from "@agent/handoff-packet";
import type { UserEscalationHistory } from "../../memory/escalation-history.js";
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
function formatSkill(manifest: SkillManifest): string {
@@ -14,16 +15,35 @@ function formatSkill(manifest: SkillManifest): string {
export function buildSupervisorSystemPrompt(
registry: SkillRegistry,
_packet?: HandoffPacket,
history?: UserEscalationHistory | null,
): string {
const skills = registry
.list()
.filter((manifest) => manifest.name !== "supervisor");
const historyBlock = history
? [
"",
"[user_escalation_history]",
`총 escalation: ${history.totalEscalations}`,
`마지막 escalation: ${history.lastEscalationAt}`,
`top reasons: ${history.topReasons.map((item) => `${item.reason}(${item.count})`).join(", ") || "-"}`,
`recent resolutions: ${
history.escalations
.slice(-3)
.map((item) => item.resolution?.action)
.filter(Boolean)
.join(", ") || "-"
}`,
"- 이 블록은 라우팅 참고용이다. 사용자에게 직접 언급하지 않는다.",
]
: [];
return [
"너는 multi-agent 시스템의 supervisor다.",
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
"",
"[사용 가능한 specialist]",
...skills.map(formatSkill),
...historyBlock,
"",
"[응답 형식]",
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}',
+1
View File
@@ -10,5 +10,6 @@ export function createSupervisorSkill(deps: SkillDeps): Skill {
deps.natsClient,
deps.llmClient,
deps.skillRegistry,
deps.escalationHistoryService,
);
}
+15 -3
View File
@@ -1,6 +1,7 @@
import type { HandoffPacket } from "@agent/handoff-packet";
import type { HandoffReason } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet";
import type { EscalationHistoryService } from "../../memory/escalation-history.js";
import {
getSupervisorResolutionsStreamMaxLen,
getSupervisorReviewQueueMaxLen,
@@ -152,13 +153,13 @@ function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean
export async function appendSupervisorReview(
garnetClient: Pick<GarnetClient, "appendStream">,
fields: Record<string, string>,
): Promise<void> {
await garnetClient.appendStream(
): Promise<string> {
return garnetClient.appendStream(
SUPERVISOR_REVIEW_STREAM_KEY,
{
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
queuedAt: fields["queuedAt"] ?? new Date().toISOString(),
},
{ maxLen: getSupervisorReviewQueueMaxLen() },
);
@@ -275,6 +276,10 @@ export async function resolveSupervisorReview(
garnetClient: SupervisorReviewResolveGarnetClient,
id: string,
resolution: SupervisorReviewResolution,
options?: {
escalationHistoryService?: EscalationHistoryService | undefined;
userId?: string | undefined;
},
): Promise<void> {
const fields: Record<string, string> = {
action: resolution.action,
@@ -297,6 +302,13 @@ export async function resolveSupervisorReview(
},
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
);
if (options?.escalationHistoryService && options.userId) {
await options.escalationHistoryService.recordResolution({
userId: options.userId,
reviewId: id,
resolution,
});
}
}
export async function getResolutionHistory(
+42 -3
View File
@@ -8,6 +8,10 @@ import type { HandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import type { LlmClient } from "@llm/llm-client";
import type {
EscalationHistoryService,
UserEscalationHistory,
} from "../../memory/escalation-history.js";
import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics";
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
@@ -87,6 +91,7 @@ export class SupervisorSkill implements Skill {
natsClient: Pick<NatsClient, "publish">,
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
private readonly skillRegistry?: SkillRegistry | undefined,
private readonly escalationHistoryService?: EscalationHistoryService | undefined,
) {
this.alertService = new AlertService(
garnetClient,
@@ -108,7 +113,8 @@ export class SupervisorSkill implements Skill {
_conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const packet = readHandoffPacket(request);
const llmDecision = await this.routeByLlm(packet);
const escalationHistory = await this.loadEscalationHistory(request.userId);
const llmDecision = await this.routeByLlm(packet, escalationHistory);
const useLlmRoute =
llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
@@ -117,14 +123,26 @@ export class SupervisorSkill implements Skill {
: routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope";
await appendSupervisorReview(this.garnetClient, {
const queuedAt = new Date().toISOString();
const reviewId = await appendSupervisorReview(this.garnetClient, {
chatroomId: request.chatroomId,
packet: JSON.stringify(packet ?? null),
queuedAt,
reason,
requestId: request.requestId,
target: target ?? "human-review",
userId: request.userId,
});
await this.escalationHistoryService?.recordEscalation({
chatroomId: request.chatroomId,
packetSummary: packet?.userGoal ?? packet?.summary ?? "",
queuedAt,
reason,
reviewId,
sourceChain: packet?.sourceChain ?? [],
target: target ?? "human-review",
userId: request.userId,
});
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
await this.alertService.sendAlert(
@@ -154,6 +172,7 @@ export class SupervisorSkill implements Skill {
routedBy: useLlmRoute ? "llm" : "rules",
source: userMessage.source,
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
suggestedTarget: target,
},
requestId: request.requestId,
@@ -164,6 +183,7 @@ export class SupervisorSkill implements Skill {
private async routeByLlm(
packet?: HandoffPacket,
escalationHistory?: UserEscalationHistory | null,
): Promise<SupervisorRoutingDecision | null> {
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
return null;
@@ -173,7 +193,11 @@ export class SupervisorSkill implements Skill {
const result = await this.llmClient.chatCompletion(
[
{
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
content: buildSupervisorSystemPrompt(
this.skillRegistry,
packet,
escalationHistory,
),
role: "system",
},
{
@@ -213,4 +237,19 @@ export class SupervisorSkill implements Skill {
target === "human-review" || Boolean(this.skillRegistry?.get(target))
);
}
private async loadEscalationHistory(userId: string) {
if (!this.escalationHistoryService) {
return null;
}
try {
return await this.escalationHistoryService.load(userId);
} catch (error) {
getLogger().warn(
{ err: error },
"[Supervisor] Failed to load escalation history",
);
return null;
}
}
}