From 211de184767c24a8d1c9b527b64d6991825a4ee4 Mon Sep 17 00:00:00 2001 From: dahyeon Date: Mon, 18 May 2026 23:01:35 -0700 Subject: [PATCH] Add hybrid memory retrieval and escalation history --- package.json | 1 + scripts/memory-eviction-cron.ts | 153 ++++++++++++++++ src/config.test.ts | 25 +++ src/config.ts | 24 +++ src/eval/golden/memory/cases.jsonl | 6 + src/eval/judges/memory-judge.ts | 95 ++++++++++ src/eval/runner.ts | 62 ++++++- src/eval/types.ts | 8 + src/main.ts | 7 + src/memory/bm25.test.ts | 16 ++ src/memory/bm25.ts | 82 +++++++++ src/memory/contradiction-detector.test.ts | 30 +++ src/memory/contradiction-detector.ts | 83 +++++++++ src/memory/escalation-history.ts | 58 ++++++ src/memory/forgetting-policy.test.ts | 32 ++++ src/memory/forgetting-policy.ts | 81 ++++++++ src/memory/long-term-summarizer.ts | 96 ++++++++++ src/memory/memory-service.ts | 4 + src/memory/memory-types.ts | 4 + src/memory/mongo-escalation-history.ts | 173 ++++++++++++++++++ src/memory/mongo-long-term-memory.ts | 130 +++++++++++-- src/memory/profile-update-policy.test.ts | 30 +++ src/memory/profile-update-policy.ts | 62 +++++++ src/memory/reranker.ts | 136 ++++++++++++++ src/skills/moderation/skill.ts | 8 +- src/skills/onboarding/prompt-data.ts | 2 + src/skills/onboarding/prompts/layers.ts | 24 +++ .../onboarding/runtime/message-builder.ts | 11 +- src/skills/onboarding/skill.ts | 37 +++- src/skills/onboarding/state/user-memory.ts | 55 +++++- src/skills/onboarding/types.ts | 2 + src/skills/skill-factory.ts | 2 + src/skills/supervisor/capability-prompt.ts | 20 ++ src/skills/supervisor/index.ts | 1 + src/skills/supervisor/review-queue.ts | 18 +- src/skills/supervisor/skill.ts | 45 ++++- src/tools/moderation/lookup-rule.ts | 8 +- src/tools/onboarding/lookup-community-rule.ts | 8 +- 38 files changed, 1602 insertions(+), 37 deletions(-) create mode 100644 scripts/memory-eviction-cron.ts create mode 100644 src/eval/golden/memory/cases.jsonl create mode 100644 src/eval/judges/memory-judge.ts create mode 100644 src/memory/bm25.test.ts create mode 100644 src/memory/bm25.ts create mode 100644 src/memory/contradiction-detector.test.ts create mode 100644 src/memory/contradiction-detector.ts create mode 100644 src/memory/escalation-history.ts create mode 100644 src/memory/forgetting-policy.test.ts create mode 100644 src/memory/forgetting-policy.ts create mode 100644 src/memory/long-term-summarizer.ts create mode 100644 src/memory/mongo-escalation-history.ts create mode 100644 src/memory/profile-update-policy.test.ts create mode 100644 src/memory/profile-update-policy.ts create mode 100644 src/memory/reranker.ts diff --git a/package.json b/package.json index 56bdcf3..e059581 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "eval:tournament:moderation": "node --import tsx src/eval/cli.ts tournament --skill=moderation", "eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results", "ingest:knowledge": "node --import tsx scripts/ingest-knowledge.ts", + "memory:evict": "node --import tsx scripts/memory-eviction-cron.ts", "review:list": "node --import tsx src/operations/cli.ts review:list", "review:show": "node --import tsx src/operations/cli.ts review:show", "review:resolve": "node --import tsx src/operations/cli.ts review:resolve", diff --git a/scripts/memory-eviction-cron.ts b/scripts/memory-eviction-cron.ts new file mode 100644 index 0000000..dff3dd1 --- /dev/null +++ b/scripts/memory-eviction-cron.ts @@ -0,0 +1,153 @@ +import { loadEnvironmentFromNodeEnv } from "../src/common/env-loader.js"; + +loadEnvironmentFromNodeEnv(); + +const { getRawSecret } = await import("../src/common/secret-manager.js"); +const { initMongoDB, closeMongoDB } = await import("../src/common/mongodb.js"); +const { initLogger, getLogger } = await import("../src/observability/logger.js"); +const { getMemoryEvictionEnabled } = await import("../src/config.js"); +const { KNOWLEDGE_COLLECTION } = await import("../src/memory/mongo-long-term-memory.js"); +const { detectContradictions } = await import("../src/memory/contradiction-detector.js"); +const { evaluateDocument } = await import("../src/memory/forgetting-policy.js"); + +initLogger(); +const logger = getLogger(); + +interface KnowledgeRecord { + _id: string; + category: string; + text: string; + metadata?: Record | undefined; + lastReferencedAt?: Date | undefined; + references?: Date[] | undefined; + updatedAt?: Date | undefined; +} + +function firstNonEmpty( + ...values: Array +): string | undefined { + for (const value of values) { + if (value?.trim()) { + return value.trim(); + } + } + return undefined; +} + +function parseSinceDays(): number { + const raw = process.argv.find((arg) => arg.startsWith("--since="))?.split("=")[1]; + if (!raw) { + return 30; + } + const match = raw.match(/^(\d+)d$/u); + return match ? Number(match[1]) : 30; +} + +const dryRun = process.argv.includes("--dry-run"); +const target = process.argv.find((arg) => arg.startsWith("--target="))?.split("=")[1] ?? "all"; +const sinceDays = parseSinceDays(); + +if (!dryRun && !getMemoryEvictionEnabled()) { + throw new Error( + "AGENT_MEMORY_EVICTION_ENABLED must be true for non-dry-run eviction", + ); +} +if (target !== "knowledge" && target !== "all") { + logger.info({ dryRun, target }, "memory eviction skipped: target has no Mongo knowledge work"); + process.exit(0); +} + +const connectionString = firstNonEmpty( + getRawSecret("MONGODB_CONNECTION_STRING"), + process.env["MONGODB_CONNECTION_STRING"], +); +if (!connectionString) { + if (dryRun) { + logger.warn( + { dryRun }, + "MONGODB_CONNECTION_STRING is not configured; memory eviction dry-run skipped", + ); + process.exit(0); + } + throw new Error("MONGODB_CONNECTION_STRING is required for memory eviction"); +} + +const connection = await initMongoDB(connectionString); + +try { + const collection = connection.collection(KNOWLEDGE_COLLECTION); + const docs = await collection + .find({ "metadata.archived": { $ne: true } }) + .sort({ updatedAt: 1 }) + .toArray(); + const contradictions = detectContradictions( + docs.map((doc) => ({ + id: doc._id, + metadata: doc.metadata, + text: doc.text, + })), + ); + const contradictionByExistingId = new Map( + contradictions.map((candidate) => [candidate.existingId, candidate.newId]), + ); + const now = new Date(); + const sinceMs = now.getTime() - sinceDays * 24 * 60 * 60 * 1000; + let archived = 0; + let kept = 0; + + for (const doc of docs) { + const references = (doc.references ?? []).filter((date) => { + const time = new Date(date).getTime(); + return Number.isFinite(time) && time >= sinceMs; + }); + const decision = evaluateDocument( + { + id: doc._id, + category: doc.category, + metadata: doc.metadata, + text: doc.text, + }, + { + contradictsId: contradictionByExistingId.get(doc._id), + lastReferencedAt: doc.lastReferencedAt ? new Date(doc.lastReferencedAt) : undefined, + now, + referenceCount30d: references.length, + updatedAt: doc.updatedAt ? new Date(doc.updatedAt) : undefined, + }, + ); + + if (decision.action === "keep") { + kept += 1; + continue; + } + + archived += 1; + if (!dryRun) { + await collection.updateOne( + { _id: doc._id }, + { + $set: { + "metadata.archived": true, + "metadata.archivedAt": now.toISOString(), + "metadata.archiveReason": decision.reason, + "metadata.archiveEvidence": decision.evidence, + }, + }, + ); + } + } + + logger.info( + { + archived, + contradictions: contradictions.length, + dryRun, + kept, + scanned: docs.length, + sinceDays, + }, + "memory eviction completed", + ); +} finally { + await closeMongoDB(); +} diff --git a/src/config.test.ts b/src/config.test.ts index a512ab5..43e6d67 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -7,8 +7,13 @@ import { getAlertThresholdP95LatencyMs, getEmbeddingModel, getEmbeddingProvider, + getLongTermSummaryEnabled, + getMemoryEvictionEnabled, getPhoenixLogPrompts, getPhoenixOtlpEndpoint, + getRagHybridEnabled, + getRagRerankerEnabled, + getRagRerankerTopK, getRagEnabled, getSlackAlertWebhookUrl, getSlackWebhookTimeoutMs, @@ -25,6 +30,11 @@ const ENV_KEYS = [ "ALERT_THRESHOLD_P95_LATENCY_MS", "ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING", "AGENT_RAG_ENABLED", + "AGENT_RAG_HYBRID_ENABLED", + "AGENT_RAG_RERANKER_ENABLED", + "AGENT_RAG_RERANKER_TOP_K", + "AGENT_LONG_TERM_SUMMARY_ENABLED", + "AGENT_MEMORY_EVICTION_ENABLED", "EMBEDDING_MODEL", "EMBEDDING_PROVIDER", "PHOENIX_LOG_PROMPTS", @@ -85,14 +95,29 @@ describe("config", () => { it("reads RAG and embedding configuration", () => { expect(getRagEnabled()).toBe(true); + expect(getRagHybridEnabled()).toBe(true); + expect(getRagRerankerEnabled()).toBe(true); + expect(getRagRerankerTopK()).toBe(5); + expect(getLongTermSummaryEnabled()).toBe(true); + expect(getMemoryEvictionEnabled()).toBe(false); expect(getEmbeddingProvider()).toBe("openrouter"); expect(getEmbeddingModel()).toBe("openai/text-embedding-3-small"); vi.stubEnv("AGENT_RAG_ENABLED", "false"); + vi.stubEnv("AGENT_RAG_HYBRID_ENABLED", "false"); + vi.stubEnv("AGENT_RAG_RERANKER_ENABLED", "false"); + vi.stubEnv("AGENT_RAG_RERANKER_TOP_K", "2"); + vi.stubEnv("AGENT_LONG_TERM_SUMMARY_ENABLED", "false"); + vi.stubEnv("AGENT_MEMORY_EVICTION_ENABLED", "true"); vi.stubEnv("EMBEDDING_PROVIDER", "openai"); vi.stubEnv("EMBEDDING_MODEL", "custom-embedding"); expect(getRagEnabled()).toBe(false); + expect(getRagHybridEnabled()).toBe(false); + expect(getRagRerankerEnabled()).toBe(false); + expect(getRagRerankerTopK()).toBe(2); + expect(getLongTermSummaryEnabled()).toBe(false); + expect(getMemoryEvictionEnabled()).toBe(true); expect(getEmbeddingProvider()).toBe("openai"); expect(getEmbeddingModel()).toBe("custom-embedding"); }); diff --git a/src/config.ts b/src/config.ts index 5ac65ea..78c7565 100644 --- a/src/config.ts +++ b/src/config.ts @@ -133,6 +133,30 @@ export function getRagEnabled(): boolean { return getBoolean("AGENT_RAG_ENABLED", true); } +export function getRagHybridEnabled(): boolean { + return getBoolean("AGENT_RAG_HYBRID_ENABLED", true); +} + +export function getRagRerankerEnabled(): boolean { + return getBoolean("AGENT_RAG_RERANKER_ENABLED", true); +} + +export function getRagRerankerTopK(): number { + return getPositiveNumber("AGENT_RAG_RERANKER_TOP_K", 5); +} + +export function getRagCandidatePoolSize(): number { + return getPositiveNumber("AGENT_RAG_CANDIDATE_POOL_SIZE", 50); +} + +export function getLongTermSummaryEnabled(): boolean { + return getBoolean("AGENT_LONG_TERM_SUMMARY_ENABLED", true); +} + +export function getMemoryEvictionEnabled(): boolean { + return getBoolean("AGENT_MEMORY_EVICTION_ENABLED", false); +} + export function getEmbeddingProvider(): "openrouter" | "openai" { const raw = process.env["EMBEDDING_PROVIDER"]?.trim().toLowerCase(); return raw === "openai" ? "openai" : "openrouter"; diff --git a/src/eval/golden/memory/cases.jsonl b/src/eval/golden/memory/cases.jsonl new file mode 100644 index 0000000..217deba --- /dev/null +++ b/src/eval/golden/memory/cases.jsonl @@ -0,0 +1,6 @@ +{"id":"memory-retrieval-accurate","name":"retrieval-accurate","input":{"request":{"requestId":"mem_mod_ads","skill":"moderation","chatroomId":"memory_room","userId":"memory_user_1","nickname":"tester","trigger":"user-message","userMessage":"광고 도배 기준이 뭐예요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["광고","도배"]}} +{"id":"memory-long-range-understanding","name":"long-range-understanding","input":{"request":{"requestId":"mem_onboarding_recall","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_2","nickname":"LolName_KR/95/남/D","trigger":"user-message","userMessage":"내 닉네임 뭐라고 했지?"},"conversationHistory":[{"role":"user","content":"디스코드 닉네임은 LolName_KR입니다","timestamp":"2026-05-01T00:00:00.000Z"},{"role":"assistant","content":"확인했어요.","timestamp":"2026-05-01T00:00:01.000Z"}],"summary":"사용자가 디스코드 닉네임은 LolName_KR이라고 말했다."},"expected":{"mustIncludeAny":["LolName_KR"],"expectedMemory":{"expectedRecalledFact":"LolName_KR"}}} +{"id":"memory-test-time-learning","name":"test-time-learning","input":{"request":{"requestId":"mem_onboarding_hint","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_3","nickname":"낭만/00/남D","trigger":"nickname-invalid"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":["닉네임 형식 오류","닉네임 형식 오류","닉네임 형식 오류"],"pastIntents":[{"kind":"nickname-question","count":5}],"preferredStyle":"concise","totalTurns":12,"userId":"user_memory"}},"expected":{"mustIncludeAny":["예시","형식"],"expectedMemory":{"expectedProphylacticHint":true}}} +{"id":"memory-selective-forgetting","name":"selective-forgetting","input":{"request":{"requestId":"mem_forgetting","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_4","nickname":"tester","trigger":"user-message","userMessage":"닉네임 규칙 최신 기준 알려줘"},"summary":"정정된 최신 규칙만 답해야 한다."},"expected":{"mustNotInclude":["예전 기준"],"expectedMemory":{"expectedArchivedDocPattern":"예전 기준"}}} +{"id":"memory-escalation-history","name":"escalation-history","input":{"request":{"requestId":"mem_supervisor_history","skill":"supervisor","chatroomId":"memory_room","userId":"memory_user_5","nickname":"tester","trigger":"user-message","context":{"handoffPacket":{"constraints":[],"currentState":{},"evidence":[],"expectedOutputSchema":"text","handoffReason":"out-of-scope","sourceChain":["onboarding"],"summary":"광고성 메시지로 보임","userGoal":"광고 홍보 가능 여부 확인"}}}},"expected":{"expectedMemory":{"expectedSupervisorTarget":"moderation"}}} +{"id":"memory-contradiction","name":"memory-contradiction","input":{"request":{"requestId":"mem_contradiction","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_6","nickname":"새닉/95/남/D","trigger":"user-message","userMessage":"닉네임 새닉으로 변경했어요"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":[],"pastIntents":[{"kind":"nickname-question","count":4}],"preferredStyle":"neutral","totalTurns":8,"userId":"user_memory","longTermSummary":{"text":"사용자는 이전 닉네임에서 새닉으로 변경했다고 말했다.","updatedAt":"2026-05-18T00:00:00.000Z","turnSnapshot":8}}},"expected":{"mustIncludeAny":["새닉"],"expectedMemory":{"expectedContradictionResolved":true,"expectedRecalledFact":"새닉"}}} diff --git a/src/eval/judges/memory-judge.ts b/src/eval/judges/memory-judge.ts new file mode 100644 index 0000000..599feca --- /dev/null +++ b/src/eval/judges/memory-judge.ts @@ -0,0 +1,95 @@ +import type { AgentResponse } from "@agent/types"; +import type { RetrievedDocument } from "../../memory/memory-types.js"; +import type { UserProfile } from "@skills/onboarding/state/user-memory"; + +export interface MemoryJudgeExpected { + expectedArchivedDocPattern?: string | undefined; + expectedContradictionResolved?: boolean | undefined; + expectedProphylacticHint?: boolean | undefined; + expectedRecalledFact?: string | undefined; + expectedRetrievedDocIds?: string[] | undefined; + expectedSupervisorTarget?: string | undefined; +} + +export interface MemoryJudgeResult { + checks: Array<{ + detail?: string | undefined; + name: keyof MemoryJudgeExpected; + passed: boolean; + skipped: boolean; + }>; + passed: boolean; + score: number; +} + +export function judgeMemory( + expected: MemoryJudgeExpected | undefined, + response: AgentResponse, + metadata?: { + profile?: UserProfile | undefined; + retrievedDocs?: RetrievedDocument[] | undefined; + }, +): MemoryJudgeResult { + if (!expected || Object.keys(expected).length === 0) { + return { checks: [], passed: true, score: 1 }; + } + + const checks: MemoryJudgeResult["checks"] = []; + if (expected.expectedRecalledFact) { + checks.push({ + name: "expectedRecalledFact", + passed: response.text.includes(expected.expectedRecalledFact), + skipped: false, + }); + } + if (expected.expectedProphylacticHint !== undefined) { + const hasHint = /(예시|형식|가이드|주의|먼저)/u.test(response.text); + checks.push({ + name: "expectedProphylacticHint", + passed: expected.expectedProphylacticHint ? hasHint : !hasHint, + skipped: false, + }); + } + if (expected.expectedSupervisorTarget) { + const suggestedTarget = response.metadata?.["suggestedTarget"]; + checks.push({ + detail: typeof suggestedTarget === "string" ? suggestedTarget : "", + name: "expectedSupervisorTarget", + passed: suggestedTarget === expected.expectedSupervisorTarget, + skipped: false, + }); + } + if (expected.expectedRetrievedDocIds) { + const ids = new Set(metadata?.retrievedDocs?.map((doc) => doc.id) ?? []); + checks.push({ + name: "expectedRetrievedDocIds", + passed: expected.expectedRetrievedDocIds.every((id) => ids.has(id)), + skipped: ids.size === 0, + }); + } + if (expected.expectedArchivedDocPattern) { + checks.push({ + name: "expectedArchivedDocPattern", + passed: !response.text.includes(expected.expectedArchivedDocPattern), + skipped: false, + }); + } + if (expected.expectedContradictionResolved !== undefined) { + checks.push({ + name: "expectedContradictionResolved", + passed: expected.expectedContradictionResolved + ? !/(예전|이전|기존)/u.test(response.text) + : true, + skipped: false, + }); + } + + const activeChecks = checks.filter((check) => !check.skipped); + const passedCount = activeChecks.filter((check) => check.passed).length; + const score = activeChecks.length === 0 ? 1 : passedCount / activeChecks.length; + return { + checks, + passed: score === 1, + score, + }; +} diff --git a/src/eval/runner.ts b/src/eval/runner.ts index e96b172..76eb8f8 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -7,6 +7,7 @@ import { GarnetClient, type StreamEntry } from "@common/garnet"; import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects"; import { NatsClient } from "@common/nats"; import { getConversationSummaryKey } from "@conv/conversation-store"; +import { anonymizeIdentifier } from "@eval/anonymize"; import { LlmClient } from "@llm/llm-client"; import type { ProviderName } from "@llm/types"; import { runWithContext } from "@obs/context"; @@ -34,6 +35,7 @@ import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js import { runEvalDispatch } from "./handoff-dispatch.js"; import { judgeHandoff } from "./judges/handoff-judge.js"; import { judgeSkillWithLlm } from "./judges/llm-judge.js"; +import { judgeMemory } from "./judges/memory-judge.js"; import { judgeProcessRule } from "./judges/process-judge.js"; import { judgeRuleCase } from "./judges/rule-judge.js"; import { getSkillJudgeConfig } from "./judges/skill-config.js"; @@ -50,6 +52,7 @@ import type { EvalSummaryDelta, } from "./types.js"; import { createMemoryService } from "../memory/memory-service.js"; +import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory"; const DEFAULT_REPORTS_DIR = "eval-reports"; const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`; @@ -71,7 +74,7 @@ const SKILL_RUNNER_FACTORIES: Record Skill> = moderation: (deps) => new ModerationSkill( deps.llm, - createMemoryService({ garnetClient: deps.garnet }), + createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }), ), onboarding: (deps) => new OnboardingSkill( @@ -80,7 +83,7 @@ const SKILL_RUNNER_FACTORIES: Record Skill> = deps.room, noopShadowEvalPublisher, undefined, - createMemoryService({ garnetClient: deps.garnet }), + createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }), ), supervisor: (deps) => new SupervisorSkill(deps.garnet, { @@ -451,6 +454,9 @@ function parseExpected(raw: Record | undefined): EvalCaseExpect if (mustNotInclude) { expected.mustNotInclude = mustNotInclude; } + if (isRecord(raw["expectedMemory"])) { + expected.expectedMemory = raw["expectedMemory"] as EvalCaseExpected["expectedMemory"]; + } const processExpectedRaw = raw["processExpected"]; if (isRecord(processExpectedRaw)) { @@ -569,6 +575,10 @@ function parseEvalCaseLine( input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null; } + if (raw.input["userProfile"] === null || isRecord(raw.input["userProfile"])) { + input.userProfile = raw.input["userProfile"] as EvalCaseInput["userProfile"]; + } + return { id: typeof raw.id === "string" && raw.id.trim() @@ -646,6 +656,15 @@ async function hydrateEvalState( input.summary, ); } + + if (input.userProfile !== undefined) { + await garnetClient.setJson( + `${USER_MEMORY_KEY_PREFIX}:${ + anonymizeIdentifier("user", userId) ?? "unknown" + }`, + input.userProfile, + ); + } } function createSummary(results: EvalCaseResult[]): EvalReportSummary { @@ -688,7 +707,11 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary { } function isCasePassed(result: EvalCaseResult): boolean { - return result.ruleJudge.passed && (result.handoffJudge?.passed ?? true); + return ( + result.ruleJudge.passed && + (result.handoffJudge?.passed ?? true) && + (result.memoryJudge?.passed ?? true) + ); } function createBaselineDelta( @@ -943,6 +966,13 @@ async function evaluateOnboardingCase( processTraceSnapshot, judgeConfig.toolRegistryFactory(), ); + const memoryJudge = judgeMemory( + evalCase.expected.expectedMemory, + response, + { + profile: input.userProfile ?? undefined, + }, + ); return { id: evalCase.id, @@ -955,6 +985,7 @@ async function evaluateOnboardingCase( response, ruleJudge, llmJudge, + memoryJudge, processJudge, processTrace: processTraceSnapshot, }; @@ -1028,13 +1059,17 @@ async function evaluateHandoffCase( skills, }); const latencyMs = Date.now() - startedAt; - const handoffJudge = judgeHandoff( - evalCase.expected, - result.response, - result.finalSkill, - ); + const handoffJudge = judgeHandoff( + evalCase.expected, + result.response, + result.finalSkill, + ); + const memoryJudge = judgeMemory( + evalCase.expected.expectedMemory, + result.response, + ); - return { + return { id: evalCase.id, lineNumber: evalCase.lineNumber, name: evalCase.name, @@ -1060,6 +1095,7 @@ async function evaluateHandoffCase( reason: "handoff-eval", }, handoffJudge, + memoryJudge, }; } catch (error) { const latencyMs = Date.now() - startedAt; @@ -1129,6 +1165,13 @@ async function evaluateGenericSkillCase( input.conversationHistory ?? [], ); const latencyMs = Date.now() - startedAt; + const memoryJudge = judgeMemory( + evalCase.expected.expectedMemory, + response, + { + profile: input.userProfile ?? undefined, + }, + ); return { id: evalCase.id, lineNumber: evalCase.lineNumber, @@ -1160,6 +1203,7 @@ async function evaluateGenericSkillCase( skipped: true, reason: "disabled", }, + memoryJudge, }; } catch (error) { const latencyMs = Date.now() - startedAt; diff --git a/src/eval/types.ts b/src/eval/types.ts index 1f774fa..5ecb952 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -5,12 +5,17 @@ import type { HandoffReason, } from "@agent/types"; import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; +import type { UserProfile } from "@skills/onboarding/state/user-memory"; import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state"; import type { ProcessTraceSnapshot, ProcessTraceStepName, } from "@skills/onboarding/runtime/trace"; import type { PrmFeatureVector } from "./judges/prm.js"; +import type { + MemoryJudgeExpected, + MemoryJudgeResult, +} from "./judges/memory-judge.js"; export type EvalProcessTraceSnapshot = ProcessTraceSnapshot; @@ -28,6 +33,7 @@ export interface EvalCaseInput { session?: OnboardingSession | null | undefined; blackboard?: Blackboard | null | undefined; summary?: string | undefined; + userProfile?: UserProfile | null | undefined; } export interface EvalCaseExpected { @@ -37,6 +43,7 @@ export interface EvalCaseExpected { expectedTarget?: string | undefined; shouldHandoff?: boolean | undefined; } | undefined; + expectedMemory?: MemoryJudgeExpected | undefined; expectedSourceChain?: { chain: string[]; mode: "contains" | "exact" | "prefix"; @@ -183,6 +190,7 @@ export interface EvalCaseResult { ruleJudge: RuleJudgeResult; llmJudge: LlmJudgeResult; handoffJudge?: HandoffJudgeResult | undefined; + memoryJudge?: MemoryJudgeResult | undefined; processJudge?: ProcessJudgeResult | undefined; processTrace?: ProcessTraceSnapshot | undefined; error?: string | undefined; diff --git a/src/main.ts b/src/main.ts index ce77289..3907540 100644 --- a/src/main.ts +++ b/src/main.ts @@ -31,6 +31,9 @@ const { LlmClient } = await import("./llm/index.js"); const { getRawSecret, getSecret } = await import("./common/secret-manager.js"); const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js"); const { createMemoryService } = await import("./memory/memory-service.js"); +const { createEscalationHistoryService } = await import( + "./memory/mongo-escalation-history.js" +); const { RoomContextService } = await import("./room-context/index.js"); const { startMetricsServer, stopMetricsServer } = await import( "./observability/metrics-server.js" @@ -226,8 +229,11 @@ async function bootstrap(): Promise { const memoryService = createMemoryService({ garnetClient, + llmClient, mongooseConnection: mongoConnection, }); + const escalationHistoryService = + createEscalationHistoryService(mongoConnection); // Agent setup const conversationStore = new ConversationStore(garnetClient); @@ -265,6 +271,7 @@ async function bootstrap(): Promise { agent.registerSkill( factory({ garnetClient, + escalationHistoryService, llmClient, memoryService, natsClient, diff --git a/src/memory/bm25.test.ts b/src/memory/bm25.test.ts new file mode 100644 index 0000000..7c007bb --- /dev/null +++ b/src/memory/bm25.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { buildBM25Index } from "./bm25.js"; + +describe("buildBM25Index", () => { + it("ranks Korean keyword matches above unrelated documents", () => { + const index = buildBM25Index([ + { id: "ads", text: "광고와 도배 홍보 링크는 제재 대상입니다." }, + { id: "age", text: "가입 가능 연령은 27세 이상입니다." }, + ]); + + const [top] = index.rank("광고 도배 기준", 2); + + expect(top?.id).toBe("ads"); + expect(top?.score).toBeGreaterThan(0); + }); +}); diff --git a/src/memory/bm25.ts b/src/memory/bm25.ts new file mode 100644 index 0000000..1120155 --- /dev/null +++ b/src/memory/bm25.ts @@ -0,0 +1,82 @@ +export interface BM25Document { + id: string; + text: string; +} + +export interface BM25Index { + rank(query: string, topK: number): Array<{ id: string; score: number }>; +} + +const K1 = 1.5; +const B = 0.75; + +function normalize(text: string): string { + return text + .normalize("NFKC") + .toLowerCase() + .replace(/[^\p{Letter}\p{Number}\s]/gu, " "); +} + +function tokenize(text: string): string[] { + const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0); + const compact = normalize(text).replace(/\s+/gu, ""); + const grams: string[] = []; + for (let i = 0; i < compact.length - 1; i += 1) { + grams.push(compact.slice(i, i + 2)); + } + return [...new Set([...words, ...grams])]; +} + +export function buildBM25Index(documents: BM25Document[]): BM25Index { + const termFrequencies = new Map>(); + const documentLengths = new Map(); + const documentFrequency = new Map(); + let totalLength = 0; + + for (const document of documents) { + const tokens = tokenize(document.text); + documentLengths.set(document.id, tokens.length); + totalLength += tokens.length; + + const frequencies = new Map(); + for (const token of tokens) { + frequencies.set(token, (frequencies.get(token) ?? 0) + 1); + } + termFrequencies.set(document.id, frequencies); + + for (const token of frequencies.keys()) { + documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1); + } + } + + const averageLength = documents.length === 0 ? 0 : totalLength / documents.length; + + return { + rank(query: string, topK: number): Array<{ id: string; score: number }> { + const queryTokens = tokenize(query); + if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) { + return []; + } + + return documents + .map((document) => { + const frequencies = termFrequencies.get(document.id) ?? new Map(); + const documentLength = documentLengths.get(document.id) ?? 0; + const score = queryTokens.reduce((sum, token) => { + const tf = frequencies.get(token) ?? 0; + if (tf === 0) { + return sum; + } + const df = documentFrequency.get(token) ?? 0; + const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5)); + const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength)); + return sum + idf * ((tf * (K1 + 1)) / denominator); + }, 0); + return { id: document.id, score }; + }) + .filter((item) => item.score > 0) + .sort((left, right) => right.score - left.score) + .slice(0, Math.max(1, topK)); + }, + }; +} diff --git a/src/memory/contradiction-detector.test.ts b/src/memory/contradiction-detector.test.ts new file mode 100644 index 0000000..bbe902f --- /dev/null +++ b/src/memory/contradiction-detector.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { detectContradictions, textSimilarity } from "./contradiction-detector.js"; + +describe("textSimilarity", () => { + it("returns lower scores for very different text", () => { + expect(textSimilarity("닉네임 규칙", "광고 제재 기준")).toBeLessThan(0.7); + }); +}); + +describe("detectContradictions", () => { + it("flags corrections in the same source section", () => { + const [candidate] = detectContradictions([ + { + id: "old", + metadata: { sourceFile: "rules.md", sectionTitle: "nickname" }, + text: "닉네임은 기존 형식을 사용합니다.", + }, + { + id: "new", + metadata: { sourceFile: "rules.md", sectionTitle: "nickname" }, + text: "정정: 닉네임은 새 형식을 사용합니다.", + }, + ]); + + expect(candidate).toMatchObject({ + existingId: "old", + newId: "new", + }); + }); +}); diff --git a/src/memory/contradiction-detector.ts b/src/memory/contradiction-detector.ts new file mode 100644 index 0000000..dfcec5b --- /dev/null +++ b/src/memory/contradiction-detector.ts @@ -0,0 +1,83 @@ +export interface ContradictionCandidate { + existingId: string; + newId: string; + reason: "keyword" | "text-distance"; + similarity: number; +} + +export interface ContradictionDocument { + id: string; + metadata?: Record | undefined; + text: string; +} + +function groupKey(doc: ContradictionDocument): string | null { + const sourceFile = doc.metadata?.["sourceFile"]; + const sectionTitle = doc.metadata?.["sectionTitle"]; + if (typeof sourceFile !== "string" || typeof sectionTitle !== "string") { + return null; + } + return `${sourceFile}#${sectionTitle}`; +} + +function bigrams(text: string): Set { + const compact = text + .normalize("NFKC") + .toLowerCase() + .replace(/\s+/gu, ""); + const grams = new Set(); + for (let i = 0; i < compact.length - 1; i += 1) { + grams.add(compact.slice(i, i + 2)); + } + return grams; +} + +export function textSimilarity(left: string, right: string): number { + const leftGrams = bigrams(left); + const rightGrams = bigrams(right); + if (leftGrams.size === 0 && rightGrams.size === 0) { + return 1; + } + const intersection = [...leftGrams].filter((gram) => rightGrams.has(gram)).length; + const union = new Set([...leftGrams, ...rightGrams]).size; + return union === 0 ? 0 : intersection / union; +} + +function hasCorrectionKeyword(text: string): boolean { + return /(변경|정정|수정|폐지|삭제|바뀜|업데이트)/u.test(text); +} + +export function detectContradictions( + documents: ContradictionDocument[], +): ContradictionCandidate[] { + const groups = new Map(); + for (const doc of documents) { + const key = groupKey(doc); + if (!key) { + continue; + } + groups.set(key, [...(groups.get(key) ?? []), doc]); + } + + const candidates: ContradictionCandidate[] = []; + for (const docs of groups.values()) { + for (let index = 1; index < docs.length; index += 1) { + const previous = docs[index - 1]; + const current = docs[index]; + if (!previous || !current) { + continue; + } + const similarity = textSimilarity(previous.text, current.text); + if (similarity < 0.7 || hasCorrectionKeyword(current.text)) { + candidates.push({ + existingId: previous.id, + newId: current.id, + reason: hasCorrectionKeyword(current.text) ? "keyword" : "text-distance", + similarity, + }); + } + } + } + + return candidates; +} diff --git a/src/memory/escalation-history.ts b/src/memory/escalation-history.ts new file mode 100644 index 0000000..60bdffc --- /dev/null +++ b/src/memory/escalation-history.ts @@ -0,0 +1,58 @@ +import type { HandoffReason } from "@agent/types"; +import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue"; + +export interface UserEscalationRecord { + queuedAt: string; + reason: HandoffReason; + reviewId: string; + target: string; + resolution?: { + resolvedAt: string; + action: "dismiss" | "manual-reply" | "reroute"; + note?: string | undefined; + } | undefined; + packetSummary: string; + sourceChain: string[]; +} + +export interface UserEscalationHistory { + userId: string; + chatroomId?: string | undefined; + escalations: UserEscalationRecord[]; + totalEscalations: number; + lastEscalationAt: string; + topReasons: Array<{ reason: HandoffReason; count: number }>; +} + +export interface EscalationHistoryService { + load(userId: string): Promise; + recordEscalation(record: { + userId: string; + chatroomId?: string | undefined; + queuedAt: string; + reason: HandoffReason; + reviewId: string; + target: string; + packetSummary: string; + sourceChain: string[]; + }): Promise; + recordResolution(record: { + userId: string; + reviewId: string; + resolution: SupervisorReviewResolution; + }): Promise; +} + +export class NoopEscalationHistoryService implements EscalationHistoryService { + load(): Promise { + return Promise.resolve(null); + } + + recordEscalation(): Promise { + return Promise.resolve(); + } + + recordResolution(): Promise { + return Promise.resolve(); + } +} diff --git a/src/memory/forgetting-policy.test.ts b/src/memory/forgetting-policy.test.ts new file mode 100644 index 0000000..0d914c6 --- /dev/null +++ b/src/memory/forgetting-policy.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { evaluateDocument, evaluateUserFlags } from "./forgetting-policy.js"; +import type { UserProfile } from "@skills/onboarding/state/user-memory"; + +describe("evaluateDocument", () => { + it("archives stale low-reference documents", () => { + const decision = evaluateDocument( + { category: "rule", id: "doc-1", text: "old rule" }, + { + now: new Date("2026-05-18T00:00:00.000Z"), + referenceCount30d: 0, + updatedAt: new Date("2025-12-01T00:00:00.000Z"), + }, + ); + + expect(decision.action).toBe("archive"); + expect(decision.reason).toBe("freshness-stale"); + }); +}); + +describe("evaluateUserFlags", () => { + it("keeps the newest unique flags within the limit", () => { + const profile = { + flaggedIssues: ["a", "b", "c"], + } as UserProfile; + + expect(evaluateUserFlags(profile, 2)).toEqual({ + archived: ["a"], + kept: ["b", "c"], + }); + }); +}); diff --git a/src/memory/forgetting-policy.ts b/src/memory/forgetting-policy.ts new file mode 100644 index 0000000..29afca1 --- /dev/null +++ b/src/memory/forgetting-policy.ts @@ -0,0 +1,81 @@ +import type { UserProfile } from "@skills/onboarding/state/user-memory"; +import type { KnowledgeDocument } from "./memory-types.js"; + +export interface ForgettingDecision { + documentId: string; + action: "archive" | "delete" | "keep"; + reason: "contradiction" | "freshness-stale" | "relevance-low" | "ttl-expired"; + evidence?: string | undefined; +} + +export interface KnowledgeForgettingSignals { + contradictsId?: string | undefined; + lastReferencedAt?: Date | undefined; + now?: Date | undefined; + referenceCount30d: number; + updatedAt?: Date | undefined; +} + +function daysSince(date: Date | undefined, now: Date): number { + if (!date) { + return Number.POSITIVE_INFINITY; + } + return Math.max(0, now.getTime() - date.getTime()) / (24 * 60 * 60 * 1000); +} + +export function evaluateDocument( + doc: KnowledgeDocument, + signals: KnowledgeForgettingSignals, +): ForgettingDecision { + const now = signals.now ?? new Date(); + const documentId = doc.id ?? "unknown"; + + if (signals.contradictsId) { + return { + action: "archive", + documentId, + evidence: `contradicts ${signals.contradictsId}`, + reason: "contradiction", + }; + } + + const staleDays = daysSince(signals.updatedAt, now); + const referenceDays = daysSince(signals.lastReferencedAt, now); + if (staleDays >= 90 && signals.referenceCount30d < 3) { + return { + action: "archive", + documentId, + evidence: `updated ${Math.round(staleDays)}d ago, refs30d=${signals.referenceCount30d}`, + reason: "freshness-stale", + }; + } + + if (referenceDays >= 30 && signals.referenceCount30d < 3) { + return { + action: "archive", + documentId, + evidence: `last referenced ${Math.round(referenceDays)}d ago, refs30d=${signals.referenceCount30d}`, + reason: "relevance-low", + }; + } + + return { + action: "keep", + documentId, + reason: "relevance-low", + }; +} + +export function evaluateUserFlags( + profile: UserProfile, + maxFlags: number, +): { archived: string[]; kept: string[] } { + const unique = [...new Set(profile.flaggedIssues.map((issue) => issue.trim()).filter(Boolean))]; + if (unique.length <= maxFlags) { + return { archived: [], kept: unique }; + } + return { + archived: unique.slice(0, Math.max(0, unique.length - maxFlags)), + kept: unique.slice(-maxFlags), + }; +} diff --git a/src/memory/long-term-summarizer.ts b/src/memory/long-term-summarizer.ts new file mode 100644 index 0000000..225f37c --- /dev/null +++ b/src/memory/long-term-summarizer.ts @@ -0,0 +1,96 @@ +import type { LlmClient } from "@llm/llm-client"; +import { getLongTermSummaryEnabled } from "../config.js"; +import type { UserProfile } from "@skills/onboarding/state/user-memory"; + +export interface LongTermSummary { + text: string; + updatedAt: string; + turnSnapshot: number; +} + +interface SummaryPayload { + text?: unknown; +} + +function daysBetween(leftIso: string, rightIso: string): number { + const left = Date.parse(leftIso); + const right = Date.parse(rightIso); + if (!Number.isFinite(left) || !Number.isFinite(right)) { + return Number.POSITIVE_INFINITY; + } + return Math.abs(right - left) / (24 * 60 * 60 * 1000); +} + +export function shouldRefreshLongTermSummary( + profile: UserProfile, + now = new Date().toISOString(), +): boolean { + if (!getLongTermSummaryEnabled()) { + return false; + } + if (profile.totalTurns < 30) { + return false; + } + const summary = profile.longTermSummary; + if (!summary) { + return true; + } + return ( + profile.totalTurns - summary.turnSnapshot >= 30 || + daysBetween(summary.updatedAt, now) >= 7 + ); +} + +export async function buildLongTermSummary( + llmClient: Pick, + profile: UserProfile, + conversationSummary: string | null, + escalationPattern?: string, +): Promise { + const now = new Date().toISOString(); + const result = await llmClient.chatCompletion( + [ + { + role: "system", + content: [ + "너는 사용자별 장기 메모를 작성한다.", + "운영자가 다음 응답에서 참고할 수 있게 3-5문장으로 요약한다.", + "개인정보나 원문 식별자는 넣지 않는다.", + 'JSON으로만 응답한다: {"text":"요약"}', + ].join("\n"), + }, + { + role: "user", + content: [ + `[totalTurns] ${profile.totalTurns}`, + `[preferredStyle] ${profile.preferredStyle}`, + `[pastIntents] ${profile.pastIntents.map((item) => `${item.kind}:${item.count}`).join(", ") || "-"}`, + `[flaggedIssues] ${profile.flaggedIssues.slice(-10).join(", ") || "-"}`, + `[conversationSummary] ${conversationSummary ?? "-"}`, + `[escalationPattern] ${escalationPattern ?? "-"}`, + ].join("\n"), + }, + ], + { + jsonMode: true, + maxTokens: 300, + temperature: 0.3, + tier: 0, + }, + ); + + if (!result?.content.trim()) { + throw new Error("Long-term summary LLM returned no content"); + } + + const parsed = JSON.parse(result.content) as SummaryPayload; + if (typeof parsed.text !== "string" || parsed.text.trim().length === 0) { + throw new Error("Long-term summary payload missing text"); + } + + return { + text: parsed.text.trim(), + updatedAt: now, + turnSnapshot: profile.totalTurns, + }; +} diff --git a/src/memory/memory-service.ts b/src/memory/memory-service.ts index 14c183f..850e809 100644 --- a/src/memory/memory-service.ts +++ b/src/memory/memory-service.ts @@ -1,5 +1,6 @@ import type mongoose from "mongoose"; import type { GarnetClient } from "@common/garnet"; +import type { LlmClient } from "@llm/llm-client"; import type { Blackboard } from "@skills/onboarding/types"; import { createEmbeddingClient } from "./embedding-client.js"; import { GarnetSessionMemory } from "./garnet-session-memory.js"; @@ -7,6 +8,7 @@ import { MongoLongTermMemory, NoopLongTermMemory, } from "./mongo-long-term-memory.js"; +import { LlmReranker } from "./reranker.js"; import type { LongTermMemory, MemoryService, @@ -17,6 +19,7 @@ import type { interface CreateMemoryServiceOptions { garnetClient: GarnetClient; + llmClient?: Pick | undefined; mongooseConnection?: mongoose.Connection | undefined; } @@ -88,6 +91,7 @@ export function createMemoryService( options.garnetClient, options.mongooseConnection, createEmbeddingClient(), + options.llmClient ? new LlmReranker(options.llmClient) : undefined, ) : new NoopLongTermMemory(options.garnetClient); diff --git a/src/memory/memory-types.ts b/src/memory/memory-types.ts index 54c5095..66cf9d8 100644 --- a/src/memory/memory-types.ts +++ b/src/memory/memory-types.ts @@ -46,6 +46,10 @@ export interface SessionMemory { export interface SearchOptions { skill?: string | undefined; category?: string | undefined; + candidatePoolSize?: number | undefined; + hybrid?: boolean | undefined; + rerank?: boolean | undefined; + rerankTopK?: number | undefined; topK?: number | undefined; } diff --git a/src/memory/mongo-escalation-history.ts b/src/memory/mongo-escalation-history.ts new file mode 100644 index 0000000..1f8380d --- /dev/null +++ b/src/memory/mongo-escalation-history.ts @@ -0,0 +1,173 @@ +import type mongoose from "mongoose"; +import { Schema } from "mongoose"; +import { anonymizeIdentifier } from "@eval/anonymize"; +import type { HandoffReason } from "@agent/types"; +import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue"; +import { + NoopEscalationHistoryService, + type EscalationHistoryService, + type UserEscalationHistory, + type UserEscalationRecord, +} from "./escalation-history.js"; + +export const ESCALATION_HISTORY_COLLECTION = "user_escalation_history"; + +interface EscalationHistoryRecord { + _id: string; + userId: string; + chatroomId?: string | undefined; + escalations: UserEscalationRecord[]; + totalEscalations: number; + lastEscalationAt: string; + topReasons: Array<{ reason: HandoffReason; count: number }>; + updatedAt: Date; +} + +const EscalationHistorySchema = new Schema( + { + _id: { type: String, required: true }, + userId: { type: String, required: true, index: true }, + chatroomId: { type: String, index: true }, + escalations: [ + { + queuedAt: String, + reason: String, + reviewId: String, + target: String, + resolution: { + resolvedAt: String, + action: String, + note: String, + }, + packetSummary: String, + sourceChain: [String], + }, + ], + totalEscalations: { type: Number, default: 0 }, + lastEscalationAt: String, + topReasons: [{ reason: String, count: Number }], + updatedAt: { type: Date, default: Date.now, index: true }, + }, + { versionKey: false }, +); + +function userKey(userId: string): string { + return anonymizeIdentifier("user", userId) ?? "user_unknown"; +} + +function computeTopReasons(records: UserEscalationRecord[]): Array<{ reason: HandoffReason; count: number }> { + const counts = new Map(); + for (const record of records) { + counts.set(record.reason, (counts.get(record.reason) ?? 0) + 1); + } + return [...counts.entries()] + .map(([reason, count]) => ({ reason, count })) + .sort((left, right) => right.count - left.count || left.reason.localeCompare(right.reason)) + .slice(0, 5); +} + +export class MongoEscalationHistoryService implements EscalationHistoryService { + private readonly model: mongoose.Model; + + constructor(connection: mongoose.Connection) { + this.model = + (connection.models["EscalationHistory"] as mongoose.Model | undefined) ?? + connection.model( + "EscalationHistory", + EscalationHistorySchema, + ESCALATION_HISTORY_COLLECTION, + ); + } + + async load(userId: string): Promise { + const doc = await this.model.findById(userKey(userId)).lean().exec(); + if (!doc) { + return null; + } + return { + userId: doc.userId, + ...(doc.chatroomId ? { chatroomId: doc.chatroomId } : {}), + escalations: doc.escalations.slice(-10), + lastEscalationAt: doc.lastEscalationAt, + topReasons: doc.topReasons, + totalEscalations: doc.totalEscalations, + }; + } + + async recordEscalation(record: { + userId: string; + chatroomId?: string | undefined; + queuedAt: string; + reason: HandoffReason; + reviewId: string; + target: string; + packetSummary: string; + sourceChain: string[]; + }): Promise { + const id = userKey(record.userId); + const item: UserEscalationRecord = { + packetSummary: record.packetSummary, + queuedAt: record.queuedAt, + reason: record.reason, + reviewId: record.reviewId, + sourceChain: record.sourceChain, + target: record.target, + }; + await this.model.updateOne( + { _id: id }, + { + $inc: { totalEscalations: 1 }, + $push: { escalations: { $each: [item], $slice: -50 } }, + $set: { + ...(record.chatroomId ? { chatroomId: record.chatroomId } : {}), + lastEscalationAt: record.queuedAt, + updatedAt: new Date(), + userId: id, + }, + $setOnInsert: { _id: id }, + }, + { upsert: true }, + ).exec(); + await this.recomputeTopReasons(id); + } + + async recordResolution(record: { + userId: string; + reviewId: string; + resolution: SupervisorReviewResolution; + }): Promise { + const id = userKey(record.userId); + await this.model.updateOne( + { _id: id, "escalations.reviewId": record.reviewId }, + { + $set: { + "escalations.$.resolution": { + action: record.resolution.action, + resolvedAt: record.resolution.resolvedAt, + ...(record.resolution.note ? { note: record.resolution.note } : {}), + }, + updatedAt: new Date(), + }, + }, + ).exec(); + } + + private async recomputeTopReasons(id: string): Promise { + const doc = await this.model.findById(id).lean().exec(); + if (!doc) { + return; + } + await this.model.updateOne( + { _id: id }, + { $set: { topReasons: computeTopReasons(doc.escalations) } }, + ).exec(); + } +} + +export function createEscalationHistoryService( + connection?: mongoose.Connection, +): EscalationHistoryService { + return connection + ? new MongoEscalationHistoryService(connection) + : new NoopEscalationHistoryService(); +} diff --git a/src/memory/mongo-long-term-memory.ts b/src/memory/mongo-long-term-memory.ts index 77b8d00..d850ddf 100644 --- a/src/memory/mongo-long-term-memory.ts +++ b/src/memory/mongo-long-term-memory.ts @@ -3,11 +3,19 @@ import type mongoose from "mongoose"; import { Schema } from "mongoose"; import { anonymizeIdentifier } from "@eval/anonymize"; import { withSpan } from "@obs/spans"; -import { getAgentUserMemoryTtlDays, getKnowledgeSearchCandidateLimit } from "../config.js"; +import { + getAgentUserMemoryTtlDays, + getKnowledgeSearchCandidateLimit, + getRagCandidatePoolSize, + getRagHybridEnabled, + getRagRerankerEnabled, + getRagRerankerTopK, +} from "../config.js"; import type { UserProfile } from "@skills/onboarding/state/user-memory"; import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory"; import type { GarnetClient } from "@common/garnet"; import { rankByCosineSimilarity } from "./cosine.js"; +import { buildBM25Index } from "./bm25.js"; import type { KnowledgeDocument, LongTermMemory, @@ -15,6 +23,7 @@ import type { SearchOptions, } from "./memory-types.js"; import type { EmbeddingClient } from "./embedding-client.js"; +import type { Reranker } from "./reranker.js"; export const KNOWLEDGE_COLLECTION = "knowledge_documents"; @@ -26,6 +35,8 @@ interface KnowledgeDocumentRecord { embedding: number[]; embeddingModel: string; metadata?: Record | undefined; + lastReferencedAt?: Date | undefined; + references?: Date[] | undefined; createdAt: Date; updatedAt: Date; } @@ -39,6 +50,8 @@ const KnowledgeSchema = new Schema( embedding: { type: [Number], required: true }, embeddingModel: { type: String, required: true }, metadata: { type: Schema.Types.Mixed }, + lastReferencedAt: { type: Date, index: true }, + references: { type: [Date], default: [] }, createdAt: { type: Date, default: Date.now }, updatedAt: { type: Date, default: Date.now, index: true }, }, @@ -83,7 +96,9 @@ function stableDocumentId(doc: KnowledgeDocument): string { } function buildFilter(options: SearchOptions): Record { - const filter: Record = {}; + const filter: Record = { + "metadata.archived": { $ne: true }, + }; if (options.skill?.trim()) { filter["skill"] = options.skill.trim(); } @@ -93,6 +108,34 @@ function buildFilter(options: SearchOptions): Record { return filter; } +function normalizeScores(scores: Map): Map { + const values = [...scores.values()].filter((value) => Number.isFinite(value)); + if (values.length === 0) { + return new Map(); + } + const min = Math.min(...values); + const max = Math.max(...values); + if (max === min) { + return new Map([...scores.keys()].map((id) => [id, max > 0 ? 1 : 0])); + } + return new Map( + [...scores.entries()].map(([id, score]) => [id, (score - min) / (max - min)]), + ); +} + +function toRetrievedDocument( + doc: KnowledgeDocumentRecord & { score: number }, +): RetrievedDocument { + return { + id: doc._id, + category: doc.category, + metadata: doc.metadata, + score: doc.score, + skill: doc.skill, + text: doc.text, + }; +} + export class MongoLongTermMemory implements LongTermMemory { private readonly model: mongoose.Model; @@ -100,6 +143,7 @@ export class MongoLongTermMemory implements LongTermMemory { private readonly garnetClient: Pick, connection: mongoose.Connection, private readonly embeddingClient: EmbeddingClient, + private readonly reranker?: Reranker | undefined, ) { this.model = (connection.models["KnowledgeDocument"] as mongoose.Model | undefined) ?? @@ -162,33 +206,93 @@ export class MongoLongTermMemory implements LongTermMemory { .limit(getKnowledgeSearchCandidateLimit()) .lean() .exec()) as KnowledgeDocumentRecord[]; - const ranked = rankByCosineSimilarity( + const candidatePoolSize = options.candidatePoolSize ?? getRagCandidatePoolSize(); + const topK = options.topK ?? 5; + const shouldHybrid = options.hybrid ?? getRagHybridEnabled(); + const shouldRerank = Boolean( + this.reranker && (options.rerank ?? getRagRerankerEnabled()), + ); + const cosineRanked = rankByCosineSimilarity( queryEmbedding, docs.filter((doc) => Array.isArray(doc.embedding) && doc.embedding.length === queryEmbedding.length ), - options.topK ?? 5, + Math.max(candidatePoolSize, topK), ); + const ranked = shouldHybrid + ? this.rankHybrid(trimmed, docs, cosineRanked, Math.max(candidatePoolSize, topK)) + : cosineRanked.slice(0, Math.max(candidatePoolSize, topK)); + const pool = ranked.slice(0, shouldRerank ? candidatePoolSize : topK); + const results = + shouldRerank && this.reranker + ? await this.reranker.rerank( + trimmed, + pool.map(toRetrievedDocument), + options.rerankTopK ?? getRagRerankerTopK(), + ) + : pool.slice(0, topK).map(toRetrievedDocument); + + await this.markReferenced(results.map((doc) => doc.id)); span.setAttributes({ "memory.retrieval.candidate_count": docs.length, "memory.retrieval.duration_ms": Math.max(0, Date.now() - startedAt), - "memory.retrieval.top_score": ranked[0]?.score, + "memory.retrieval.hybrid": shouldHybrid, + "memory.retrieval.rerank": shouldRerank, + "memory.retrieval.top_score": results[0]?.score, }); - return ranked.map((doc) => ({ - id: doc._id, - category: doc.category, - metadata: doc.metadata, - score: doc.score, - skill: doc.skill, - text: doc.text, - })); + return results.slice(0, topK); }, ); } + private rankHybrid( + query: string, + docs: KnowledgeDocumentRecord[], + cosineRanked: Array, + limit: number, + ): Array { + const cosineScores = normalizeScores( + new Map(cosineRanked.map((doc) => [doc._id, doc.score])), + ); + const bm25Scores = normalizeScores( + new Map( + buildBM25Index(docs.map((doc) => ({ id: doc._id, text: doc.text }))) + .rank(query, docs.length) + .map((item) => [item.id, item.score]), + ), + ); + const byId = new Map(docs.map((doc) => [doc._id, doc])); + + return [...byId.keys()] + .map((id) => ({ + doc: byId.get(id), + score: ((cosineScores.get(id) ?? 0) + (bm25Scores.get(id) ?? 0)) / 2, + })) + .filter((item): item is { doc: KnowledgeDocumentRecord; score: number } => + Boolean(item.doc) && item.score > 0 + ) + .map((item) => ({ ...item.doc, score: item.score })) + .sort((left, right) => right.score - left.score) + .slice(0, limit); + } + + private async markReferenced(ids: string[]): Promise { + if (ids.length === 0) { + return; + } + const now = new Date(); + await this.model.updateMany( + { _id: { $in: ids } }, + { + $set: { lastReferencedAt: now }, + $push: { references: { $each: [now], $slice: -100 } }, + }, + ).exec(); + } + async upsertDocument(doc: KnowledgeDocument): Promise { const id = doc.id?.trim() || stableDocumentId(doc); const embedding = doc.embedding ?? await this.embeddingClient.embed(doc.text); diff --git a/src/memory/profile-update-policy.test.ts b/src/memory/profile-update-policy.test.ts new file mode 100644 index 0000000..d9b4478 --- /dev/null +++ b/src/memory/profile-update-policy.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { deriveProfilePromptSnippets } from "./profile-update-policy.js"; +import type { UserProfile } from "@skills/onboarding/state/user-memory"; +import type { DerivedState } from "@skills/onboarding/state/onboarding-state"; + +const profile: UserProfile = { + firstSeenAt: "2026-05-01T00:00:00.000Z", + flaggedIssues: ["닉네임 형식 오류", "닉네임 형식 오류", "닉네임 형식 오류"], + lastSeenAt: "2026-05-18T00:00:00.000Z", + pastIntents: [{ kind: "nickname-question", count: 5 }], + preferredStyle: "concise", + totalTurns: 10, + userId: "user_x", +}; + +const derived = { + intent: { kind: "nickname-question" }, +} as DerivedState; + +describe("deriveProfilePromptSnippets", () => { + it("derives frequent intent, repeated issue, and preferred style snippets", () => { + const snippets = deriveProfilePromptSnippets(profile, derived); + + expect(snippets.map((snippet) => snippet.reason)).toEqual([ + "frequent-intent", + "repeated-issue", + "preferred-style", + ]); + }); +}); diff --git a/src/memory/profile-update-policy.ts b/src/memory/profile-update-policy.ts new file mode 100644 index 0000000..d550477 --- /dev/null +++ b/src/memory/profile-update-policy.ts @@ -0,0 +1,62 @@ +import type { UserProfile } from "@skills/onboarding/state/user-memory"; +import type { DerivedState } from "@skills/onboarding/state/onboarding-state"; + +export interface ProfilePromptSnippet { + block: string; + reason: "frequent-intent" | "preferred-style" | "repeated-issue"; +} + +function normalizeIssue(issue: string): string { + return issue + .normalize("NFKC") + .replace(/\[[^\]]+\]/gu, "") + .replace(/[0-9]/gu, "") + .replace(/\s+/gu, " ") + .trim() + .slice(0, 32); +} + +export function deriveProfilePromptSnippets( + profile: UserProfile, + derived: DerivedState, +): ProfilePromptSnippet[] { + const snippets: ProfilePromptSnippet[] = []; + const frequentIntent = profile.pastIntents.find( + (intent) => intent.kind === derived.intent.kind && intent.count >= 5, + ); + if (frequentIntent) { + snippets.push({ + reason: "frequent-intent", + block: `사용자는 ${frequentIntent.kind} 유형을 자주 물어봤어요. 핵심 답부터 짧게 말하고, 질문 범위를 넘기지 마세요.`, + }); + } + + const issueCounts = new Map(); + for (const issue of profile.flaggedIssues) { + const normalized = normalizeIssue(issue); + if (normalized) { + issueCounts.set(normalized, (issueCounts.get(normalized) ?? 0) + 1); + } + } + const repeatedIssue = [...issueCounts.entries()].find(([, count]) => count >= 3); + if (repeatedIssue) { + snippets.push({ + reason: "repeated-issue", + block: `반복된 이슈가 있었어요: ${repeatedIssue[0]}. 같은 실수가 이어지지 않도록 정확한 형식 예시를 먼저 확인하세요.`, + }); + } + + if (profile.preferredStyle === "concise") { + snippets.push({ + reason: "preferred-style", + block: "사용자는 짧고 바로 실행 가능한 답변을 선호해요.", + }); + } else if (profile.preferredStyle === "warm") { + snippets.push({ + reason: "preferred-style", + block: "사용자는 따뜻한 톤에 잘 반응했어요. 다만 불필요한 절차 설명은 붙이지 마세요.", + }); + } + + return snippets; +} diff --git a/src/memory/reranker.ts b/src/memory/reranker.ts new file mode 100644 index 0000000..900f220 --- /dev/null +++ b/src/memory/reranker.ts @@ -0,0 +1,136 @@ +import type { LlmClient } from "@llm/llm-client"; +import { getRequestLogger } from "@obs/context"; +import { withSpan } from "@obs/spans"; +import type { RetrievedDocument } from "./memory-types.js"; + +export interface Reranker { + rerank( + query: string, + candidates: RetrievedDocument[], + topN: number, + ): Promise; +} + +interface RankingItem { + id?: unknown; + score?: unknown; +} + +interface RerankerPayload { + rankings?: unknown; +} + +function truncate(text: string, maxLength: number): string { + return text.length <= maxLength ? text : `${text.slice(0, maxLength)}...`; +} + +function parseRankings(raw: string): Array<{ id: string; score: number }> { + const parsed = JSON.parse(raw) as RerankerPayload; + if (!Array.isArray(parsed.rankings)) { + return []; + } + + return parsed.rankings.flatMap((item: RankingItem) => { + if (typeof item.id !== "string") { + return []; + } + const score = + typeof item.score === "number" && Number.isFinite(item.score) + ? Math.max(0, Math.min(1, item.score)) + : 0; + return [{ id: item.id, score }]; + }); +} + +export class LlmReranker implements Reranker { + constructor(private readonly llmClient: Pick) {} + + async rerank( + query: string, + candidates: RetrievedDocument[], + topN: number, + ): Promise { + if (candidates.length <= 1) { + return candidates.slice(0, topN); + } + + return withSpan( + "memory.reranker", + { + "memory.reranker.candidates_count": candidates.length, + "memory.reranker.top_n": topN, + }, + async (span) => { + const startedAt = Date.now(); + try { + const result = await this.llmClient.chatCompletion( + [ + { + role: "system", + content: [ + "너는 검색 결과 reranker다.", + "질문과 후보 문서를 보고 관련도가 높은 순서로만 정렬한다.", + 'JSON으로만 응답한다: {"rankings":[{"id":"문서 id","score":0.0-1.0}]}', + ].join("\n"), + }, + { + role: "user", + content: [ + `[query]\n${query}`, + "", + "[candidates]", + ...candidates.map((candidate, index) => + [ + `#${index + 1} id=${candidate.id} score=${candidate.score.toFixed(3)}`, + truncate(candidate.text, 600), + ].join("\n"), + ), + ].join("\n"), + }, + ], + { + jsonMode: true, + maxTokens: 200, + temperature: 0, + tier: 0, + }, + ); + + if (!result?.content.trim()) { + return candidates.slice(0, topN); + } + + const byId = new Map(candidates.map((candidate) => [candidate.id, candidate])); + const rankings = parseRankings(result.content); + const ranked = rankings + .flatMap((ranking) => { + const candidate = byId.get(ranking.id); + return candidate ? [{ ...candidate, score: ranking.score }] : []; + }) + .slice(0, topN); + const seen = new Set(ranked.map((candidate) => candidate.id)); + const next = [ + ...ranked, + ...candidates.filter((candidate) => !seen.has(candidate.id)), + ].slice(0, topN); + + span.setAttributes({ + "memory.reranker.changed_order": next[0]?.id !== candidates[0]?.id, + "memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt), + }); + return next; + } catch (error) { + getRequestLogger().warn( + { err: error }, + "[Memory] Reranker failed; using original retrieval order", + ); + span.setAttributes({ + "memory.reranker.changed_order": false, + "memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt), + }); + return candidates.slice(0, topN); + } + }, + ); + } +} diff --git a/src/skills/moderation/skill.ts b/src/skills/moderation/skill.ts index dae90af..1f99365 100644 --- a/src/skills/moderation/skill.ts +++ b/src/skills/moderation/skill.ts @@ -10,7 +10,11 @@ import { getRequestLogger } from "@obs/context"; import type { Skill } from "@skills/skill"; import type { SkillCapabilityCard } from "@skills/skill-loader"; import { lookupRuleTool } from "@tools/moderation/lookup-rule"; -import { getRagEnabled } from "../../config.js"; +import { + getRagEnabled, + getRagHybridEnabled, + getRagRerankerEnabled, +} from "../../config.js"; import { classifyModerationIntent, type ModerationIntent } from "./intents.js"; import { MODERATION_PROMPT_DATA } from "./prompt-data.js"; import { createModerationToolRegistry } from "./tools.js"; @@ -150,6 +154,8 @@ async function lookupReferenceText( try { const docs = await memoryService.longTerm.search(message, { category: "rule", + hybrid: getRagHybridEnabled(), + rerank: getRagRerankerEnabled(), skill: "moderation", topK: 3, }); diff --git a/src/skills/onboarding/prompt-data.ts b/src/skills/onboarding/prompt-data.ts index 892e2cd..4c779c5 100644 --- a/src/skills/onboarding/prompt-data.ts +++ b/src/skills/onboarding/prompt-data.ts @@ -66,6 +66,8 @@ export const ONBOARDING_PROMPT_DATA = { "[답변 원칙]", "- 마지막 메시지에서 직접 물은 내용에만 정확히 답해.", "- 사용자가 요청하지 않은 다음 단계, 방법, 위치, 예시는 먼저 붙이지 마.", + "- [user_context] 블록이 있으면 그 안의 정보를 응답 톤과 우선순위에 반영해.", + "- [long_term_memory] 블록이 있으면 사용자의 과거 패턴을 참고하되, 사용자에게 직접 언급하지 마.", "- 마크다운 문법, 링크, URL은 쓰지 마.", '- 온보딩과 무관한 잡담이거나 다른 사람에게 하는 말이면 "[NO_RESPONSE]"만 출력해.', ], diff --git a/src/skills/onboarding/prompts/layers.ts b/src/skills/onboarding/prompts/layers.ts index c53f7a2..c8dcec8 100644 --- a/src/skills/onboarding/prompts/layers.ts +++ b/src/skills/onboarding/prompts/layers.ts @@ -144,6 +144,28 @@ function formatRetrievedKnowledge(context: PromptContext): string[] { ]; } +function formatProfileSnippets(context: PromptContext): string[] { + const snippets = context.profileSnippets ?? []; + if (snippets.length === 0) { + return []; + } + + return [ + '', + '[user_context]', + snippets.map((snippet) => `- ${snippet.block}`).join('\n'), + ]; +} + +function formatLongTermMemory(context: PromptContext): string[] { + const summary = context.userProfile?.longTermSummary?.text?.trim(); + if (!summary) { + return []; + } + + return ['', '[long_term_memory]', summary]; +} + export function personaLayer(): string { return ONBOARDING_PROMPT_DATA.persona; } @@ -222,6 +244,8 @@ export function contextLayer(context: PromptContext): string { formatUserProfile(context.userProfile), ] : []), + ...formatProfileSnippets(context), + ...formatLongTermMemory(context), ...formatRetrievedKnowledge(context), ].join('\n'); } diff --git a/src/skills/onboarding/runtime/message-builder.ts b/src/skills/onboarding/runtime/message-builder.ts index 7f3bb40..a6456fb 100644 --- a/src/skills/onboarding/runtime/message-builder.ts +++ b/src/skills/onboarding/runtime/message-builder.ts @@ -3,6 +3,7 @@ import type { GarnetClient } from '@common/garnet'; import { getConversationSummaryKey } from '@conv/conversation-store'; import type { ChatMessage } from '@llm/types'; import type { MemoryService } from '../../../memory/memory-types.js'; +import { deriveProfilePromptSnippets } from '../../../memory/profile-update-policy.js'; import { getRequestLogger } from '@obs/context'; import type { RoomContextService } from '@room/room-context-service'; import { meetsTrustTier } from '../../../eval/governance/trust.js'; @@ -10,6 +11,8 @@ import { getAgentTrustMinTier, getAgentUserMemoryEnabled, getRagEnabled, + getRagHybridEnabled, + getRagRerankerEnabled, getPromptVariantId, } from '../../../config.js'; import { @@ -127,6 +130,8 @@ export async function buildLlmMessages( query: buildRetrievalQuery(request, derived), options: { category: 'onboarding-step', + hybrid: getRagHybridEnabled(), + rerank: getRagRerankerEnabled(), skill: 'onboarding', topK: 2, }, @@ -173,6 +178,7 @@ export async function buildLlmMessages( throw new Error('Failed to load onboarding blackboard'); } + const userProfile = workingMemory?.userProfile ?? legacyUserProfile; const context: PromptContext = { blackboard, conversationHistory: @@ -180,10 +186,13 @@ export async function buildLlmMessages( ? workingMemory.conversationHistory : conversationHistory, roomContext, + profileSnippets: userProfile + ? deriveProfilePromptSnippets(userProfile, derived) + : undefined, retrievedKnowledge: workingMemory?.retrievedKnowledge, summary: workingMemory?.summary ?? legacySummary, userMessage: buildPromptUserMessage(request, derived), - userProfile: workingMemory?.userProfile ?? legacyUserProfile, + userProfile, }; const builder = PromptBuilder.forDerived(derived, session, context); diff --git a/src/skills/onboarding/skill.ts b/src/skills/onboarding/skill.ts index f7de0b6..d20c6bc 100644 --- a/src/skills/onboarding/skill.ts +++ b/src/skills/onboarding/skill.ts @@ -2,6 +2,10 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/ty import type { GarnetClient } from '@common/garnet'; import type { NatsClient } from '@common/nats'; import type { MemoryService } from '../../memory/memory-types.js'; +import { + buildLongTermSummary, + shouldRefreshLongTermSummary, +} from '../../memory/long-term-summarizer.js'; import { getContext, getRequestLogger } from '@obs/context'; import type { LlmClient } from '@llm/llm-client'; import type { RoomContextService } from '@room/room-context-service'; @@ -427,12 +431,43 @@ export class OnboardingSkill implements Skill { return; } - await this.userMemoryStore.observeTurn(request.userId, derived, response, { + const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, { flaggedIssues: derived.phase.kind === 'nickname-rejected' ? derived.phase.issues.map((issue) => issue.message) : [], }); + if (!shouldRefreshLongTermSummary(profile)) { + return; + } + + void this.refreshLongTermSummary(request, profile); + } + + private async refreshLongTermSummary( + request: AgentRequest, + profile: Awaited>, + ): Promise { + try { + const conversationSummary = + (await this.memoryService?.session.getSummary( + request.chatroomId, + request.userId, + )) ?? null; + const longTermSummary = await buildLongTermSummary( + this.llmClient, + profile, + conversationSummary, + ); + await this.userMemoryStore.updateProfile(request.userId, { + longTermSummary, + }); + } catch (error) { + getRequestLogger().warn( + { err: error }, + '[OnboardingSkill] Failed to refresh long-term user summary', + ); + } } private makeTemplateFallbackResponse( diff --git a/src/skills/onboarding/state/user-memory.ts b/src/skills/onboarding/state/user-memory.ts index 8681d7b..04c1c71 100644 --- a/src/skills/onboarding/state/user-memory.ts +++ b/src/skills/onboarding/state/user-memory.ts @@ -4,6 +4,8 @@ import { anonymizeText, } from "@eval/anonymize"; import { getAgentUserMemoryTtlDays } from "../../../config.js"; +import { evaluateUserFlags } from "../../../memory/forgetting-policy.js"; +import type { LongTermSummary } from "../../../memory/long-term-summarizer.js"; import type { LongTermMemory } from "../../../memory/memory-types.js"; import type { DerivedState, Intent } from "./onboarding-state.js"; @@ -11,9 +13,11 @@ export type PreferredStyle = "concise" | "neutral" | "warm"; export interface UserProfile { completedOnboardingAt?: string | undefined; + archivedFlaggedIssues?: string[] | undefined; firstSeenAt: string; flaggedIssues: string[]; lastSeenAt: string; + longTermSummary?: LongTermSummary | undefined; pastIntents: Array<{ count: number; kind: Intent["kind"] }>; preferredStyle: PreferredStyle; totalTurns: number; @@ -115,14 +119,32 @@ export class UserMemoryStore { derived.phase.kind === "completed" ? current.completedOnboardingAt ?? now : current.completedOnboardingAt; + const flagDecision = evaluateUserFlags( + { + ...current, + flaggedIssues: sanitizeIssues([ + ...current.flaggedIssues, + ...(options?.flaggedIssues ?? []), + ]), + }, + 10, + ); const next: UserProfile = { + ...(current.archivedFlaggedIssues || flagDecision.archived.length > 0 + ? { + archivedFlaggedIssues: [ + ...(current.archivedFlaggedIssues ?? []), + ...flagDecision.archived, + ].slice(-50), + } + : {}), ...(completedOnboardingAt ? { completedOnboardingAt } : {}), firstSeenAt: current.firstSeenAt, - flaggedIssues: sanitizeIssues([ - ...current.flaggedIssues, - ...(options?.flaggedIssues ?? []), - ]), + flaggedIssues: flagDecision.kept, lastSeenAt: now, + ...(current.longTermSummary + ? { longTermSummary: current.longTermSummary } + : {}), pastIntents: mergeIntent(current.pastIntents, derived.intent.kind), preferredStyle: inferPreferredStyle( current.preferredStyle, @@ -140,4 +162,29 @@ export class UserMemoryStore { } return next; } + + async updateProfile( + userId: string, + patch: Partial, + ): Promise { + const current = (await this.load(userId)) ?? emptyProfile(userId); + const next: UserProfile = { + ...current, + ...patch, + firstSeenAt: patch.firstSeenAt ?? current.firstSeenAt, + flaggedIssues: patch.flaggedIssues ?? current.flaggedIssues, + lastSeenAt: patch.lastSeenAt ?? nowIso(), + pastIntents: patch.pastIntents ?? current.pastIntents, + preferredStyle: patch.preferredStyle ?? current.preferredStyle, + totalTurns: patch.totalTurns ?? current.totalTurns, + userId: current.userId, + }; + + if (this.longTermMemory) { + await this.longTermMemory.updateUserProfile(userId, next); + } else { + await this.garnetClient.setJson(profileKey(userId), next, ttlSeconds()); + } + return next; + } } diff --git a/src/skills/onboarding/types.ts b/src/skills/onboarding/types.ts index c5dc47c..5807403 100644 --- a/src/skills/onboarding/types.ts +++ b/src/skills/onboarding/types.ts @@ -1,4 +1,5 @@ import type { ConversationMessage } from "@agent/types"; +import type { ProfilePromptSnippet } from "../../memory/profile-update-policy.js"; import type { RetrievedDocument } from "../../memory/memory-types.js"; import type { RoomMessage } from "@room/types"; import type { UserProfile } from "./state/user-memory.js"; @@ -27,6 +28,7 @@ export interface Example { export interface PromptContext { blackboard: Blackboard; conversationHistory: ConversationMessage[]; + profileSnippets?: ProfilePromptSnippet[] | undefined; roomContext?: RoomMessage[] | undefined; retrievedKnowledge?: RetrievedDocument[] | undefined; summary?: string | undefined; diff --git a/src/skills/skill-factory.ts b/src/skills/skill-factory.ts index b567ebc..a04d7d2 100644 --- a/src/skills/skill-factory.ts +++ b/src/skills/skill-factory.ts @@ -2,12 +2,14 @@ import type { GarnetClient } from "@common/garnet"; import type { NatsClient } from "@common/nats"; import type { ShadowEvalBus } from "@eval/shadow-bus"; import type { LlmClient } from "@llm/llm-client"; +import type { EscalationHistoryService } from "../memory/escalation-history.js"; import type { MemoryService } from "../memory/memory-types.js"; import type { RoomContextService } from "@room/room-context-service"; import type { Skill } from "./skill.js"; import type { SkillRegistry } from "./skill-loader.js"; export interface SkillDeps { + escalationHistoryService?: EscalationHistoryService | undefined; garnetClient: GarnetClient; llmClient: LlmClient; memoryService?: MemoryService | undefined; diff --git a/src/skills/supervisor/capability-prompt.ts b/src/skills/supervisor/capability-prompt.ts index fa2029b..a020192 100644 --- a/src/skills/supervisor/capability-prompt.ts +++ b/src/skills/supervisor/capability-prompt.ts @@ -1,4 +1,5 @@ import type { HandoffPacket } from "@agent/handoff-packet"; +import type { UserEscalationHistory } from "../../memory/escalation-history.js"; import type { SkillManifest, SkillRegistry } from "@skills/skill-loader"; function formatSkill(manifest: SkillManifest): string { @@ -14,16 +15,35 @@ function formatSkill(manifest: SkillManifest): string { export function buildSupervisorSystemPrompt( registry: SkillRegistry, _packet?: HandoffPacket, + history?: UserEscalationHistory | null, ): string { const skills = registry .list() .filter((manifest) => manifest.name !== "supervisor"); + const historyBlock = history + ? [ + "", + "[user_escalation_history]", + `총 escalation: ${history.totalEscalations}`, + `마지막 escalation: ${history.lastEscalationAt}`, + `top reasons: ${history.topReasons.map((item) => `${item.reason}(${item.count})`).join(", ") || "-"}`, + `recent resolutions: ${ + history.escalations + .slice(-3) + .map((item) => item.resolution?.action) + .filter(Boolean) + .join(", ") || "-" + }`, + "- 이 블록은 라우팅 참고용이다. 사용자에게 직접 언급하지 않는다.", + ] + : []; return [ "너는 multi-agent 시스템의 supervisor다.", "사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.", "", "[사용 가능한 specialist]", ...skills.map(formatSkill), + ...historyBlock, "", "[응답 형식]", 'JSON으로만 응답: {"target":"","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}', diff --git a/src/skills/supervisor/index.ts b/src/skills/supervisor/index.ts index 84703a3..f0168db 100644 --- a/src/skills/supervisor/index.ts +++ b/src/skills/supervisor/index.ts @@ -10,5 +10,6 @@ export function createSupervisorSkill(deps: SkillDeps): Skill { deps.natsClient, deps.llmClient, deps.skillRegistry, + deps.escalationHistoryService, ); } diff --git a/src/skills/supervisor/review-queue.ts b/src/skills/supervisor/review-queue.ts index aa72263..37433aa 100644 --- a/src/skills/supervisor/review-queue.ts +++ b/src/skills/supervisor/review-queue.ts @@ -1,6 +1,7 @@ import type { HandoffPacket } from "@agent/handoff-packet"; import type { HandoffReason } from "@agent/types"; import type { GarnetClient, StreamEntry } from "@common/garnet"; +import type { EscalationHistoryService } from "../../memory/escalation-history.js"; import { getSupervisorResolutionsStreamMaxLen, getSupervisorReviewQueueMaxLen, @@ -152,13 +153,13 @@ function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean export async function appendSupervisorReview( garnetClient: Pick, fields: Record, -): Promise { - await garnetClient.appendStream( +): Promise { + return garnetClient.appendStream( SUPERVISOR_REVIEW_STREAM_KEY, { ...fields, type: "supervisor-handoff", - queuedAt: new Date().toISOString(), + queuedAt: fields["queuedAt"] ?? new Date().toISOString(), }, { maxLen: getSupervisorReviewQueueMaxLen() }, ); @@ -275,6 +276,10 @@ export async function resolveSupervisorReview( garnetClient: SupervisorReviewResolveGarnetClient, id: string, resolution: SupervisorReviewResolution, + options?: { + escalationHistoryService?: EscalationHistoryService | undefined; + userId?: string | undefined; + }, ): Promise { const fields: Record = { action: resolution.action, @@ -297,6 +302,13 @@ export async function resolveSupervisorReview( }, { maxLen: getSupervisorResolutionsStreamMaxLen() }, ); + if (options?.escalationHistoryService && options.userId) { + await options.escalationHistoryService.recordResolution({ + userId: options.userId, + reviewId: id, + resolution, + }); + } } export async function getResolutionHistory( diff --git a/src/skills/supervisor/skill.ts b/src/skills/supervisor/skill.ts index 2fe6b21..55c57a2 100644 --- a/src/skills/supervisor/skill.ts +++ b/src/skills/supervisor/skill.ts @@ -8,6 +8,10 @@ import type { HandoffPacket } from "@agent/handoff-packet"; import type { GarnetClient } from "@common/garnet"; import type { NatsClient } from "@common/nats"; import type { LlmClient } from "@llm/llm-client"; +import type { + EscalationHistoryService, + UserEscalationHistory, +} from "../../memory/escalation-history.js"; import { AlertService } from "@obs/alert"; import { metrics } from "@obs/metrics"; import { createSlackNotifierFromEnv } from "@obs/slack-notifier"; @@ -87,6 +91,7 @@ export class SupervisorSkill implements Skill { natsClient: Pick, private readonly llmClient?: Pick | undefined, private readonly skillRegistry?: SkillRegistry | undefined, + private readonly escalationHistoryService?: EscalationHistoryService | undefined, ) { this.alertService = new AlertService( garnetClient, @@ -108,7 +113,8 @@ export class SupervisorSkill implements Skill { _conversationHistory: ConversationMessage[], ): Promise { const packet = readHandoffPacket(request); - const llmDecision = await this.routeByLlm(packet); + const escalationHistory = await this.loadEscalationHistory(request.userId); + const llmDecision = await this.routeByLlm(packet, escalationHistory); const useLlmRoute = llmDecision !== null && llmDecision.confidence >= getSupervisorLlmConfidenceThreshold(); @@ -117,14 +123,26 @@ export class SupervisorSkill implements Skill { : routeSupervisorTarget(packet); const reason = packet?.handoffReason ?? "out-of-scope"; - await appendSupervisorReview(this.garnetClient, { + const queuedAt = new Date().toISOString(); + const reviewId = await appendSupervisorReview(this.garnetClient, { chatroomId: request.chatroomId, packet: JSON.stringify(packet ?? null), + queuedAt, reason, requestId: request.requestId, target: target ?? "human-review", userId: request.userId, }); + await this.escalationHistoryService?.recordEscalation({ + chatroomId: request.chatroomId, + packetSummary: packet?.userGoal ?? packet?.summary ?? "", + queuedAt, + reason, + reviewId, + sourceChain: packet?.sourceChain ?? [], + target: target ?? "human-review", + userId: request.userId, + }); metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name }); await this.alertService.sendAlert( @@ -154,6 +172,7 @@ export class SupervisorSkill implements Skill { routedBy: useLlmRoute ? "llm" : "rules", source: userMessage.source, supervisorReason: useLlmRoute ? llmDecision.reason : reason, + supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0, suggestedTarget: target, }, requestId: request.requestId, @@ -164,6 +183,7 @@ export class SupervisorSkill implements Skill { private async routeByLlm( packet?: HandoffPacket, + escalationHistory?: UserEscalationHistory | null, ): Promise { if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) { return null; @@ -173,7 +193,11 @@ export class SupervisorSkill implements Skill { const result = await this.llmClient.chatCompletion( [ { - content: buildSupervisorSystemPrompt(this.skillRegistry, packet), + content: buildSupervisorSystemPrompt( + this.skillRegistry, + packet, + escalationHistory, + ), role: "system", }, { @@ -213,4 +237,19 @@ export class SupervisorSkill implements Skill { target === "human-review" || Boolean(this.skillRegistry?.get(target)) ); } + + private async loadEscalationHistory(userId: string) { + if (!this.escalationHistoryService) { + return null; + } + try { + return await this.escalationHistoryService.load(userId); + } catch (error) { + getLogger().warn( + { err: error }, + "[Supervisor] Failed to load escalation history", + ); + return null; + } + } } diff --git a/src/tools/moderation/lookup-rule.ts b/src/tools/moderation/lookup-rule.ts index 0c3a6c6..4976f0a 100644 --- a/src/tools/moderation/lookup-rule.ts +++ b/src/tools/moderation/lookup-rule.ts @@ -1,7 +1,11 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { z } from "zod"; -import { getRagEnabled } from "../../config.js"; +import { + getRagEnabled, + getRagHybridEnabled, + getRagRerankerEnabled, +} from "../../config.js"; import { getRequestLogger } from "../../observability/context.js"; import type { Tool } from "../tool.js"; @@ -96,6 +100,8 @@ export const lookupRuleTool: Tool< try { const docs = await ctx.memoryService.longTerm.search(args.topic, { category: "rule", + hybrid: getRagHybridEnabled(), + rerank: getRagRerankerEnabled(), skill: "moderation", topK: args.topK ?? 3, }); diff --git a/src/tools/onboarding/lookup-community-rule.ts b/src/tools/onboarding/lookup-community-rule.ts index b7bdccb..282757d 100644 --- a/src/tools/onboarding/lookup-community-rule.ts +++ b/src/tools/onboarding/lookup-community-rule.ts @@ -1,5 +1,9 @@ import { z } from "zod"; -import { getRagEnabled } from "../../config.js"; +import { + getRagEnabled, + getRagHybridEnabled, + getRagRerankerEnabled, +} from "../../config.js"; import { getRequestLogger } from "../../observability/context.js"; import { DEADLINE_NEAR_THRESHOLD_MIN, @@ -93,6 +97,8 @@ export const lookupCommunityRuleTool: Tool< try { const docs = await ctx.memoryService.longTerm.search(args.topic, { category: "onboarding-step", + hybrid: getRagHybridEnabled(), + rerank: getRagRerankerEnabled(), skill: "onboarding", topK: args.topK ?? 3, });