Add hybrid memory retrieval and escalation history

This commit is contained in:
dahyeon committed 2026-05-18 23:04:34 -07:00
1 parent 1a86bded0b
commit 211de18476
38 files changed
+1595 -30

No files matched your search

+1
View File
@@ -31,6 +31,7 @@
"eval:tournament:moderation": "node --import tsx src/eval/cli.ts tournament --skill=moderation",
"eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results",
"ingest:knowledge": "node --import tsx scripts/ingest-knowledge.ts",
"memory:evict": "node --import tsx scripts/memory-eviction-cron.ts",
"review:list": "node --import tsx src/operations/cli.ts review:list",
"review:show": "node --import tsx src/operations/cli.ts review:show",
"review:resolve": "node --import tsx src/operations/cli.ts review:resolve",
+153
View File
@@ -0,0 +1,153 @@
import { loadEnvironmentFromNodeEnv } from "../src/common/env-loader.js";
loadEnvironmentFromNodeEnv();
const { getRawSecret } = await import("../src/common/secret-manager.js");
const { initMongoDB, closeMongoDB } = await import("../src/common/mongodb.js");
const { initLogger, getLogger } = await import("../src/observability/logger.js");
const { getMemoryEvictionEnabled } = await import("../src/config.js");
const { KNOWLEDGE_COLLECTION } = await import("../src/memory/mongo-long-term-memory.js");
const { detectContradictions } = await import("../src/memory/contradiction-detector.js");
const { evaluateDocument } = await import("../src/memory/forgetting-policy.js");
initLogger();
const logger = getLogger();
interface KnowledgeRecord {
_id: string;
category: string;
text: string;
metadata?: Record<string, unknown> | undefined;
lastReferencedAt?: Date | undefined;
references?: Date[] | undefined;
updatedAt?: Date | undefined;
}
function firstNonEmpty(
...values: Array<string | null | undefined>
): string | undefined {
for (const value of values) {
if (value?.trim()) {
return value.trim();
}
}
return undefined;
}
function parseSinceDays(): number {
const raw = process.argv.find((arg) => arg.startsWith("--since="))?.split("=")[1];
if (!raw) {
return 30;
}
const match = raw.match(/^(\d+)d$/u);
return match ? Number(match[1]) : 30;
}
const dryRun = process.argv.includes("--dry-run");
const target = process.argv.find((arg) => arg.startsWith("--target="))?.split("=")[1] ?? "all";
const sinceDays = parseSinceDays();
if (!dryRun && !getMemoryEvictionEnabled()) {
throw new Error(
"AGENT_MEMORY_EVICTION_ENABLED must be true for non-dry-run eviction",
);
}
if (target !== "knowledge" && target !== "all") {
logger.info({ dryRun, target }, "memory eviction skipped: target has no Mongo knowledge work");
process.exit(0);
}
const connectionString = firstNonEmpty(
getRawSecret("MONGODB_CONNECTION_STRING"),
process.env["MONGODB_CONNECTION_STRING"],
);
if (!connectionString) {
if (dryRun) {
logger.warn(
{ dryRun },
"MONGODB_CONNECTION_STRING is not configured; memory eviction dry-run skipped",
);
process.exit(0);
}
throw new Error("MONGODB_CONNECTION_STRING is required for memory eviction");
}
const connection = await initMongoDB(connectionString);
try {
const collection = connection.collection<KnowledgeRecord>(KNOWLEDGE_COLLECTION);
const docs = await collection
.find({ "metadata.archived": { $ne: true } })
.sort({ updatedAt: 1 })
.toArray();
const contradictions = detectContradictions(
docs.map((doc) => ({
id: doc._id,
metadata: doc.metadata,
text: doc.text,
})),
);
const contradictionByExistingId = new Map(
contradictions.map((candidate) => [candidate.existingId, candidate.newId]),
);
const now = new Date();
const sinceMs = now.getTime() - sinceDays * 24 * 60 * 60 * 1000;
let archived = 0;
let kept = 0;
for (const doc of docs) {
const references = (doc.references ?? []).filter((date) => {
const time = new Date(date).getTime();
return Number.isFinite(time) && time >= sinceMs;
});
const decision = evaluateDocument(
{
id: doc._id,
category: doc.category,
metadata: doc.metadata,
text: doc.text,
},
{
contradictsId: contradictionByExistingId.get(doc._id),
lastReferencedAt: doc.lastReferencedAt ? new Date(doc.lastReferencedAt) : undefined,
now,
referenceCount30d: references.length,
updatedAt: doc.updatedAt ? new Date(doc.updatedAt) : undefined,
},
);
if (decision.action === "keep") {
kept += 1;
continue;
}
archived += 1;
if (!dryRun) {
await collection.updateOne(
{ _id: doc._id },
{
$set: {
"metadata.archived": true,
"metadata.archivedAt": now.toISOString(),
"metadata.archiveReason": decision.reason,
"metadata.archiveEvidence": decision.evidence,
},
},
);
}
}
logger.info(
{
archived,
contradictions: contradictions.length,
dryRun,
kept,
scanned: docs.length,
sinceDays,
},
"memory eviction completed",
);
} finally {
await closeMongoDB();
}
+25
View File
@@ -7,8 +7,13 @@ import {
getAlertThresholdP95LatencyMs,
getEmbeddingModel,
getEmbeddingProvider,
getLongTermSummaryEnabled,
getMemoryEvictionEnabled,
getPhoenixLogPrompts,
getPhoenixOtlpEndpoint,
getRagHybridEnabled,
getRagRerankerEnabled,
getRagRerankerTopK,
getRagEnabled,
getSlackAlertWebhookUrl,
getSlackWebhookTimeoutMs,
@@ -25,6 +30,11 @@ const ENV_KEYS = [
"ALERT_THRESHOLD_P95_LATENCY_MS",
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
"AGENT_RAG_ENABLED",
"AGENT_RAG_HYBRID_ENABLED",
"AGENT_RAG_RERANKER_ENABLED",
"AGENT_RAG_RERANKER_TOP_K",
"AGENT_LONG_TERM_SUMMARY_ENABLED",
"AGENT_MEMORY_EVICTION_ENABLED",
"EMBEDDING_MODEL",
"EMBEDDING_PROVIDER",
"PHOENIX_LOG_PROMPTS",
@@ -85,14 +95,29 @@ describe("config", () => {
it("reads RAG and embedding configuration", () => {
expect(getRagEnabled()).toBe(true);
expect(getRagHybridEnabled()).toBe(true);
expect(getRagRerankerEnabled()).toBe(true);
expect(getRagRerankerTopK()).toBe(5);
expect(getLongTermSummaryEnabled()).toBe(true);
expect(getMemoryEvictionEnabled()).toBe(false);
expect(getEmbeddingProvider()).toBe("openrouter");
expect(getEmbeddingModel()).toBe("openai/text-embedding-3-small");
vi.stubEnv("AGENT_RAG_ENABLED", "false");
vi.stubEnv("AGENT_RAG_HYBRID_ENABLED", "false");
vi.stubEnv("AGENT_RAG_RERANKER_ENABLED", "false");
vi.stubEnv("AGENT_RAG_RERANKER_TOP_K", "2");
vi.stubEnv("AGENT_LONG_TERM_SUMMARY_ENABLED", "false");
vi.stubEnv("AGENT_MEMORY_EVICTION_ENABLED", "true");
vi.stubEnv("EMBEDDING_PROVIDER", "openai");
vi.stubEnv("EMBEDDING_MODEL", "custom-embedding");
expect(getRagEnabled()).toBe(false);
expect(getRagHybridEnabled()).toBe(false);
expect(getRagRerankerEnabled()).toBe(false);
expect(getRagRerankerTopK()).toBe(2);
expect(getLongTermSummaryEnabled()).toBe(false);
expect(getMemoryEvictionEnabled()).toBe(true);
expect(getEmbeddingProvider()).toBe("openai");
expect(getEmbeddingModel()).toBe("custom-embedding");
});
+24
View File
@@ -133,6 +133,30 @@ export function getRagEnabled(): boolean {
return getBoolean("AGENT_RAG_ENABLED", true);
}
export function getRagHybridEnabled(): boolean {
return getBoolean("AGENT_RAG_HYBRID_ENABLED", true);
}
export function getRagRerankerEnabled(): boolean {
return getBoolean("AGENT_RAG_RERANKER_ENABLED", true);
}
export function getRagRerankerTopK(): number {
return getPositiveNumber("AGENT_RAG_RERANKER_TOP_K", 5);
}
export function getRagCandidatePoolSize(): number {
return getPositiveNumber("AGENT_RAG_CANDIDATE_POOL_SIZE", 50);
}
export function getLongTermSummaryEnabled(): boolean {
return getBoolean("AGENT_LONG_TERM_SUMMARY_ENABLED", true);
}
export function getMemoryEvictionEnabled(): boolean {
return getBoolean("AGENT_MEMORY_EVICTION_ENABLED", false);
}
export function getEmbeddingProvider(): "openrouter" | "openai" {
const raw = process.env["EMBEDDING_PROVIDER"]?.trim().toLowerCase();
return raw === "openai" ? "openai" : "openrouter";
+6
View File
@@ -0,0 +1,6 @@
{"id":"memory-retrieval-accurate","name":"retrieval-accurate","input":{"request":{"requestId":"mem_mod_ads","skill":"moderation","chatroomId":"memory_room","userId":"memory_user_1","nickname":"tester","trigger":"user-message","userMessage":"광고 도배 기준이 뭐예요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["광고","도배"]}}
{"id":"memory-long-range-understanding","name":"long-range-understanding","input":{"request":{"requestId":"mem_onboarding_recall","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_2","nickname":"LolName_KR/95/남/D","trigger":"user-message","userMessage":"내 닉네임 뭐라고 했지?"},"conversationHistory":[{"role":"user","content":"디스코드 닉네임은 LolName_KR입니다","timestamp":"2026-05-01T00:00:00.000Z"},{"role":"assistant","content":"확인했어요.","timestamp":"2026-05-01T00:00:01.000Z"}],"summary":"사용자가 디스코드 닉네임은 LolName_KR이라고 말했다."},"expected":{"mustIncludeAny":["LolName_KR"],"expectedMemory":{"expectedRecalledFact":"LolName_KR"}}}
{"id":"memory-test-time-learning","name":"test-time-learning","input":{"request":{"requestId":"mem_onboarding_hint","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_3","nickname":"낭만/00/남D","trigger":"nickname-invalid"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":["닉네임 형식 오류","닉네임 형식 오류","닉네임 형식 오류"],"pastIntents":[{"kind":"nickname-question","count":5}],"preferredStyle":"concise","totalTurns":12,"userId":"user_memory"}},"expected":{"mustIncludeAny":["예시","형식"],"expectedMemory":{"expectedProphylacticHint":true}}}
{"id":"memory-selective-forgetting","name":"selective-forgetting","input":{"request":{"requestId":"mem_forgetting","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_4","nickname":"tester","trigger":"user-message","userMessage":"닉네임 규칙 최신 기준 알려줘"},"summary":"정정된 최신 규칙만 답해야 한다."},"expected":{"mustNotInclude":["예전 기준"],"expectedMemory":{"expectedArchivedDocPattern":"예전 기준"}}}
{"id":"memory-escalation-history","name":"escalation-history","input":{"request":{"requestId":"mem_supervisor_history","skill":"supervisor","chatroomId":"memory_room","userId":"memory_user_5","nickname":"tester","trigger":"user-message","context":{"handoffPacket":{"constraints":[],"currentState":{},"evidence":[],"expectedOutputSchema":"text","handoffReason":"out-of-scope","sourceChain":["onboarding"],"summary":"광고성 메시지로 보임","userGoal":"광고 홍보 가능 여부 확인"}}}},"expected":{"expectedMemory":{"expectedSupervisorTarget":"moderation"}}}
{"id":"memory-contradiction","name":"memory-contradiction","input":{"request":{"requestId":"mem_contradiction","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_6","nickname":"새닉/95/남/D","trigger":"user-message","userMessage":"닉네임 새닉으로 변경했어요"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":[],"pastIntents":[{"kind":"nickname-question","count":4}],"preferredStyle":"neutral","totalTurns":8,"userId":"user_memory","longTermSummary":{"text":"사용자는 이전 닉네임에서 새닉으로 변경했다고 말했다.","updatedAt":"2026-05-18T00:00:00.000Z","turnSnapshot":8}}},"expected":{"mustIncludeAny":["새닉"],"expectedMemory":{"expectedContradictionResolved":true,"expectedRecalledFact":"새닉"}}}
+95
View File
@@ -0,0 +1,95 @@
import type { AgentResponse } from "@agent/types";
import type { RetrievedDocument } from "../../memory/memory-types.js";
import type { UserProfile } from "@skills/onboarding/state/user-memory";
export interface MemoryJudgeExpected {
expectedArchivedDocPattern?: string | undefined;
expectedContradictionResolved?: boolean | undefined;
expectedProphylacticHint?: boolean | undefined;
expectedRecalledFact?: string | undefined;
expectedRetrievedDocIds?: string[] | undefined;
expectedSupervisorTarget?: string | undefined;
}
export interface MemoryJudgeResult {
checks: Array<{
detail?: string | undefined;
name: keyof MemoryJudgeExpected;
passed: boolean;
skipped: boolean;
}>;
passed: boolean;
score: number;
}
export function judgeMemory(
expected: MemoryJudgeExpected | undefined,
response: AgentResponse,
metadata?: {
profile?: UserProfile | undefined;
retrievedDocs?: RetrievedDocument[] | undefined;
},
): MemoryJudgeResult {
if (!expected || Object.keys(expected).length === 0) {
return { checks: [], passed: true, score: 1 };
}
const checks: MemoryJudgeResult["checks"] = [];
if (expected.expectedRecalledFact) {
checks.push({
name: "expectedRecalledFact",
passed: response.text.includes(expected.expectedRecalledFact),
skipped: false,
});
}
if (expected.expectedProphylacticHint !== undefined) {
const hasHint = /(예시|형식|가이드|주의|먼저)/u.test(response.text);
checks.push({
name: "expectedProphylacticHint",
passed: expected.expectedProphylacticHint ? hasHint : !hasHint,
skipped: false,
});
}
if (expected.expectedSupervisorTarget) {
const suggestedTarget = response.metadata?.["suggestedTarget"];
checks.push({
detail: typeof suggestedTarget === "string" ? suggestedTarget : "",
name: "expectedSupervisorTarget",
passed: suggestedTarget === expected.expectedSupervisorTarget,
skipped: false,
});
}
if (expected.expectedRetrievedDocIds) {
const ids = new Set(metadata?.retrievedDocs?.map((doc) => doc.id) ?? []);
checks.push({
name: "expectedRetrievedDocIds",
passed: expected.expectedRetrievedDocIds.every((id) => ids.has(id)),
skipped: ids.size === 0,
});
}
if (expected.expectedArchivedDocPattern) {
checks.push({
name: "expectedArchivedDocPattern",
passed: !response.text.includes(expected.expectedArchivedDocPattern),
skipped: false,
});
}
if (expected.expectedContradictionResolved !== undefined) {
checks.push({
name: "expectedContradictionResolved",
passed: expected.expectedContradictionResolved
? !/(예전|이전|기존)/u.test(response.text)
: true,
skipped: false,
});
}
const activeChecks = checks.filter((check) => !check.skipped);
const passedCount = activeChecks.filter((check) => check.passed).length;
const score = activeChecks.length === 0 ? 1 : passedCount / activeChecks.length;
return {
checks,
passed: score === 1,
score,
};
}
+47 -3
View File
@@ -7,6 +7,7 @@ import { GarnetClient, type StreamEntry } from "@common/garnet";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { NatsClient } from "@common/nats";
import { getConversationSummaryKey } from "@conv/conversation-store";
import { anonymizeIdentifier } from "@eval/anonymize";
import { LlmClient } from "@llm/llm-client";
import type { ProviderName } from "@llm/types";
import { runWithContext } from "@obs/context";
@@ -34,6 +35,7 @@ import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js
import { runEvalDispatch } from "./handoff-dispatch.js";
import { judgeHandoff } from "./judges/handoff-judge.js";
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
import { judgeMemory } from "./judges/memory-judge.js";
import { judgeProcessRule } from "./judges/process-judge.js";
import { judgeRuleCase } from "./judges/rule-judge.js";
import { getSkillJudgeConfig } from "./judges/skill-config.js";
@@ -50,6 +52,7 @@ import type {
EvalSummaryDelta,
} from "./types.js";
import { createMemoryService } from "../memory/memory-service.js";
import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory";
const DEFAULT_REPORTS_DIR = "eval-reports";
const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
@@ -71,7 +74,7 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
moderation: (deps) =>
new ModerationSkill(
deps.llm,
createMemoryService({ garnetClient: deps.garnet }),
createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }),
),
onboarding: (deps) =>
new OnboardingSkill(
@@ -80,7 +83,7 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
deps.room,
noopShadowEvalPublisher,
undefined,
createMemoryService({ garnetClient: deps.garnet }),
createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }),
),
supervisor: (deps) =>
new SupervisorSkill(deps.garnet, {
@@ -451,6 +454,9 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
if (mustNotInclude) {
expected.mustNotInclude = mustNotInclude;
}
if (isRecord(raw["expectedMemory"])) {
expected.expectedMemory = raw["expectedMemory"] as EvalCaseExpected["expectedMemory"];
}
const processExpectedRaw = raw["processExpected"];
if (isRecord(processExpectedRaw)) {
@@ -569,6 +575,10 @@ function parseEvalCaseLine(
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
}
if (raw.input["userProfile"] === null || isRecord(raw.input["userProfile"])) {
input.userProfile = raw.input["userProfile"] as EvalCaseInput["userProfile"];
}
return {
id:
typeof raw.id === "string" && raw.id.trim()
@@ -646,6 +656,15 @@ async function hydrateEvalState(
input.summary,
);
}
if (input.userProfile !== undefined) {
await garnetClient.setJson(
`${USER_MEMORY_KEY_PREFIX}:${
anonymizeIdentifier("user", userId) ?? "unknown"
}`,
input.userProfile,
);
}
}
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
@@ -688,7 +707,11 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary {
}
function isCasePassed(result: EvalCaseResult): boolean {
return result.ruleJudge.passed && (result.handoffJudge?.passed ?? true);
return (
result.ruleJudge.passed &&
(result.handoffJudge?.passed ?? true) &&
(result.memoryJudge?.passed ?? true)
);
}
function createBaselineDelta(
@@ -943,6 +966,13 @@ async function evaluateOnboardingCase(
processTraceSnapshot,
judgeConfig.toolRegistryFactory(),
);
const memoryJudge = judgeMemory(
evalCase.expected.expectedMemory,
response,
{
profile: input.userProfile ?? undefined,
},
);
return {
id: evalCase.id,
@@ -955,6 +985,7 @@ async function evaluateOnboardingCase(
response,
ruleJudge,
llmJudge,
memoryJudge,
processJudge,
processTrace: processTraceSnapshot,
};
@@ -1033,6 +1064,10 @@ async function evaluateHandoffCase(
result.response,
result.finalSkill,
);
const memoryJudge = judgeMemory(
evalCase.expected.expectedMemory,
result.response,
);
return {
id: evalCase.id,
@@ -1060,6 +1095,7 @@ async function evaluateHandoffCase(
reason: "handoff-eval",
},
handoffJudge,
memoryJudge,
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
@@ -1129,6 +1165,13 @@ async function evaluateGenericSkillCase(
input.conversationHistory ?? [],
);
const latencyMs = Date.now() - startedAt;
const memoryJudge = judgeMemory(
evalCase.expected.expectedMemory,
response,
{
profile: input.userProfile ?? undefined,
},
);
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
@@ -1160,6 +1203,7 @@ async function evaluateGenericSkillCase(
skipped: true,
reason: "disabled",
},
memoryJudge,
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
+8
View File
@@ -5,12 +5,17 @@ import type {
HandoffReason,
} from "@agent/types";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import type { UserProfile } from "@skills/onboarding/state/user-memory";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
import type {
ProcessTraceSnapshot,
ProcessTraceStepName,
} from "@skills/onboarding/runtime/trace";
import type { PrmFeatureVector } from "./judges/prm.js";
import type {
MemoryJudgeExpected,
MemoryJudgeResult,
} from "./judges/memory-judge.js";
export type EvalProcessTraceSnapshot = ProcessTraceSnapshot;
@@ -28,6 +33,7 @@ export interface EvalCaseInput {
session?: OnboardingSession | null | undefined;
blackboard?: Blackboard | null | undefined;
summary?: string | undefined;
userProfile?: UserProfile | null | undefined;
}
export interface EvalCaseExpected {
@@ -37,6 +43,7 @@ export interface EvalCaseExpected {
expectedTarget?: string | undefined;
shouldHandoff?: boolean | undefined;
} | undefined;
expectedMemory?: MemoryJudgeExpected | undefined;
expectedSourceChain?: {
chain: string[];
mode: "contains" | "exact" | "prefix";
@@ -183,6 +190,7 @@ export interface EvalCaseResult {
ruleJudge: RuleJudgeResult;
llmJudge: LlmJudgeResult;
handoffJudge?: HandoffJudgeResult | undefined;
memoryJudge?: MemoryJudgeResult | undefined;
processJudge?: ProcessJudgeResult | undefined;
processTrace?: ProcessTraceSnapshot | undefined;
error?: string | undefined;
+7
View File
@@ -31,6 +31,9 @@ const { LlmClient } = await import("./llm/index.js");
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
const { createMemoryService } = await import("./memory/memory-service.js");
const { createEscalationHistoryService } = await import(
"./memory/mongo-escalation-history.js"
);
const { RoomContextService } = await import("./room-context/index.js");
const { startMetricsServer, stopMetricsServer } = await import(
"./observability/metrics-server.js"
@@ -226,8 +229,11 @@ async function bootstrap(): Promise<void> {
const memoryService = createMemoryService({
garnetClient,
llmClient,
mongooseConnection: mongoConnection,
});
const escalationHistoryService =
createEscalationHistoryService(mongoConnection);
// Agent setup
const conversationStore = new ConversationStore(garnetClient);
@@ -265,6 +271,7 @@ async function bootstrap(): Promise<void> {
agent.registerSkill(
factory({
garnetClient,
escalationHistoryService,
llmClient,
memoryService,
natsClient,
+16
View File
@@ -0,0 +1,16 @@
import { describe, expect, it } from "vitest";
import { buildBM25Index } from "./bm25.js";
describe("buildBM25Index", () => {
it("ranks Korean keyword matches above unrelated documents", () => {
const index = buildBM25Index([
{ id: "ads", text: "광고와 도배 홍보 링크는 제재 대상입니다." },
{ id: "age", text: "가입 가능 연령은 27세 이상입니다." },
]);
const [top] = index.rank("광고 도배 기준", 2);
expect(top?.id).toBe("ads");
expect(top?.score).toBeGreaterThan(0);
});
});
+82
View File
@@ -0,0 +1,82 @@
export interface BM25Document {
id: string;
text: string;
}
export interface BM25Index {
rank(query: string, topK: number): Array<{ id: string; score: number }>;
}
const K1 = 1.5;
const B = 0.75;
function normalize(text: string): string {
return text
.normalize("NFKC")
.toLowerCase()
.replace(/[^\p{Letter}\p{Number}\s]/gu, " ");
}
function tokenize(text: string): string[] {
const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0);
const compact = normalize(text).replace(/\s+/gu, "");
const grams: string[] = [];
for (let i = 0; i < compact.length - 1; i += 1) {
grams.push(compact.slice(i, i + 2));
}
return [...new Set([...words, ...grams])];
}
export function buildBM25Index(documents: BM25Document[]): BM25Index {
const termFrequencies = new Map<string, Map<string, number>>();
const documentLengths = new Map<string, number>();
const documentFrequency = new Map<string, number>();
let totalLength = 0;
for (const document of documents) {
const tokens = tokenize(document.text);
documentLengths.set(document.id, tokens.length);
totalLength += tokens.length;
const frequencies = new Map<string, number>();
for (const token of tokens) {
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
}
termFrequencies.set(document.id, frequencies);
for (const token of frequencies.keys()) {
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
}
}
const averageLength = documents.length === 0 ? 0 : totalLength / documents.length;
return {
rank(query: string, topK: number): Array<{ id: string; score: number }> {
const queryTokens = tokenize(query);
if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) {
return [];
}
return documents
.map((document) => {
const frequencies = termFrequencies.get(document.id) ?? new Map<string, number>();
const documentLength = documentLengths.get(document.id) ?? 0;
const score = queryTokens.reduce((sum, token) => {
const tf = frequencies.get(token) ?? 0;
if (tf === 0) {
return sum;
}
const df = documentFrequency.get(token) ?? 0;
const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5));
const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength));
return sum + idf * ((tf * (K1 + 1)) / denominator);
}, 0);
return { id: document.id, score };
})
.filter((item) => item.score > 0)
.sort((left, right) => right.score - left.score)
.slice(0, Math.max(1, topK));
},
};
}
+30
View File
@@ -0,0 +1,30 @@
import { describe, expect, it } from "vitest";
import { detectContradictions, textSimilarity } from "./contradiction-detector.js";
describe("textSimilarity", () => {
it("returns lower scores for very different text", () => {
expect(textSimilarity("닉네임 규칙", "광고 제재 기준")).toBeLessThan(0.7);
});
});
describe("detectContradictions", () => {
it("flags corrections in the same source section", () => {
const [candidate] = detectContradictions([
{
id: "old",
metadata: { sourceFile: "rules.md", sectionTitle: "nickname" },
text: "닉네임은 기존 형식을 사용합니다.",
},
{
id: "new",
metadata: { sourceFile: "rules.md", sectionTitle: "nickname" },
text: "정정: 닉네임은 새 형식을 사용합니다.",
},
]);
expect(candidate).toMatchObject({
existingId: "old",
newId: "new",
});
});
});
+83
View File
@@ -0,0 +1,83 @@
export interface ContradictionCandidate {
existingId: string;
newId: string;
reason: "keyword" | "text-distance";
similarity: number;
}
export interface ContradictionDocument {
id: string;
metadata?: Record<string, unknown> | undefined;
text: string;
}
function groupKey(doc: ContradictionDocument): string | null {
const sourceFile = doc.metadata?.["sourceFile"];
const sectionTitle = doc.metadata?.["sectionTitle"];
if (typeof sourceFile !== "string" || typeof sectionTitle !== "string") {
return null;
}
return `${sourceFile}#${sectionTitle}`;
}
function bigrams(text: string): Set<string> {
const compact = text
.normalize("NFKC")
.toLowerCase()
.replace(/\s+/gu, "");
const grams = new Set<string>();
for (let i = 0; i < compact.length - 1; i += 1) {
grams.add(compact.slice(i, i + 2));
}
return grams;
}
export function textSimilarity(left: string, right: string): number {
const leftGrams = bigrams(left);
const rightGrams = bigrams(right);
if (leftGrams.size === 0 && rightGrams.size === 0) {
return 1;
}
const intersection = [...leftGrams].filter((gram) => rightGrams.has(gram)).length;
const union = new Set([...leftGrams, ...rightGrams]).size;
return union === 0 ? 0 : intersection / union;
}
function hasCorrectionKeyword(text: string): boolean {
return /(변경|정정|수정|폐지|삭제|바뀜|업데이트)/u.test(text);
}
export function detectContradictions(
documents: ContradictionDocument[],
): ContradictionCandidate[] {
const groups = new Map<string, ContradictionDocument[]>();
for (const doc of documents) {
const key = groupKey(doc);
if (!key) {
continue;
}
groups.set(key, [...(groups.get(key) ?? []), doc]);
}
const candidates: ContradictionCandidate[] = [];
for (const docs of groups.values()) {
for (let index = 1; index < docs.length; index += 1) {
const previous = docs[index - 1];
const current = docs[index];
if (!previous || !current) {
continue;
}
const similarity = textSimilarity(previous.text, current.text);
if (similarity < 0.7 || hasCorrectionKeyword(current.text)) {
candidates.push({
existingId: previous.id,
newId: current.id,
reason: hasCorrectionKeyword(current.text) ? "keyword" : "text-distance",
similarity,
});
}
}
}
return candidates;
}
+58
View File
@@ -0,0 +1,58 @@
import type { HandoffReason } from "@agent/types";
import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue";
export interface UserEscalationRecord {
queuedAt: string;
reason: HandoffReason;
reviewId: string;
target: string;
resolution?: {
resolvedAt: string;
action: "dismiss" | "manual-reply" | "reroute";
note?: string | undefined;
} | undefined;
packetSummary: string;
sourceChain: string[];
}
export interface UserEscalationHistory {
userId: string;
chatroomId?: string | undefined;
escalations: UserEscalationRecord[];
totalEscalations: number;
lastEscalationAt: string;
topReasons: Array<{ reason: HandoffReason; count: number }>;
}
export interface EscalationHistoryService {
load(userId: string): Promise<UserEscalationHistory | null>;
recordEscalation(record: {
userId: string;
chatroomId?: string | undefined;
queuedAt: string;
reason: HandoffReason;
reviewId: string;
target: string;
packetSummary: string;
sourceChain: string[];
}): Promise<void>;
recordResolution(record: {
userId: string;
reviewId: string;
resolution: SupervisorReviewResolution;
}): Promise<void>;
}
export class NoopEscalationHistoryService implements EscalationHistoryService {
load(): Promise<UserEscalationHistory | null> {
return Promise.resolve(null);
}
recordEscalation(): Promise<void> {
return Promise.resolve();
}
recordResolution(): Promise<void> {
return Promise.resolve();
}
}
+32
View File
@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import { evaluateDocument, evaluateUserFlags } from "./forgetting-policy.js";
import type { UserProfile } from "@skills/onboarding/state/user-memory";
describe("evaluateDocument", () => {
it("archives stale low-reference documents", () => {
const decision = evaluateDocument(
{ category: "rule", id: "doc-1", text: "old rule" },
{
now: new Date("2026-05-18T00:00:00.000Z"),
referenceCount30d: 0,
updatedAt: new Date("2025-12-01T00:00:00.000Z"),
},
);
expect(decision.action).toBe("archive");
expect(decision.reason).toBe("freshness-stale");
});
});
describe("evaluateUserFlags", () => {
it("keeps the newest unique flags within the limit", () => {
const profile = {
flaggedIssues: ["a", "b", "c"],
} as UserProfile;
expect(evaluateUserFlags(profile, 2)).toEqual({
archived: ["a"],
kept: ["b", "c"],
});
});
});
+81
View File
@@ -0,0 +1,81 @@
import type { UserProfile } from "@skills/onboarding/state/user-memory";
import type { KnowledgeDocument } from "./memory-types.js";
export interface ForgettingDecision {
documentId: string;
action: "archive" | "delete" | "keep";
reason: "contradiction" | "freshness-stale" | "relevance-low" | "ttl-expired";
evidence?: string | undefined;
}
export interface KnowledgeForgettingSignals {
contradictsId?: string | undefined;
lastReferencedAt?: Date | undefined;
now?: Date | undefined;
referenceCount30d: number;
updatedAt?: Date | undefined;
}
function daysSince(date: Date | undefined, now: Date): number {
if (!date) {
return Number.POSITIVE_INFINITY;
}
return Math.max(0, now.getTime() - date.getTime()) / (24 * 60 * 60 * 1000);
}
export function evaluateDocument(
doc: KnowledgeDocument,
signals: KnowledgeForgettingSignals,
): ForgettingDecision {
const now = signals.now ?? new Date();
const documentId = doc.id ?? "unknown";
if (signals.contradictsId) {
return {
action: "archive",
documentId,
evidence: `contradicts ${signals.contradictsId}`,
reason: "contradiction",
};
}
const staleDays = daysSince(signals.updatedAt, now);
const referenceDays = daysSince(signals.lastReferencedAt, now);
if (staleDays >= 90 && signals.referenceCount30d < 3) {
return {
action: "archive",
documentId,
evidence: `updated ${Math.round(staleDays)}d ago, refs30d=${signals.referenceCount30d}`,
reason: "freshness-stale",
};
}
if (referenceDays >= 30 && signals.referenceCount30d < 3) {
return {
action: "archive",
documentId,
evidence: `last referenced ${Math.round(referenceDays)}d ago, refs30d=${signals.referenceCount30d}`,
reason: "relevance-low",
};
}
return {
action: "keep",
documentId,
reason: "relevance-low",
};
}
export function evaluateUserFlags(
profile: UserProfile,
maxFlags: number,
): { archived: string[]; kept: string[] } {
const unique = [...new Set(profile.flaggedIssues.map((issue) => issue.trim()).filter(Boolean))];
if (unique.length <= maxFlags) {
return { archived: [], kept: unique };
}
return {
archived: unique.slice(0, Math.max(0, unique.length - maxFlags)),
kept: unique.slice(-maxFlags),
};
}
+96
View File
@@ -0,0 +1,96 @@
import type { LlmClient } from "@llm/llm-client";
import { getLongTermSummaryEnabled } from "../config.js";
import type { UserProfile } from "@skills/onboarding/state/user-memory";
export interface LongTermSummary {
text: string;
updatedAt: string;
turnSnapshot: number;
}
interface SummaryPayload {
text?: unknown;
}
function daysBetween(leftIso: string, rightIso: string): number {
const left = Date.parse(leftIso);
const right = Date.parse(rightIso);
if (!Number.isFinite(left) || !Number.isFinite(right)) {
return Number.POSITIVE_INFINITY;
}
return Math.abs(right - left) / (24 * 60 * 60 * 1000);
}
export function shouldRefreshLongTermSummary(
profile: UserProfile,
now = new Date().toISOString(),
): boolean {
if (!getLongTermSummaryEnabled()) {
return false;
}
if (profile.totalTurns < 30) {
return false;
}
const summary = profile.longTermSummary;
if (!summary) {
return true;
}
return (
profile.totalTurns - summary.turnSnapshot >= 30 ||
daysBetween(summary.updatedAt, now) >= 7
);
}
export async function buildLongTermSummary(
llmClient: Pick<LlmClient, "chatCompletion">,
profile: UserProfile,
conversationSummary: string | null,
escalationPattern?: string,
): Promise<LongTermSummary> {
const now = new Date().toISOString();
const result = await llmClient.chatCompletion(
[
{
role: "system",
content: [
"너는 사용자별 장기 메모를 작성한다.",
"운영자가 다음 응답에서 참고할 수 있게 3-5문장으로 요약한다.",
"개인정보나 원문 식별자는 넣지 않는다.",
'JSON으로만 응답한다: {"text":"요약"}',
].join("\n"),
},
{
role: "user",
content: [
`[totalTurns] ${profile.totalTurns}`,
`[preferredStyle] ${profile.preferredStyle}`,
`[pastIntents] ${profile.pastIntents.map((item) => `${item.kind}:${item.count}`).join(", ") || "-"}`,
`[flaggedIssues] ${profile.flaggedIssues.slice(-10).join(", ") || "-"}`,
`[conversationSummary] ${conversationSummary ?? "-"}`,
`[escalationPattern] ${escalationPattern ?? "-"}`,
].join("\n"),
},
],
{
jsonMode: true,
maxTokens: 300,
temperature: 0.3,
tier: 0,
},
);
if (!result?.content.trim()) {
throw new Error("Long-term summary LLM returned no content");
}
const parsed = JSON.parse(result.content) as SummaryPayload;
if (typeof parsed.text !== "string" || parsed.text.trim().length === 0) {
throw new Error("Long-term summary payload missing text");
}
return {
text: parsed.text.trim(),
updatedAt: now,
turnSnapshot: profile.totalTurns,
};
}
+4
View File
@@ -1,5 +1,6 @@
import type mongoose from "mongoose";
import type { GarnetClient } from "@common/garnet";
import type { LlmClient } from "@llm/llm-client";
import type { Blackboard } from "@skills/onboarding/types";
import { createEmbeddingClient } from "./embedding-client.js";
import { GarnetSessionMemory } from "./garnet-session-memory.js";
@@ -7,6 +8,7 @@ import {
MongoLongTermMemory,
NoopLongTermMemory,
} from "./mongo-long-term-memory.js";
import { LlmReranker } from "./reranker.js";
import type {
LongTermMemory,
MemoryService,
@@ -17,6 +19,7 @@ import type {
interface CreateMemoryServiceOptions {
garnetClient: GarnetClient;
llmClient?: Pick<LlmClient, "chatCompletion"> | undefined;
mongooseConnection?: mongoose.Connection | undefined;
}
@@ -88,6 +91,7 @@ export function createMemoryService(
options.garnetClient,
options.mongooseConnection,
createEmbeddingClient(),
options.llmClient ? new LlmReranker(options.llmClient) : undefined,
)
: new NoopLongTermMemory(options.garnetClient);
+4
View File
@@ -46,6 +46,10 @@ export interface SessionMemory {
export interface SearchOptions {
skill?: string | undefined;
category?: string | undefined;
candidatePoolSize?: number | undefined;
hybrid?: boolean | undefined;
rerank?: boolean | undefined;
rerankTopK?: number | undefined;
topK?: number | undefined;
}
+173
View File
@@ -0,0 +1,173 @@
import type mongoose from "mongoose";
import { Schema } from "mongoose";
import { anonymizeIdentifier } from "@eval/anonymize";
import type { HandoffReason } from "@agent/types";
import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue";
import {
NoopEscalationHistoryService,
type EscalationHistoryService,
type UserEscalationHistory,
type UserEscalationRecord,
} from "./escalation-history.js";
export const ESCALATION_HISTORY_COLLECTION = "user_escalation_history";
interface EscalationHistoryRecord {
_id: string;
userId: string;
chatroomId?: string | undefined;
escalations: UserEscalationRecord[];
totalEscalations: number;
lastEscalationAt: string;
topReasons: Array<{ reason: HandoffReason; count: number }>;
updatedAt: Date;
}
const EscalationHistorySchema = new Schema<EscalationHistoryRecord>(
{
_id: { type: String, required: true },
userId: { type: String, required: true, index: true },
chatroomId: { type: String, index: true },
escalations: [
{
queuedAt: String,
reason: String,
reviewId: String,
target: String,
resolution: {
resolvedAt: String,
action: String,
note: String,
},
packetSummary: String,
sourceChain: [String],
},
],
totalEscalations: { type: Number, default: 0 },
lastEscalationAt: String,
topReasons: [{ reason: String, count: Number }],
updatedAt: { type: Date, default: Date.now, index: true },
},
{ versionKey: false },
);
function userKey(userId: string): string {
return anonymizeIdentifier("user", userId) ?? "user_unknown";
}
function computeTopReasons(records: UserEscalationRecord[]): Array<{ reason: HandoffReason; count: number }> {
const counts = new Map<HandoffReason, number>();
for (const record of records) {
counts.set(record.reason, (counts.get(record.reason) ?? 0) + 1);
}
return [...counts.entries()]
.map(([reason, count]) => ({ reason, count }))
.sort((left, right) => right.count - left.count || left.reason.localeCompare(right.reason))
.slice(0, 5);
}
export class MongoEscalationHistoryService implements EscalationHistoryService {
private readonly model: mongoose.Model<EscalationHistoryRecord>;
constructor(connection: mongoose.Connection) {
this.model =
(connection.models["EscalationHistory"] as mongoose.Model<EscalationHistoryRecord> | undefined) ??
connection.model<EscalationHistoryRecord>(
"EscalationHistory",
EscalationHistorySchema,
ESCALATION_HISTORY_COLLECTION,
);
}
async load(userId: string): Promise<UserEscalationHistory | null> {
const doc = await this.model.findById(userKey(userId)).lean().exec();
if (!doc) {
return null;
}
return {
userId: doc.userId,
...(doc.chatroomId ? { chatroomId: doc.chatroomId } : {}),
escalations: doc.escalations.slice(-10),
lastEscalationAt: doc.lastEscalationAt,
topReasons: doc.topReasons,
totalEscalations: doc.totalEscalations,
};
}
async recordEscalation(record: {
userId: string;
chatroomId?: string | undefined;
queuedAt: string;
reason: HandoffReason;
reviewId: string;
target: string;
packetSummary: string;
sourceChain: string[];
}): Promise<void> {
const id = userKey(record.userId);
const item: UserEscalationRecord = {
packetSummary: record.packetSummary,
queuedAt: record.queuedAt,
reason: record.reason,
reviewId: record.reviewId,
sourceChain: record.sourceChain,
target: record.target,
};
await this.model.updateOne(
{ _id: id },
{
$inc: { totalEscalations: 1 },
$push: { escalations: { $each: [item], $slice: -50 } },
$set: {
...(record.chatroomId ? { chatroomId: record.chatroomId } : {}),
lastEscalationAt: record.queuedAt,
updatedAt: new Date(),
userId: id,
},
$setOnInsert: { _id: id },
},
{ upsert: true },
).exec();
await this.recomputeTopReasons(id);
}
async recordResolution(record: {
userId: string;
reviewId: string;
resolution: SupervisorReviewResolution;
}): Promise<void> {
const id = userKey(record.userId);
await this.model.updateOne(
{ _id: id, "escalations.reviewId": record.reviewId },
{
$set: {
"escalations.$.resolution": {
action: record.resolution.action,
resolvedAt: record.resolution.resolvedAt,
...(record.resolution.note ? { note: record.resolution.note } : {}),
},
updatedAt: new Date(),
},
},
).exec();
}
private async recomputeTopReasons(id: string): Promise<void> {
const doc = await this.model.findById(id).lean().exec();
if (!doc) {
return;
}
await this.model.updateOne(
{ _id: id },
{ $set: { topReasons: computeTopReasons(doc.escalations) } },
).exec();
}
}
export function createEscalationHistoryService(
connection?: mongoose.Connection,
): EscalationHistoryService {
return connection
? new MongoEscalationHistoryService(connection)
: new NoopEscalationHistoryService();
}
+117 -13
View File
@@ -3,11 +3,19 @@ import type mongoose from "mongoose";
import { Schema } from "mongoose";
import { anonymizeIdentifier } from "@eval/anonymize";
import { withSpan } from "@obs/spans";
import { getAgentUserMemoryTtlDays, getKnowledgeSearchCandidateLimit } from "../config.js";
import {
getAgentUserMemoryTtlDays,
getKnowledgeSearchCandidateLimit,
getRagCandidatePoolSize,
getRagHybridEnabled,
getRagRerankerEnabled,
getRagRerankerTopK,
} from "../config.js";
import type { UserProfile } from "@skills/onboarding/state/user-memory";
import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory";
import type { GarnetClient } from "@common/garnet";
import { rankByCosineSimilarity } from "./cosine.js";
import { buildBM25Index } from "./bm25.js";
import type {
KnowledgeDocument,
LongTermMemory,
@@ -15,6 +23,7 @@ import type {
SearchOptions,
} from "./memory-types.js";
import type { EmbeddingClient } from "./embedding-client.js";
import type { Reranker } from "./reranker.js";
export const KNOWLEDGE_COLLECTION = "knowledge_documents";
@@ -26,6 +35,8 @@ interface KnowledgeDocumentRecord {
embedding: number[];
embeddingModel: string;
metadata?: Record<string, unknown> | undefined;
lastReferencedAt?: Date | undefined;
references?: Date[] | undefined;
createdAt: Date;
updatedAt: Date;
}
@@ -39,6 +50,8 @@ const KnowledgeSchema = new Schema<KnowledgeDocumentRecord>(
embedding: { type: [Number], required: true },
embeddingModel: { type: String, required: true },
metadata: { type: Schema.Types.Mixed },
lastReferencedAt: { type: Date, index: true },
references: { type: [Date], default: [] },
createdAt: { type: Date, default: Date.now },
updatedAt: { type: Date, default: Date.now, index: true },
},
@@ -83,7 +96,9 @@ function stableDocumentId(doc: KnowledgeDocument): string {
}
function buildFilter(options: SearchOptions): Record<string, unknown> {
const filter: Record<string, unknown> = {};
const filter: Record<string, unknown> = {
"metadata.archived": { $ne: true },
};
if (options.skill?.trim()) {
filter["skill"] = options.skill.trim();
}
@@ -93,6 +108,34 @@ function buildFilter(options: SearchOptions): Record<string, unknown> {
return filter;
}
function normalizeScores(scores: Map<string, number>): Map<string, number> {
const values = [...scores.values()].filter((value) => Number.isFinite(value));
if (values.length === 0) {
return new Map();
}
const min = Math.min(...values);
const max = Math.max(...values);
if (max === min) {
return new Map([...scores.keys()].map((id) => [id, max > 0 ? 1 : 0]));
}
return new Map(
[...scores.entries()].map(([id, score]) => [id, (score - min) / (max - min)]),
);
}
function toRetrievedDocument(
doc: KnowledgeDocumentRecord & { score: number },
): RetrievedDocument {
return {
id: doc._id,
category: doc.category,
metadata: doc.metadata,
score: doc.score,
skill: doc.skill,
text: doc.text,
};
}
export class MongoLongTermMemory implements LongTermMemory {
private readonly model: mongoose.Model<KnowledgeDocumentRecord>;
@@ -100,6 +143,7 @@ export class MongoLongTermMemory implements LongTermMemory {
private readonly garnetClient: Pick<GarnetClient, "getJson" | "setJson">,
connection: mongoose.Connection,
private readonly embeddingClient: EmbeddingClient,
private readonly reranker?: Reranker | undefined,
) {
this.model =
(connection.models["KnowledgeDocument"] as mongoose.Model<KnowledgeDocumentRecord> | undefined) ??
@@ -162,33 +206,93 @@ export class MongoLongTermMemory implements LongTermMemory {
.limit(getKnowledgeSearchCandidateLimit())
.lean()
.exec()) as KnowledgeDocumentRecord[];
const ranked = rankByCosineSimilarity(
const candidatePoolSize = options.candidatePoolSize ?? getRagCandidatePoolSize();
const topK = options.topK ?? 5;
const shouldHybrid = options.hybrid ?? getRagHybridEnabled();
const shouldRerank = Boolean(
this.reranker && (options.rerank ?? getRagRerankerEnabled()),
);
const cosineRanked = rankByCosineSimilarity(
queryEmbedding,
docs.filter((doc) =>
Array.isArray(doc.embedding) &&
doc.embedding.length === queryEmbedding.length
),
options.topK ?? 5,
Math.max(candidatePoolSize, topK),
);
const ranked = shouldHybrid
? this.rankHybrid(trimmed, docs, cosineRanked, Math.max(candidatePoolSize, topK))
: cosineRanked.slice(0, Math.max(candidatePoolSize, topK));
const pool = ranked.slice(0, shouldRerank ? candidatePoolSize : topK);
const results =
shouldRerank && this.reranker
? await this.reranker.rerank(
trimmed,
pool.map(toRetrievedDocument),
options.rerankTopK ?? getRagRerankerTopK(),
)
: pool.slice(0, topK).map(toRetrievedDocument);
await this.markReferenced(results.map((doc) => doc.id));
span.setAttributes({
"memory.retrieval.candidate_count": docs.length,
"memory.retrieval.duration_ms": Math.max(0, Date.now() - startedAt),
"memory.retrieval.top_score": ranked[0]?.score,
"memory.retrieval.hybrid": shouldHybrid,
"memory.retrieval.rerank": shouldRerank,
"memory.retrieval.top_score": results[0]?.score,
});
return ranked.map((doc) => ({
id: doc._id,
category: doc.category,
metadata: doc.metadata,
score: doc.score,
skill: doc.skill,
text: doc.text,
}));
return results.slice(0, topK);
},
);
}
private rankHybrid(
query: string,
docs: KnowledgeDocumentRecord[],
cosineRanked: Array<KnowledgeDocumentRecord & { score: number }>,
limit: number,
): Array<KnowledgeDocumentRecord & { score: number }> {
const cosineScores = normalizeScores(
new Map(cosineRanked.map((doc) => [doc._id, doc.score])),
);
const bm25Scores = normalizeScores(
new Map(
buildBM25Index(docs.map((doc) => ({ id: doc._id, text: doc.text })))
.rank(query, docs.length)
.map((item) => [item.id, item.score]),
),
);
const byId = new Map(docs.map((doc) => [doc._id, doc]));
return [...byId.keys()]
.map((id) => ({
doc: byId.get(id),
score: ((cosineScores.get(id) ?? 0) + (bm25Scores.get(id) ?? 0)) / 2,
}))
.filter((item): item is { doc: KnowledgeDocumentRecord; score: number } =>
Boolean(item.doc) && item.score > 0
)
.map((item) => ({ ...item.doc, score: item.score }))
.sort((left, right) => right.score - left.score)
.slice(0, limit);
}
private async markReferenced(ids: string[]): Promise<void> {
if (ids.length === 0) {
return;
}
const now = new Date();
await this.model.updateMany(
{ _id: { $in: ids } },
{
$set: { lastReferencedAt: now },
$push: { references: { $each: [now], $slice: -100 } },
},
).exec();
}
async upsertDocument(doc: KnowledgeDocument): Promise<string> {
const id = doc.id?.trim() || stableDocumentId(doc);
const embedding = doc.embedding ?? await this.embeddingClient.embed(doc.text);
+30
View File
@@ -0,0 +1,30 @@
import { describe, expect, it } from "vitest";
import { deriveProfilePromptSnippets } from "./profile-update-policy.js";
import type { UserProfile } from "@skills/onboarding/state/user-memory";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
const profile: UserProfile = {
firstSeenAt: "2026-05-01T00:00:00.000Z",
flaggedIssues: ["닉네임 형식 오류", "닉네임 형식 오류", "닉네임 형식 오류"],
lastSeenAt: "2026-05-18T00:00:00.000Z",
pastIntents: [{ kind: "nickname-question", count: 5 }],
preferredStyle: "concise",
totalTurns: 10,
userId: "user_x",
};
const derived = {
intent: { kind: "nickname-question" },
} as DerivedState;
describe("deriveProfilePromptSnippets", () => {
it("derives frequent intent, repeated issue, and preferred style snippets", () => {
const snippets = deriveProfilePromptSnippets(profile, derived);
expect(snippets.map((snippet) => snippet.reason)).toEqual([
"frequent-intent",
"repeated-issue",
"preferred-style",
]);
});
});
+62
View File
@@ -0,0 +1,62 @@
import type { UserProfile } from "@skills/onboarding/state/user-memory";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
export interface ProfilePromptSnippet {
block: string;
reason: "frequent-intent" | "preferred-style" | "repeated-issue";
}
function normalizeIssue(issue: string): string {
return issue
.normalize("NFKC")
.replace(/\[[^\]]+\]/gu, "")
.replace(/[0-9]/gu, "")
.replace(/\s+/gu, " ")
.trim()
.slice(0, 32);
}
export function deriveProfilePromptSnippets(
profile: UserProfile,
derived: DerivedState,
): ProfilePromptSnippet[] {
const snippets: ProfilePromptSnippet[] = [];
const frequentIntent = profile.pastIntents.find(
(intent) => intent.kind === derived.intent.kind && intent.count >= 5,
);
if (frequentIntent) {
snippets.push({
reason: "frequent-intent",
block: `사용자는 ${frequentIntent.kind} 유형을 자주 물어봤어요. 핵심 답부터 짧게 말하고, 질문 범위를 넘기지 마세요.`,
});
}
const issueCounts = new Map<string, number>();
for (const issue of profile.flaggedIssues) {
const normalized = normalizeIssue(issue);
if (normalized) {
issueCounts.set(normalized, (issueCounts.get(normalized) ?? 0) + 1);
}
}
const repeatedIssue = [...issueCounts.entries()].find(([, count]) => count >= 3);
if (repeatedIssue) {
snippets.push({
reason: "repeated-issue",
block: `반복된 이슈가 있었어요: ${repeatedIssue[0]}. 같은 실수가 이어지지 않도록 정확한 형식 예시를 먼저 확인하세요.`,
});
}
if (profile.preferredStyle === "concise") {
snippets.push({
reason: "preferred-style",
block: "사용자는 짧고 바로 실행 가능한 답변을 선호해요.",
});
} else if (profile.preferredStyle === "warm") {
snippets.push({
reason: "preferred-style",
block: "사용자는 따뜻한 톤에 잘 반응했어요. 다만 불필요한 절차 설명은 붙이지 마세요.",
});
}
return snippets;
}
+136
View File
@@ -0,0 +1,136 @@
import type { LlmClient } from "@llm/llm-client";
import { getRequestLogger } from "@obs/context";
import { withSpan } from "@obs/spans";
import type { RetrievedDocument } from "./memory-types.js";
export interface Reranker {
rerank(
query: string,
candidates: RetrievedDocument[],
topN: number,
): Promise<RetrievedDocument[]>;
}
interface RankingItem {
id?: unknown;
score?: unknown;
}
interface RerankerPayload {
rankings?: unknown;
}
function truncate(text: string, maxLength: number): string {
return text.length <= maxLength ? text : `${text.slice(0, maxLength)}...`;
}
function parseRankings(raw: string): Array<{ id: string; score: number }> {
const parsed = JSON.parse(raw) as RerankerPayload;
if (!Array.isArray(parsed.rankings)) {
return [];
}
return parsed.rankings.flatMap((item: RankingItem) => {
if (typeof item.id !== "string") {
return [];
}
const score =
typeof item.score === "number" && Number.isFinite(item.score)
? Math.max(0, Math.min(1, item.score))
: 0;
return [{ id: item.id, score }];
});
}
export class LlmReranker implements Reranker {
constructor(private readonly llmClient: Pick<LlmClient, "chatCompletion">) {}
async rerank(
query: string,
candidates: RetrievedDocument[],
topN: number,
): Promise<RetrievedDocument[]> {
if (candidates.length <= 1) {
return candidates.slice(0, topN);
}
return withSpan(
"memory.reranker",
{
"memory.reranker.candidates_count": candidates.length,
"memory.reranker.top_n": topN,
},
async (span) => {
const startedAt = Date.now();
try {
const result = await this.llmClient.chatCompletion(
[
{
role: "system",
content: [
"너는 검색 결과 reranker다.",
"질문과 후보 문서를 보고 관련도가 높은 순서로만 정렬한다.",
'JSON으로만 응답한다: {"rankings":[{"id":"문서 id","score":0.0-1.0}]}',
].join("\n"),
},
{
role: "user",
content: [
`[query]\n${query}`,
"",
"[candidates]",
...candidates.map((candidate, index) =>
[
`#${index + 1} id=${candidate.id} score=${candidate.score.toFixed(3)}`,
truncate(candidate.text, 600),
].join("\n"),
),
].join("\n"),
},
],
{
jsonMode: true,
maxTokens: 200,
temperature: 0,
tier: 0,
},
);
if (!result?.content.trim()) {
return candidates.slice(0, topN);
}
const byId = new Map(candidates.map((candidate) => [candidate.id, candidate]));
const rankings = parseRankings(result.content);
const ranked = rankings
.flatMap((ranking) => {
const candidate = byId.get(ranking.id);
return candidate ? [{ ...candidate, score: ranking.score }] : [];
})
.slice(0, topN);
const seen = new Set(ranked.map((candidate) => candidate.id));
const next = [
...ranked,
...candidates.filter((candidate) => !seen.has(candidate.id)),
].slice(0, topN);
span.setAttributes({
"memory.reranker.changed_order": next[0]?.id !== candidates[0]?.id,
"memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt),
});
return next;
} catch (error) {
getRequestLogger().warn(
{ err: error },
"[Memory] Reranker failed; using original retrieval order",
);
span.setAttributes({
"memory.reranker.changed_order": false,
"memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt),
});
return candidates.slice(0, topN);
}
},
);
}
}
+7 -1
View File
@@ -10,7 +10,11 @@ import { getRequestLogger } from "@obs/context";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader";
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
import { getRagEnabled } from "../../config.js";
import {
getRagEnabled,
getRagHybridEnabled,
getRagRerankerEnabled,
} from "../../config.js";
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
import { createModerationToolRegistry } from "./tools.js";
@@ -150,6 +154,8 @@ async function lookupReferenceText(
try {
const docs = await memoryService.longTerm.search(message, {
category: "rule",
hybrid: getRagHybridEnabled(),
rerank: getRagRerankerEnabled(),
skill: "moderation",
topK: 3,
});
+2
View File
@@ -66,6 +66,8 @@ export const ONBOARDING_PROMPT_DATA = {
"[답변 원칙]",
"- 마지막 메시지에서 직접 물은 내용에만 정확히 답해.",
"- 사용자가 요청하지 않은 다음 단계, 방법, 위치, 예시는 먼저 붙이지 마.",
"- [user_context] 블록이 있으면 그 안의 정보를 응답 톤과 우선순위에 반영해.",
"- [long_term_memory] 블록이 있으면 사용자의 과거 패턴을 참고하되, 사용자에게 직접 언급하지 마.",
"- 마크다운 문법, 링크, URL은 쓰지 마.",
'- 온보딩과 무관한 잡담이거나 다른 사람에게 하는 말이면 "[NO_RESPONSE]"만 출력해.',
],
+24
View File
@@ -144,6 +144,28 @@ function formatRetrievedKnowledge(context: PromptContext): string[] {
];
}
function formatProfileSnippets(context: PromptContext): string[] {
const snippets = context.profileSnippets ?? [];
if (snippets.length === 0) {
return [];
}
return [
'',
'[user_context]',
snippets.map((snippet) => `- ${snippet.block}`).join('\n'),
];
}
function formatLongTermMemory(context: PromptContext): string[] {
const summary = context.userProfile?.longTermSummary?.text?.trim();
if (!summary) {
return [];
}
return ['', '[long_term_memory]', summary];
}
export function personaLayer(): string {
return ONBOARDING_PROMPT_DATA.persona;
}
@@ -222,6 +244,8 @@ export function contextLayer(context: PromptContext): string {
formatUserProfile(context.userProfile),
]
: []),
...formatProfileSnippets(context),
...formatLongTermMemory(context),
...formatRetrievedKnowledge(context),
].join('\n');
}
@@ -3,6 +3,7 @@ import type { GarnetClient } from '@common/garnet';
import { getConversationSummaryKey } from '@conv/conversation-store';
import type { ChatMessage } from '@llm/types';
import type { MemoryService } from '../../../memory/memory-types.js';
import { deriveProfilePromptSnippets } from '../../../memory/profile-update-policy.js';
import { getRequestLogger } from '@obs/context';
import type { RoomContextService } from '@room/room-context-service';
import { meetsTrustTier } from '../../../eval/governance/trust.js';
@@ -10,6 +11,8 @@ import {
getAgentTrustMinTier,
getAgentUserMemoryEnabled,
getRagEnabled,
getRagHybridEnabled,
getRagRerankerEnabled,
getPromptVariantId,
} from '../../../config.js';
import {
@@ -127,6 +130,8 @@ export async function buildLlmMessages(
query: buildRetrievalQuery(request, derived),
options: {
category: 'onboarding-step',
hybrid: getRagHybridEnabled(),
rerank: getRagRerankerEnabled(),
skill: 'onboarding',
topK: 2,
},
@@ -173,6 +178,7 @@ export async function buildLlmMessages(
throw new Error('Failed to load onboarding blackboard');
}
const userProfile = workingMemory?.userProfile ?? legacyUserProfile;
const context: PromptContext = {
blackboard,
conversationHistory:
@@ -180,10 +186,13 @@ export async function buildLlmMessages(
? workingMemory.conversationHistory
: conversationHistory,
roomContext,
profileSnippets: userProfile
? deriveProfilePromptSnippets(userProfile, derived)
: undefined,
retrievedKnowledge: workingMemory?.retrievedKnowledge,
summary: workingMemory?.summary ?? legacySummary,
userMessage: buildPromptUserMessage(request, derived),
userProfile: workingMemory?.userProfile ?? legacyUserProfile,
userProfile,
};
const builder = PromptBuilder.forDerived(derived, session, context);
+36 -1
View File
@@ -2,6 +2,10 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/ty
import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats';
import type { MemoryService } from '../../memory/memory-types.js';
import {
buildLongTermSummary,
shouldRefreshLongTermSummary,
} from '../../memory/long-term-summarizer.js';
import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { RoomContextService } from '@room/room-context-service';
@@ -427,12 +431,43 @@ export class OnboardingSkill implements Skill {
return;
}
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, {
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: [],
});
if (!shouldRefreshLongTermSummary(profile)) {
return;
}
void this.refreshLongTermSummary(request, profile);
}
private async refreshLongTermSummary(
request: AgentRequest,
profile: Awaited<ReturnType<UserMemoryStore['observeTurn']>>,
): Promise<void> {
try {
const conversationSummary =
(await this.memoryService?.session.getSummary(
request.chatroomId,
request.userId,
)) ?? null;
const longTermSummary = await buildLongTermSummary(
this.llmClient,
profile,
conversationSummary,
);
await this.userMemoryStore.updateProfile(request.userId, {
longTermSummary,
});
} catch (error) {
getRequestLogger().warn(
{ err: error },
'[OnboardingSkill] Failed to refresh long-term user summary',
);
}
}
private makeTemplateFallbackResponse(
+50 -3
View File
@@ -4,6 +4,8 @@ import {
anonymizeText,
} from "@eval/anonymize";
import { getAgentUserMemoryTtlDays } from "../../../config.js";
import { evaluateUserFlags } from "../../../memory/forgetting-policy.js";
import type { LongTermSummary } from "../../../memory/long-term-summarizer.js";
import type { LongTermMemory } from "../../../memory/memory-types.js";
import type { DerivedState, Intent } from "./onboarding-state.js";
@@ -11,9 +13,11 @@ export type PreferredStyle = "concise" | "neutral" | "warm";
export interface UserProfile {
completedOnboardingAt?: string | undefined;
archivedFlaggedIssues?: string[] | undefined;
firstSeenAt: string;
flaggedIssues: string[];
lastSeenAt: string;
longTermSummary?: LongTermSummary | undefined;
pastIntents: Array<{ count: number; kind: Intent["kind"] }>;
preferredStyle: PreferredStyle;
totalTurns: number;
@@ -115,14 +119,32 @@ export class UserMemoryStore {
derived.phase.kind === "completed"
? current.completedOnboardingAt ?? now
: current.completedOnboardingAt;
const next: UserProfile = {
...(completedOnboardingAt ? { completedOnboardingAt } : {}),
firstSeenAt: current.firstSeenAt,
const flagDecision = evaluateUserFlags(
{
...current,
flaggedIssues: sanitizeIssues([
...current.flaggedIssues,
...(options?.flaggedIssues ?? []),
]),
},
10,
);
const next: UserProfile = {
...(current.archivedFlaggedIssues || flagDecision.archived.length > 0
? {
archivedFlaggedIssues: [
...(current.archivedFlaggedIssues ?? []),
...flagDecision.archived,
].slice(-50),
}
: {}),
...(completedOnboardingAt ? { completedOnboardingAt } : {}),
firstSeenAt: current.firstSeenAt,
flaggedIssues: flagDecision.kept,
lastSeenAt: now,
...(current.longTermSummary
? { longTermSummary: current.longTermSummary }
: {}),
pastIntents: mergeIntent(current.pastIntents, derived.intent.kind),
preferredStyle: inferPreferredStyle(
current.preferredStyle,
@@ -140,4 +162,29 @@ export class UserMemoryStore {
}
return next;
}
async updateProfile(
userId: string,
patch: Partial<UserProfile>,
): Promise<UserProfile> {
const current = (await this.load(userId)) ?? emptyProfile(userId);
const next: UserProfile = {
...current,
...patch,
firstSeenAt: patch.firstSeenAt ?? current.firstSeenAt,
flaggedIssues: patch.flaggedIssues ?? current.flaggedIssues,
lastSeenAt: patch.lastSeenAt ?? nowIso(),
pastIntents: patch.pastIntents ?? current.pastIntents,
preferredStyle: patch.preferredStyle ?? current.preferredStyle,
totalTurns: patch.totalTurns ?? current.totalTurns,
userId: current.userId,
};
if (this.longTermMemory) {
await this.longTermMemory.updateUserProfile(userId, next);
} else {
await this.garnetClient.setJson(profileKey(userId), next, ttlSeconds());
}
return next;
}
}
+2
View File
@@ -1,4 +1,5 @@
import type { ConversationMessage } from "@agent/types";
import type { ProfilePromptSnippet } from "../../memory/profile-update-policy.js";
import type { RetrievedDocument } from "../../memory/memory-types.js";
import type { RoomMessage } from "@room/types";
import type { UserProfile } from "./state/user-memory.js";
@@ -27,6 +28,7 @@ export interface Example {
export interface PromptContext {
blackboard: Blackboard;
conversationHistory: ConversationMessage[];
profileSnippets?: ProfilePromptSnippet[] | undefined;
roomContext?: RoomMessage[] | undefined;
retrievedKnowledge?: RetrievedDocument[] | undefined;
summary?: string | undefined;
+2
View File
@@ -2,12 +2,14 @@ import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import type { ShadowEvalBus } from "@eval/shadow-bus";
import type { LlmClient } from "@llm/llm-client";
import type { EscalationHistoryService } from "../memory/escalation-history.js";
import type { MemoryService } from "../memory/memory-types.js";
import type { RoomContextService } from "@room/room-context-service";
import type { Skill } from "./skill.js";
import type { SkillRegistry } from "./skill-loader.js";
export interface SkillDeps {
escalationHistoryService?: EscalationHistoryService | undefined;
garnetClient: GarnetClient;
llmClient: LlmClient;
memoryService?: MemoryService | undefined;
@@ -1,4 +1,5 @@
import type { HandoffPacket } from "@agent/handoff-packet";
import type { UserEscalationHistory } from "../../memory/escalation-history.js";
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
function formatSkill(manifest: SkillManifest): string {
@@ -14,16 +15,35 @@ function formatSkill(manifest: SkillManifest): string {
export function buildSupervisorSystemPrompt(
registry: SkillRegistry,
_packet?: HandoffPacket,
history?: UserEscalationHistory | null,
): string {
const skills = registry
.list()
.filter((manifest) => manifest.name !== "supervisor");
const historyBlock = history
? [
"",
"[user_escalation_history]",
`총 escalation: ${history.totalEscalations}`,
`마지막 escalation: ${history.lastEscalationAt}`,
`top reasons: ${history.topReasons.map((item) => `${item.reason}(${item.count})`).join(", ") || "-"}`,
`recent resolutions: ${
history.escalations
.slice(-3)
.map((item) => item.resolution?.action)
.filter(Boolean)
.join(", ") || "-"
}`,
"- 이 블록은 라우팅 참고용이다. 사용자에게 직접 언급하지 않는다.",
]
: [];
return [
"너는 multi-agent 시스템의 supervisor다.",
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
"",
"[사용 가능한 specialist]",
...skills.map(formatSkill),
...historyBlock,
"",
"[응답 형식]",
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}',
+1
View File
@@ -10,5 +10,6 @@ export function createSupervisorSkill(deps: SkillDeps): Skill {
deps.natsClient,
deps.llmClient,
deps.skillRegistry,
deps.escalationHistoryService,
);
}
+15 -3
View File
@@ -1,6 +1,7 @@
import type { HandoffPacket } from "@agent/handoff-packet";
import type { HandoffReason } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet";
import type { EscalationHistoryService } from "../../memory/escalation-history.js";
import {
getSupervisorResolutionsStreamMaxLen,
getSupervisorReviewQueueMaxLen,
@@ -152,13 +153,13 @@ function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean
export async function appendSupervisorReview(
garnetClient: Pick<GarnetClient, "appendStream">,
fields: Record<string, string>,
): Promise<void> {
await garnetClient.appendStream(
): Promise<string> {
return garnetClient.appendStream(
SUPERVISOR_REVIEW_STREAM_KEY,
{
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
queuedAt: fields["queuedAt"] ?? new Date().toISOString(),
},
{ maxLen: getSupervisorReviewQueueMaxLen() },
);
@@ -275,6 +276,10 @@ export async function resolveSupervisorReview(
garnetClient: SupervisorReviewResolveGarnetClient,
id: string,
resolution: SupervisorReviewResolution,
options?: {
escalationHistoryService?: EscalationHistoryService | undefined;
userId?: string | undefined;
},
): Promise<void> {
const fields: Record<string, string> = {
action: resolution.action,
@@ -297,6 +302,13 @@ export async function resolveSupervisorReview(
},
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
);
if (options?.escalationHistoryService && options.userId) {
await options.escalationHistoryService.recordResolution({
userId: options.userId,
reviewId: id,
resolution,
});
}
}
export async function getResolutionHistory(
+42 -3
View File
@@ -8,6 +8,10 @@ import type { HandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import type { LlmClient } from "@llm/llm-client";
import type {
EscalationHistoryService,
UserEscalationHistory,
} from "../../memory/escalation-history.js";
import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics";
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
@@ -87,6 +91,7 @@ export class SupervisorSkill implements Skill {
natsClient: Pick<NatsClient, "publish">,
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
private readonly skillRegistry?: SkillRegistry | undefined,
private readonly escalationHistoryService?: EscalationHistoryService | undefined,
) {
this.alertService = new AlertService(
garnetClient,
@@ -108,7 +113,8 @@ export class SupervisorSkill implements Skill {
_conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const packet = readHandoffPacket(request);
const llmDecision = await this.routeByLlm(packet);
const escalationHistory = await this.loadEscalationHistory(request.userId);
const llmDecision = await this.routeByLlm(packet, escalationHistory);
const useLlmRoute =
llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
@@ -117,14 +123,26 @@ export class SupervisorSkill implements Skill {
: routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope";
await appendSupervisorReview(this.garnetClient, {
const queuedAt = new Date().toISOString();
const reviewId = await appendSupervisorReview(this.garnetClient, {
chatroomId: request.chatroomId,
packet: JSON.stringify(packet ?? null),
queuedAt,
reason,
requestId: request.requestId,
target: target ?? "human-review",
userId: request.userId,
});
await this.escalationHistoryService?.recordEscalation({
chatroomId: request.chatroomId,
packetSummary: packet?.userGoal ?? packet?.summary ?? "",
queuedAt,
reason,
reviewId,
sourceChain: packet?.sourceChain ?? [],
target: target ?? "human-review",
userId: request.userId,
});
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
await this.alertService.sendAlert(
@@ -154,6 +172,7 @@ export class SupervisorSkill implements Skill {
routedBy: useLlmRoute ? "llm" : "rules",
source: userMessage.source,
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
suggestedTarget: target,
},
requestId: request.requestId,
@@ -164,6 +183,7 @@ export class SupervisorSkill implements Skill {
private async routeByLlm(
packet?: HandoffPacket,
escalationHistory?: UserEscalationHistory | null,
): Promise<SupervisorRoutingDecision | null> {
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
return null;
@@ -173,7 +193,11 @@ export class SupervisorSkill implements Skill {
const result = await this.llmClient.chatCompletion(
[
{
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
content: buildSupervisorSystemPrompt(
this.skillRegistry,
packet,
escalationHistory,
),
role: "system",
},
{
@@ -213,4 +237,19 @@ export class SupervisorSkill implements Skill {
target === "human-review" || Boolean(this.skillRegistry?.get(target))
);
}
private async loadEscalationHistory(userId: string) {
if (!this.escalationHistoryService) {
return null;
}
try {
return await this.escalationHistoryService.load(userId);
} catch (error) {
getLogger().warn(
{ err: error },
"[Supervisor] Failed to load escalation history",
);
return null;
}
}
}
+7 -1
View File
@@ -1,7 +1,11 @@
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { z } from "zod";
import { getRagEnabled } from "../../config.js";
import {
getRagEnabled,
getRagHybridEnabled,
getRagRerankerEnabled,
} from "../../config.js";
import { getRequestLogger } from "../../observability/context.js";
import type { Tool } from "../tool.js";
@@ -96,6 +100,8 @@ export const lookupRuleTool: Tool<
try {
const docs = await ctx.memoryService.longTerm.search(args.topic, {
category: "rule",
hybrid: getRagHybridEnabled(),
rerank: getRagRerankerEnabled(),
skill: "moderation",
topK: args.topK ?? 3,
});
@@ -1,5 +1,9 @@
import { z } from "zod";
import { getRagEnabled } from "../../config.js";
import {
getRagEnabled,
getRagHybridEnabled,
getRagRerankerEnabled,
} from "../../config.js";
import { getRequestLogger } from "../../observability/context.js";
import {
DEADLINE_NEAR_THRESHOLD_MIN,
@@ -93,6 +97,8 @@ export const lookupCommunityRuleTool: Tool<
try {
const docs = await ctx.memoryService.longTerm.search(args.topic, {
category: "onboarding-step",
hybrid: getRagHybridEnabled(),
rerank: getRagRerankerEnabled(),
skill: "onboarding",
topK: args.topK ?? 3,
});