Add hybrid memory retrieval and escalation history
This commit is contained in:
1 parent
1a86bded0b
commit
211de18476
38 files changed
+1595
-30
No files matched your search
@@ -31,6 +31,7 @@
|
|||||||
"eval:tournament:moderation": "node --import tsx src/eval/cli.ts tournament --skill=moderation",
|
"eval:tournament:moderation": "node --import tsx src/eval/cli.ts tournament --skill=moderation",
|
||||||
"eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results",
|
"eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results",
|
||||||
"ingest:knowledge": "node --import tsx scripts/ingest-knowledge.ts",
|
"ingest:knowledge": "node --import tsx scripts/ingest-knowledge.ts",
|
||||||
|
"memory:evict": "node --import tsx scripts/memory-eviction-cron.ts",
|
||||||
"review:list": "node --import tsx src/operations/cli.ts review:list",
|
"review:list": "node --import tsx src/operations/cli.ts review:list",
|
||||||
"review:show": "node --import tsx src/operations/cli.ts review:show",
|
"review:show": "node --import tsx src/operations/cli.ts review:show",
|
||||||
"review:resolve": "node --import tsx src/operations/cli.ts review:resolve",
|
"review:resolve": "node --import tsx src/operations/cli.ts review:resolve",
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
import { loadEnvironmentFromNodeEnv } from "../src/common/env-loader.js";
|
||||||
|
|
||||||
|
loadEnvironmentFromNodeEnv();
|
||||||
|
|
||||||
|
const { getRawSecret } = await import("../src/common/secret-manager.js");
|
||||||
|
const { initMongoDB, closeMongoDB } = await import("../src/common/mongodb.js");
|
||||||
|
const { initLogger, getLogger } = await import("../src/observability/logger.js");
|
||||||
|
const { getMemoryEvictionEnabled } = await import("../src/config.js");
|
||||||
|
const { KNOWLEDGE_COLLECTION } = await import("../src/memory/mongo-long-term-memory.js");
|
||||||
|
const { detectContradictions } = await import("../src/memory/contradiction-detector.js");
|
||||||
|
const { evaluateDocument } = await import("../src/memory/forgetting-policy.js");
|
||||||
|
|
||||||
|
initLogger();
|
||||||
|
const logger = getLogger();
|
||||||
|
|
||||||
|
interface KnowledgeRecord {
|
||||||
|
_id: string;
|
||||||
|
category: string;
|
||||||
|
text: string;
|
||||||
|
metadata?: Record<string, unknown> | undefined;
|
||||||
|
lastReferencedAt?: Date | undefined;
|
||||||
|
references?: Date[] | undefined;
|
||||||
|
updatedAt?: Date | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function firstNonEmpty(
|
||||||
|
...values: Array<string | null | undefined>
|
||||||
|
): string | undefined {
|
||||||
|
for (const value of values) {
|
||||||
|
if (value?.trim()) {
|
||||||
|
return value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSinceDays(): number {
|
||||||
|
const raw = process.argv.find((arg) => arg.startsWith("--since="))?.split("=")[1];
|
||||||
|
if (!raw) {
|
||||||
|
return 30;
|
||||||
|
}
|
||||||
|
const match = raw.match(/^(\d+)d$/u);
|
||||||
|
return match ? Number(match[1]) : 30;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dryRun = process.argv.includes("--dry-run");
|
||||||
|
const target = process.argv.find((arg) => arg.startsWith("--target="))?.split("=")[1] ?? "all";
|
||||||
|
const sinceDays = parseSinceDays();
|
||||||
|
|
||||||
|
if (!dryRun && !getMemoryEvictionEnabled()) {
|
||||||
|
throw new Error(
|
||||||
|
"AGENT_MEMORY_EVICTION_ENABLED must be true for non-dry-run eviction",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (target !== "knowledge" && target !== "all") {
|
||||||
|
logger.info({ dryRun, target }, "memory eviction skipped: target has no Mongo knowledge work");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const connectionString = firstNonEmpty(
|
||||||
|
getRawSecret("MONGODB_CONNECTION_STRING"),
|
||||||
|
process.env["MONGODB_CONNECTION_STRING"],
|
||||||
|
);
|
||||||
|
if (!connectionString) {
|
||||||
|
if (dryRun) {
|
||||||
|
logger.warn(
|
||||||
|
{ dryRun },
|
||||||
|
"MONGODB_CONNECTION_STRING is not configured; memory eviction dry-run skipped",
|
||||||
|
);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
throw new Error("MONGODB_CONNECTION_STRING is required for memory eviction");
|
||||||
|
}
|
||||||
|
|
||||||
|
const connection = await initMongoDB(connectionString);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const collection = connection.collection<KnowledgeRecord>(KNOWLEDGE_COLLECTION);
|
||||||
|
const docs = await collection
|
||||||
|
.find({ "metadata.archived": { $ne: true } })
|
||||||
|
.sort({ updatedAt: 1 })
|
||||||
|
.toArray();
|
||||||
|
const contradictions = detectContradictions(
|
||||||
|
docs.map((doc) => ({
|
||||||
|
id: doc._id,
|
||||||
|
metadata: doc.metadata,
|
||||||
|
text: doc.text,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
const contradictionByExistingId = new Map(
|
||||||
|
contradictions.map((candidate) => [candidate.existingId, candidate.newId]),
|
||||||
|
);
|
||||||
|
const now = new Date();
|
||||||
|
const sinceMs = now.getTime() - sinceDays * 24 * 60 * 60 * 1000;
|
||||||
|
let archived = 0;
|
||||||
|
let kept = 0;
|
||||||
|
|
||||||
|
for (const doc of docs) {
|
||||||
|
const references = (doc.references ?? []).filter((date) => {
|
||||||
|
const time = new Date(date).getTime();
|
||||||
|
return Number.isFinite(time) && time >= sinceMs;
|
||||||
|
});
|
||||||
|
const decision = evaluateDocument(
|
||||||
|
{
|
||||||
|
id: doc._id,
|
||||||
|
category: doc.category,
|
||||||
|
metadata: doc.metadata,
|
||||||
|
text: doc.text,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
contradictsId: contradictionByExistingId.get(doc._id),
|
||||||
|
lastReferencedAt: doc.lastReferencedAt ? new Date(doc.lastReferencedAt) : undefined,
|
||||||
|
now,
|
||||||
|
referenceCount30d: references.length,
|
||||||
|
updatedAt: doc.updatedAt ? new Date(doc.updatedAt) : undefined,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (decision.action === "keep") {
|
||||||
|
kept += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
archived += 1;
|
||||||
|
if (!dryRun) {
|
||||||
|
await collection.updateOne(
|
||||||
|
{ _id: doc._id },
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
"metadata.archived": true,
|
||||||
|
"metadata.archivedAt": now.toISOString(),
|
||||||
|
"metadata.archiveReason": decision.reason,
|
||||||
|
"metadata.archiveEvidence": decision.evidence,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
archived,
|
||||||
|
contradictions: contradictions.length,
|
||||||
|
dryRun,
|
||||||
|
kept,
|
||||||
|
scanned: docs.length,
|
||||||
|
sinceDays,
|
||||||
|
},
|
||||||
|
"memory eviction completed",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await closeMongoDB();
|
||||||
|
}
|
||||||
@@ -7,8 +7,13 @@ import {
|
|||||||
getAlertThresholdP95LatencyMs,
|
getAlertThresholdP95LatencyMs,
|
||||||
getEmbeddingModel,
|
getEmbeddingModel,
|
||||||
getEmbeddingProvider,
|
getEmbeddingProvider,
|
||||||
|
getLongTermSummaryEnabled,
|
||||||
|
getMemoryEvictionEnabled,
|
||||||
getPhoenixLogPrompts,
|
getPhoenixLogPrompts,
|
||||||
getPhoenixOtlpEndpoint,
|
getPhoenixOtlpEndpoint,
|
||||||
|
getRagHybridEnabled,
|
||||||
|
getRagRerankerEnabled,
|
||||||
|
getRagRerankerTopK,
|
||||||
getRagEnabled,
|
getRagEnabled,
|
||||||
getSlackAlertWebhookUrl,
|
getSlackAlertWebhookUrl,
|
||||||
getSlackWebhookTimeoutMs,
|
getSlackWebhookTimeoutMs,
|
||||||
@@ -25,6 +30,11 @@ const ENV_KEYS = [
|
|||||||
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
||||||
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
||||||
"AGENT_RAG_ENABLED",
|
"AGENT_RAG_ENABLED",
|
||||||
|
"AGENT_RAG_HYBRID_ENABLED",
|
||||||
|
"AGENT_RAG_RERANKER_ENABLED",
|
||||||
|
"AGENT_RAG_RERANKER_TOP_K",
|
||||||
|
"AGENT_LONG_TERM_SUMMARY_ENABLED",
|
||||||
|
"AGENT_MEMORY_EVICTION_ENABLED",
|
||||||
"EMBEDDING_MODEL",
|
"EMBEDDING_MODEL",
|
||||||
"EMBEDDING_PROVIDER",
|
"EMBEDDING_PROVIDER",
|
||||||
"PHOENIX_LOG_PROMPTS",
|
"PHOENIX_LOG_PROMPTS",
|
||||||
@@ -85,14 +95,29 @@ describe("config", () => {
|
|||||||
|
|
||||||
it("reads RAG and embedding configuration", () => {
|
it("reads RAG and embedding configuration", () => {
|
||||||
expect(getRagEnabled()).toBe(true);
|
expect(getRagEnabled()).toBe(true);
|
||||||
|
expect(getRagHybridEnabled()).toBe(true);
|
||||||
|
expect(getRagRerankerEnabled()).toBe(true);
|
||||||
|
expect(getRagRerankerTopK()).toBe(5);
|
||||||
|
expect(getLongTermSummaryEnabled()).toBe(true);
|
||||||
|
expect(getMemoryEvictionEnabled()).toBe(false);
|
||||||
expect(getEmbeddingProvider()).toBe("openrouter");
|
expect(getEmbeddingProvider()).toBe("openrouter");
|
||||||
expect(getEmbeddingModel()).toBe("openai/text-embedding-3-small");
|
expect(getEmbeddingModel()).toBe("openai/text-embedding-3-small");
|
||||||
|
|
||||||
vi.stubEnv("AGENT_RAG_ENABLED", "false");
|
vi.stubEnv("AGENT_RAG_ENABLED", "false");
|
||||||
|
vi.stubEnv("AGENT_RAG_HYBRID_ENABLED", "false");
|
||||||
|
vi.stubEnv("AGENT_RAG_RERANKER_ENABLED", "false");
|
||||||
|
vi.stubEnv("AGENT_RAG_RERANKER_TOP_K", "2");
|
||||||
|
vi.stubEnv("AGENT_LONG_TERM_SUMMARY_ENABLED", "false");
|
||||||
|
vi.stubEnv("AGENT_MEMORY_EVICTION_ENABLED", "true");
|
||||||
vi.stubEnv("EMBEDDING_PROVIDER", "openai");
|
vi.stubEnv("EMBEDDING_PROVIDER", "openai");
|
||||||
vi.stubEnv("EMBEDDING_MODEL", "custom-embedding");
|
vi.stubEnv("EMBEDDING_MODEL", "custom-embedding");
|
||||||
|
|
||||||
expect(getRagEnabled()).toBe(false);
|
expect(getRagEnabled()).toBe(false);
|
||||||
|
expect(getRagHybridEnabled()).toBe(false);
|
||||||
|
expect(getRagRerankerEnabled()).toBe(false);
|
||||||
|
expect(getRagRerankerTopK()).toBe(2);
|
||||||
|
expect(getLongTermSummaryEnabled()).toBe(false);
|
||||||
|
expect(getMemoryEvictionEnabled()).toBe(true);
|
||||||
expect(getEmbeddingProvider()).toBe("openai");
|
expect(getEmbeddingProvider()).toBe("openai");
|
||||||
expect(getEmbeddingModel()).toBe("custom-embedding");
|
expect(getEmbeddingModel()).toBe("custom-embedding");
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -133,6 +133,30 @@ export function getRagEnabled(): boolean {
|
|||||||
return getBoolean("AGENT_RAG_ENABLED", true);
|
return getBoolean("AGENT_RAG_ENABLED", true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getRagHybridEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_RAG_HYBRID_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getRagRerankerEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_RAG_RERANKER_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getRagRerankerTopK(): number {
|
||||||
|
return getPositiveNumber("AGENT_RAG_RERANKER_TOP_K", 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getRagCandidatePoolSize(): number {
|
||||||
|
return getPositiveNumber("AGENT_RAG_CANDIDATE_POOL_SIZE", 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getLongTermSummaryEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_LONG_TERM_SUMMARY_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getMemoryEvictionEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_MEMORY_EVICTION_ENABLED", false);
|
||||||
|
}
|
||||||
|
|
||||||
export function getEmbeddingProvider(): "openrouter" | "openai" {
|
export function getEmbeddingProvider(): "openrouter" | "openai" {
|
||||||
const raw = process.env["EMBEDDING_PROVIDER"]?.trim().toLowerCase();
|
const raw = process.env["EMBEDDING_PROVIDER"]?.trim().toLowerCase();
|
||||||
return raw === "openai" ? "openai" : "openrouter";
|
return raw === "openai" ? "openai" : "openrouter";
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
{"id":"memory-retrieval-accurate","name":"retrieval-accurate","input":{"request":{"requestId":"mem_mod_ads","skill":"moderation","chatroomId":"memory_room","userId":"memory_user_1","nickname":"tester","trigger":"user-message","userMessage":"광고 도배 기준이 뭐예요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["광고","도배"]}}
|
||||||
|
{"id":"memory-long-range-understanding","name":"long-range-understanding","input":{"request":{"requestId":"mem_onboarding_recall","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_2","nickname":"LolName_KR/95/남/D","trigger":"user-message","userMessage":"내 닉네임 뭐라고 했지?"},"conversationHistory":[{"role":"user","content":"디스코드 닉네임은 LolName_KR입니다","timestamp":"2026-05-01T00:00:00.000Z"},{"role":"assistant","content":"확인했어요.","timestamp":"2026-05-01T00:00:01.000Z"}],"summary":"사용자가 디스코드 닉네임은 LolName_KR이라고 말했다."},"expected":{"mustIncludeAny":["LolName_KR"],"expectedMemory":{"expectedRecalledFact":"LolName_KR"}}}
|
||||||
|
{"id":"memory-test-time-learning","name":"test-time-learning","input":{"request":{"requestId":"mem_onboarding_hint","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_3","nickname":"낭만/00/남D","trigger":"nickname-invalid"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":["닉네임 형식 오류","닉네임 형식 오류","닉네임 형식 오류"],"pastIntents":[{"kind":"nickname-question","count":5}],"preferredStyle":"concise","totalTurns":12,"userId":"user_memory"}},"expected":{"mustIncludeAny":["예시","형식"],"expectedMemory":{"expectedProphylacticHint":true}}}
|
||||||
|
{"id":"memory-selective-forgetting","name":"selective-forgetting","input":{"request":{"requestId":"mem_forgetting","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_4","nickname":"tester","trigger":"user-message","userMessage":"닉네임 규칙 최신 기준 알려줘"},"summary":"정정된 최신 규칙만 답해야 한다."},"expected":{"mustNotInclude":["예전 기준"],"expectedMemory":{"expectedArchivedDocPattern":"예전 기준"}}}
|
||||||
|
{"id":"memory-escalation-history","name":"escalation-history","input":{"request":{"requestId":"mem_supervisor_history","skill":"supervisor","chatroomId":"memory_room","userId":"memory_user_5","nickname":"tester","trigger":"user-message","context":{"handoffPacket":{"constraints":[],"currentState":{},"evidence":[],"expectedOutputSchema":"text","handoffReason":"out-of-scope","sourceChain":["onboarding"],"summary":"광고성 메시지로 보임","userGoal":"광고 홍보 가능 여부 확인"}}}},"expected":{"expectedMemory":{"expectedSupervisorTarget":"moderation"}}}
|
||||||
|
{"id":"memory-contradiction","name":"memory-contradiction","input":{"request":{"requestId":"mem_contradiction","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_6","nickname":"새닉/95/남/D","trigger":"user-message","userMessage":"닉네임 새닉으로 변경했어요"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":[],"pastIntents":[{"kind":"nickname-question","count":4}],"preferredStyle":"neutral","totalTurns":8,"userId":"user_memory","longTermSummary":{"text":"사용자는 이전 닉네임에서 새닉으로 변경했다고 말했다.","updatedAt":"2026-05-18T00:00:00.000Z","turnSnapshot":8}}},"expected":{"mustIncludeAny":["새닉"],"expectedMemory":{"expectedContradictionResolved":true,"expectedRecalledFact":"새닉"}}}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import type { AgentResponse } from "@agent/types";
|
||||||
|
import type { RetrievedDocument } from "../../memory/memory-types.js";
|
||||||
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
|
|
||||||
|
export interface MemoryJudgeExpected {
|
||||||
|
expectedArchivedDocPattern?: string | undefined;
|
||||||
|
expectedContradictionResolved?: boolean | undefined;
|
||||||
|
expectedProphylacticHint?: boolean | undefined;
|
||||||
|
expectedRecalledFact?: string | undefined;
|
||||||
|
expectedRetrievedDocIds?: string[] | undefined;
|
||||||
|
expectedSupervisorTarget?: string | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MemoryJudgeResult {
|
||||||
|
checks: Array<{
|
||||||
|
detail?: string | undefined;
|
||||||
|
name: keyof MemoryJudgeExpected;
|
||||||
|
passed: boolean;
|
||||||
|
skipped: boolean;
|
||||||
|
}>;
|
||||||
|
passed: boolean;
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function judgeMemory(
|
||||||
|
expected: MemoryJudgeExpected | undefined,
|
||||||
|
response: AgentResponse,
|
||||||
|
metadata?: {
|
||||||
|
profile?: UserProfile | undefined;
|
||||||
|
retrievedDocs?: RetrievedDocument[] | undefined;
|
||||||
|
},
|
||||||
|
): MemoryJudgeResult {
|
||||||
|
if (!expected || Object.keys(expected).length === 0) {
|
||||||
|
return { checks: [], passed: true, score: 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
const checks: MemoryJudgeResult["checks"] = [];
|
||||||
|
if (expected.expectedRecalledFact) {
|
||||||
|
checks.push({
|
||||||
|
name: "expectedRecalledFact",
|
||||||
|
passed: response.text.includes(expected.expectedRecalledFact),
|
||||||
|
skipped: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (expected.expectedProphylacticHint !== undefined) {
|
||||||
|
const hasHint = /(예시|형식|가이드|주의|먼저)/u.test(response.text);
|
||||||
|
checks.push({
|
||||||
|
name: "expectedProphylacticHint",
|
||||||
|
passed: expected.expectedProphylacticHint ? hasHint : !hasHint,
|
||||||
|
skipped: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (expected.expectedSupervisorTarget) {
|
||||||
|
const suggestedTarget = response.metadata?.["suggestedTarget"];
|
||||||
|
checks.push({
|
||||||
|
detail: typeof suggestedTarget === "string" ? suggestedTarget : "",
|
||||||
|
name: "expectedSupervisorTarget",
|
||||||
|
passed: suggestedTarget === expected.expectedSupervisorTarget,
|
||||||
|
skipped: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (expected.expectedRetrievedDocIds) {
|
||||||
|
const ids = new Set(metadata?.retrievedDocs?.map((doc) => doc.id) ?? []);
|
||||||
|
checks.push({
|
||||||
|
name: "expectedRetrievedDocIds",
|
||||||
|
passed: expected.expectedRetrievedDocIds.every((id) => ids.has(id)),
|
||||||
|
skipped: ids.size === 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (expected.expectedArchivedDocPattern) {
|
||||||
|
checks.push({
|
||||||
|
name: "expectedArchivedDocPattern",
|
||||||
|
passed: !response.text.includes(expected.expectedArchivedDocPattern),
|
||||||
|
skipped: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (expected.expectedContradictionResolved !== undefined) {
|
||||||
|
checks.push({
|
||||||
|
name: "expectedContradictionResolved",
|
||||||
|
passed: expected.expectedContradictionResolved
|
||||||
|
? !/(예전|이전|기존)/u.test(response.text)
|
||||||
|
: true,
|
||||||
|
skipped: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeChecks = checks.filter((check) => !check.skipped);
|
||||||
|
const passedCount = activeChecks.filter((check) => check.passed).length;
|
||||||
|
const score = activeChecks.length === 0 ? 1 : passedCount / activeChecks.length;
|
||||||
|
return {
|
||||||
|
checks,
|
||||||
|
passed: score === 1,
|
||||||
|
score,
|
||||||
|
};
|
||||||
|
}
|
||||||
+47
-3
@@ -7,6 +7,7 @@ import { GarnetClient, type StreamEntry } from "@common/garnet";
|
|||||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||||
import { NatsClient } from "@common/nats";
|
import { NatsClient } from "@common/nats";
|
||||||
import { getConversationSummaryKey } from "@conv/conversation-store";
|
import { getConversationSummaryKey } from "@conv/conversation-store";
|
||||||
|
import { anonymizeIdentifier } from "@eval/anonymize";
|
||||||
import { LlmClient } from "@llm/llm-client";
|
import { LlmClient } from "@llm/llm-client";
|
||||||
import type { ProviderName } from "@llm/types";
|
import type { ProviderName } from "@llm/types";
|
||||||
import { runWithContext } from "@obs/context";
|
import { runWithContext } from "@obs/context";
|
||||||
@@ -34,6 +35,7 @@ import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js
|
|||||||
import { runEvalDispatch } from "./handoff-dispatch.js";
|
import { runEvalDispatch } from "./handoff-dispatch.js";
|
||||||
import { judgeHandoff } from "./judges/handoff-judge.js";
|
import { judgeHandoff } from "./judges/handoff-judge.js";
|
||||||
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||||
|
import { judgeMemory } from "./judges/memory-judge.js";
|
||||||
import { judgeProcessRule } from "./judges/process-judge.js";
|
import { judgeProcessRule } from "./judges/process-judge.js";
|
||||||
import { judgeRuleCase } from "./judges/rule-judge.js";
|
import { judgeRuleCase } from "./judges/rule-judge.js";
|
||||||
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||||
@@ -50,6 +52,7 @@ import type {
|
|||||||
EvalSummaryDelta,
|
EvalSummaryDelta,
|
||||||
} from "./types.js";
|
} from "./types.js";
|
||||||
import { createMemoryService } from "../memory/memory-service.js";
|
import { createMemoryService } from "../memory/memory-service.js";
|
||||||
|
import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory";
|
||||||
|
|
||||||
const DEFAULT_REPORTS_DIR = "eval-reports";
|
const DEFAULT_REPORTS_DIR = "eval-reports";
|
||||||
const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
|
const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
|
||||||
@@ -71,7 +74,7 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
|
|||||||
moderation: (deps) =>
|
moderation: (deps) =>
|
||||||
new ModerationSkill(
|
new ModerationSkill(
|
||||||
deps.llm,
|
deps.llm,
|
||||||
createMemoryService({ garnetClient: deps.garnet }),
|
createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }),
|
||||||
),
|
),
|
||||||
onboarding: (deps) =>
|
onboarding: (deps) =>
|
||||||
new OnboardingSkill(
|
new OnboardingSkill(
|
||||||
@@ -80,7 +83,7 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
|
|||||||
deps.room,
|
deps.room,
|
||||||
noopShadowEvalPublisher,
|
noopShadowEvalPublisher,
|
||||||
undefined,
|
undefined,
|
||||||
createMemoryService({ garnetClient: deps.garnet }),
|
createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }),
|
||||||
),
|
),
|
||||||
supervisor: (deps) =>
|
supervisor: (deps) =>
|
||||||
new SupervisorSkill(deps.garnet, {
|
new SupervisorSkill(deps.garnet, {
|
||||||
@@ -451,6 +454,9 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
|
|||||||
if (mustNotInclude) {
|
if (mustNotInclude) {
|
||||||
expected.mustNotInclude = mustNotInclude;
|
expected.mustNotInclude = mustNotInclude;
|
||||||
}
|
}
|
||||||
|
if (isRecord(raw["expectedMemory"])) {
|
||||||
|
expected.expectedMemory = raw["expectedMemory"] as EvalCaseExpected["expectedMemory"];
|
||||||
|
}
|
||||||
|
|
||||||
const processExpectedRaw = raw["processExpected"];
|
const processExpectedRaw = raw["processExpected"];
|
||||||
if (isRecord(processExpectedRaw)) {
|
if (isRecord(processExpectedRaw)) {
|
||||||
@@ -569,6 +575,10 @@ function parseEvalCaseLine(
|
|||||||
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
|
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (raw.input["userProfile"] === null || isRecord(raw.input["userProfile"])) {
|
||||||
|
input.userProfile = raw.input["userProfile"] as EvalCaseInput["userProfile"];
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
id:
|
id:
|
||||||
typeof raw.id === "string" && raw.id.trim()
|
typeof raw.id === "string" && raw.id.trim()
|
||||||
@@ -646,6 +656,15 @@ async function hydrateEvalState(
|
|||||||
input.summary,
|
input.summary,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (input.userProfile !== undefined) {
|
||||||
|
await garnetClient.setJson(
|
||||||
|
`${USER_MEMORY_KEY_PREFIX}:${
|
||||||
|
anonymizeIdentifier("user", userId) ?? "unknown"
|
||||||
|
}`,
|
||||||
|
input.userProfile,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
||||||
@@ -688,7 +707,11 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function isCasePassed(result: EvalCaseResult): boolean {
|
function isCasePassed(result: EvalCaseResult): boolean {
|
||||||
return result.ruleJudge.passed && (result.handoffJudge?.passed ?? true);
|
return (
|
||||||
|
result.ruleJudge.passed &&
|
||||||
|
(result.handoffJudge?.passed ?? true) &&
|
||||||
|
(result.memoryJudge?.passed ?? true)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createBaselineDelta(
|
function createBaselineDelta(
|
||||||
@@ -943,6 +966,13 @@ async function evaluateOnboardingCase(
|
|||||||
processTraceSnapshot,
|
processTraceSnapshot,
|
||||||
judgeConfig.toolRegistryFactory(),
|
judgeConfig.toolRegistryFactory(),
|
||||||
);
|
);
|
||||||
|
const memoryJudge = judgeMemory(
|
||||||
|
evalCase.expected.expectedMemory,
|
||||||
|
response,
|
||||||
|
{
|
||||||
|
profile: input.userProfile ?? undefined,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
id: evalCase.id,
|
id: evalCase.id,
|
||||||
@@ -955,6 +985,7 @@ async function evaluateOnboardingCase(
|
|||||||
response,
|
response,
|
||||||
ruleJudge,
|
ruleJudge,
|
||||||
llmJudge,
|
llmJudge,
|
||||||
|
memoryJudge,
|
||||||
processJudge,
|
processJudge,
|
||||||
processTrace: processTraceSnapshot,
|
processTrace: processTraceSnapshot,
|
||||||
};
|
};
|
||||||
@@ -1033,6 +1064,10 @@ async function evaluateHandoffCase(
|
|||||||
result.response,
|
result.response,
|
||||||
result.finalSkill,
|
result.finalSkill,
|
||||||
);
|
);
|
||||||
|
const memoryJudge = judgeMemory(
|
||||||
|
evalCase.expected.expectedMemory,
|
||||||
|
result.response,
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
id: evalCase.id,
|
id: evalCase.id,
|
||||||
@@ -1060,6 +1095,7 @@ async function evaluateHandoffCase(
|
|||||||
reason: "handoff-eval",
|
reason: "handoff-eval",
|
||||||
},
|
},
|
||||||
handoffJudge,
|
handoffJudge,
|
||||||
|
memoryJudge,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const latencyMs = Date.now() - startedAt;
|
const latencyMs = Date.now() - startedAt;
|
||||||
@@ -1129,6 +1165,13 @@ async function evaluateGenericSkillCase(
|
|||||||
input.conversationHistory ?? [],
|
input.conversationHistory ?? [],
|
||||||
);
|
);
|
||||||
const latencyMs = Date.now() - startedAt;
|
const latencyMs = Date.now() - startedAt;
|
||||||
|
const memoryJudge = judgeMemory(
|
||||||
|
evalCase.expected.expectedMemory,
|
||||||
|
response,
|
||||||
|
{
|
||||||
|
profile: input.userProfile ?? undefined,
|
||||||
|
},
|
||||||
|
);
|
||||||
return {
|
return {
|
||||||
id: evalCase.id,
|
id: evalCase.id,
|
||||||
lineNumber: evalCase.lineNumber,
|
lineNumber: evalCase.lineNumber,
|
||||||
@@ -1160,6 +1203,7 @@ async function evaluateGenericSkillCase(
|
|||||||
skipped: true,
|
skipped: true,
|
||||||
reason: "disabled",
|
reason: "disabled",
|
||||||
},
|
},
|
||||||
|
memoryJudge,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const latencyMs = Date.now() - startedAt;
|
const latencyMs = Date.now() - startedAt;
|
||||||
|
|||||||
@@ -5,12 +5,17 @@ import type {
|
|||||||
HandoffReason,
|
HandoffReason,
|
||||||
} from "@agent/types";
|
} from "@agent/types";
|
||||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||||
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||||
import type {
|
import type {
|
||||||
ProcessTraceSnapshot,
|
ProcessTraceSnapshot,
|
||||||
ProcessTraceStepName,
|
ProcessTraceStepName,
|
||||||
} from "@skills/onboarding/runtime/trace";
|
} from "@skills/onboarding/runtime/trace";
|
||||||
import type { PrmFeatureVector } from "./judges/prm.js";
|
import type { PrmFeatureVector } from "./judges/prm.js";
|
||||||
|
import type {
|
||||||
|
MemoryJudgeExpected,
|
||||||
|
MemoryJudgeResult,
|
||||||
|
} from "./judges/memory-judge.js";
|
||||||
|
|
||||||
export type EvalProcessTraceSnapshot = ProcessTraceSnapshot;
|
export type EvalProcessTraceSnapshot = ProcessTraceSnapshot;
|
||||||
|
|
||||||
@@ -28,6 +33,7 @@ export interface EvalCaseInput {
|
|||||||
session?: OnboardingSession | null | undefined;
|
session?: OnboardingSession | null | undefined;
|
||||||
blackboard?: Blackboard | null | undefined;
|
blackboard?: Blackboard | null | undefined;
|
||||||
summary?: string | undefined;
|
summary?: string | undefined;
|
||||||
|
userProfile?: UserProfile | null | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface EvalCaseExpected {
|
export interface EvalCaseExpected {
|
||||||
@@ -37,6 +43,7 @@ export interface EvalCaseExpected {
|
|||||||
expectedTarget?: string | undefined;
|
expectedTarget?: string | undefined;
|
||||||
shouldHandoff?: boolean | undefined;
|
shouldHandoff?: boolean | undefined;
|
||||||
} | undefined;
|
} | undefined;
|
||||||
|
expectedMemory?: MemoryJudgeExpected | undefined;
|
||||||
expectedSourceChain?: {
|
expectedSourceChain?: {
|
||||||
chain: string[];
|
chain: string[];
|
||||||
mode: "contains" | "exact" | "prefix";
|
mode: "contains" | "exact" | "prefix";
|
||||||
@@ -183,6 +190,7 @@ export interface EvalCaseResult {
|
|||||||
ruleJudge: RuleJudgeResult;
|
ruleJudge: RuleJudgeResult;
|
||||||
llmJudge: LlmJudgeResult;
|
llmJudge: LlmJudgeResult;
|
||||||
handoffJudge?: HandoffJudgeResult | undefined;
|
handoffJudge?: HandoffJudgeResult | undefined;
|
||||||
|
memoryJudge?: MemoryJudgeResult | undefined;
|
||||||
processJudge?: ProcessJudgeResult | undefined;
|
processJudge?: ProcessJudgeResult | undefined;
|
||||||
processTrace?: ProcessTraceSnapshot | undefined;
|
processTrace?: ProcessTraceSnapshot | undefined;
|
||||||
error?: string | undefined;
|
error?: string | undefined;
|
||||||
|
|||||||
@@ -31,6 +31,9 @@ const { LlmClient } = await import("./llm/index.js");
|
|||||||
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
|
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
|
||||||
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
|
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
|
||||||
const { createMemoryService } = await import("./memory/memory-service.js");
|
const { createMemoryService } = await import("./memory/memory-service.js");
|
||||||
|
const { createEscalationHistoryService } = await import(
|
||||||
|
"./memory/mongo-escalation-history.js"
|
||||||
|
);
|
||||||
const { RoomContextService } = await import("./room-context/index.js");
|
const { RoomContextService } = await import("./room-context/index.js");
|
||||||
const { startMetricsServer, stopMetricsServer } = await import(
|
const { startMetricsServer, stopMetricsServer } = await import(
|
||||||
"./observability/metrics-server.js"
|
"./observability/metrics-server.js"
|
||||||
@@ -226,8 +229,11 @@ async function bootstrap(): Promise<void> {
|
|||||||
|
|
||||||
const memoryService = createMemoryService({
|
const memoryService = createMemoryService({
|
||||||
garnetClient,
|
garnetClient,
|
||||||
|
llmClient,
|
||||||
mongooseConnection: mongoConnection,
|
mongooseConnection: mongoConnection,
|
||||||
});
|
});
|
||||||
|
const escalationHistoryService =
|
||||||
|
createEscalationHistoryService(mongoConnection);
|
||||||
|
|
||||||
// Agent setup
|
// Agent setup
|
||||||
const conversationStore = new ConversationStore(garnetClient);
|
const conversationStore = new ConversationStore(garnetClient);
|
||||||
@@ -265,6 +271,7 @@ async function bootstrap(): Promise<void> {
|
|||||||
agent.registerSkill(
|
agent.registerSkill(
|
||||||
factory({
|
factory({
|
||||||
garnetClient,
|
garnetClient,
|
||||||
|
escalationHistoryService,
|
||||||
llmClient,
|
llmClient,
|
||||||
memoryService,
|
memoryService,
|
||||||
natsClient,
|
natsClient,
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { buildBM25Index } from "./bm25.js";
|
||||||
|
|
||||||
|
describe("buildBM25Index", () => {
|
||||||
|
it("ranks Korean keyword matches above unrelated documents", () => {
|
||||||
|
const index = buildBM25Index([
|
||||||
|
{ id: "ads", text: "광고와 도배 홍보 링크는 제재 대상입니다." },
|
||||||
|
{ id: "age", text: "가입 가능 연령은 27세 이상입니다." },
|
||||||
|
]);
|
||||||
|
|
||||||
|
const [top] = index.rank("광고 도배 기준", 2);
|
||||||
|
|
||||||
|
expect(top?.id).toBe("ads");
|
||||||
|
expect(top?.score).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
export interface BM25Document {
|
||||||
|
id: string;
|
||||||
|
text: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BM25Index {
|
||||||
|
rank(query: string, topK: number): Array<{ id: string; score: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const K1 = 1.5;
|
||||||
|
const B = 0.75;
|
||||||
|
|
||||||
|
function normalize(text: string): string {
|
||||||
|
return text
|
||||||
|
.normalize("NFKC")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^\p{Letter}\p{Number}\s]/gu, " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenize(text: string): string[] {
|
||||||
|
const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0);
|
||||||
|
const compact = normalize(text).replace(/\s+/gu, "");
|
||||||
|
const grams: string[] = [];
|
||||||
|
for (let i = 0; i < compact.length - 1; i += 1) {
|
||||||
|
grams.push(compact.slice(i, i + 2));
|
||||||
|
}
|
||||||
|
return [...new Set([...words, ...grams])];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildBM25Index(documents: BM25Document[]): BM25Index {
|
||||||
|
const termFrequencies = new Map<string, Map<string, number>>();
|
||||||
|
const documentLengths = new Map<string, number>();
|
||||||
|
const documentFrequency = new Map<string, number>();
|
||||||
|
let totalLength = 0;
|
||||||
|
|
||||||
|
for (const document of documents) {
|
||||||
|
const tokens = tokenize(document.text);
|
||||||
|
documentLengths.set(document.id, tokens.length);
|
||||||
|
totalLength += tokens.length;
|
||||||
|
|
||||||
|
const frequencies = new Map<string, number>();
|
||||||
|
for (const token of tokens) {
|
||||||
|
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
|
||||||
|
}
|
||||||
|
termFrequencies.set(document.id, frequencies);
|
||||||
|
|
||||||
|
for (const token of frequencies.keys()) {
|
||||||
|
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const averageLength = documents.length === 0 ? 0 : totalLength / documents.length;
|
||||||
|
|
||||||
|
return {
|
||||||
|
rank(query: string, topK: number): Array<{ id: string; score: number }> {
|
||||||
|
const queryTokens = tokenize(query);
|
||||||
|
if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return documents
|
||||||
|
.map((document) => {
|
||||||
|
const frequencies = termFrequencies.get(document.id) ?? new Map<string, number>();
|
||||||
|
const documentLength = documentLengths.get(document.id) ?? 0;
|
||||||
|
const score = queryTokens.reduce((sum, token) => {
|
||||||
|
const tf = frequencies.get(token) ?? 0;
|
||||||
|
if (tf === 0) {
|
||||||
|
return sum;
|
||||||
|
}
|
||||||
|
const df = documentFrequency.get(token) ?? 0;
|
||||||
|
const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5));
|
||||||
|
const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength));
|
||||||
|
return sum + idf * ((tf * (K1 + 1)) / denominator);
|
||||||
|
}, 0);
|
||||||
|
return { id: document.id, score };
|
||||||
|
})
|
||||||
|
.filter((item) => item.score > 0)
|
||||||
|
.sort((left, right) => right.score - left.score)
|
||||||
|
.slice(0, Math.max(1, topK));
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { detectContradictions, textSimilarity } from "./contradiction-detector.js";
|
||||||
|
|
||||||
|
describe("textSimilarity", () => {
|
||||||
|
it("returns lower scores for very different text", () => {
|
||||||
|
expect(textSimilarity("닉네임 규칙", "광고 제재 기준")).toBeLessThan(0.7);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("detectContradictions", () => {
|
||||||
|
it("flags corrections in the same source section", () => {
|
||||||
|
const [candidate] = detectContradictions([
|
||||||
|
{
|
||||||
|
id: "old",
|
||||||
|
metadata: { sourceFile: "rules.md", sectionTitle: "nickname" },
|
||||||
|
text: "닉네임은 기존 형식을 사용합니다.",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "new",
|
||||||
|
metadata: { sourceFile: "rules.md", sectionTitle: "nickname" },
|
||||||
|
text: "정정: 닉네임은 새 형식을 사용합니다.",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(candidate).toMatchObject({
|
||||||
|
existingId: "old",
|
||||||
|
newId: "new",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
export interface ContradictionCandidate {
|
||||||
|
existingId: string;
|
||||||
|
newId: string;
|
||||||
|
reason: "keyword" | "text-distance";
|
||||||
|
similarity: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ContradictionDocument {
|
||||||
|
id: string;
|
||||||
|
metadata?: Record<string, unknown> | undefined;
|
||||||
|
text: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function groupKey(doc: ContradictionDocument): string | null {
|
||||||
|
const sourceFile = doc.metadata?.["sourceFile"];
|
||||||
|
const sectionTitle = doc.metadata?.["sectionTitle"];
|
||||||
|
if (typeof sourceFile !== "string" || typeof sectionTitle !== "string") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return `${sourceFile}#${sectionTitle}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function bigrams(text: string): Set<string> {
|
||||||
|
const compact = text
|
||||||
|
.normalize("NFKC")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/\s+/gu, "");
|
||||||
|
const grams = new Set<string>();
|
||||||
|
for (let i = 0; i < compact.length - 1; i += 1) {
|
||||||
|
grams.add(compact.slice(i, i + 2));
|
||||||
|
}
|
||||||
|
return grams;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function textSimilarity(left: string, right: string): number {
|
||||||
|
const leftGrams = bigrams(left);
|
||||||
|
const rightGrams = bigrams(right);
|
||||||
|
if (leftGrams.size === 0 && rightGrams.size === 0) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
const intersection = [...leftGrams].filter((gram) => rightGrams.has(gram)).length;
|
||||||
|
const union = new Set([...leftGrams, ...rightGrams]).size;
|
||||||
|
return union === 0 ? 0 : intersection / union;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasCorrectionKeyword(text: string): boolean {
|
||||||
|
return /(변경|정정|수정|폐지|삭제|바뀜|업데이트)/u.test(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function detectContradictions(
|
||||||
|
documents: ContradictionDocument[],
|
||||||
|
): ContradictionCandidate[] {
|
||||||
|
const groups = new Map<string, ContradictionDocument[]>();
|
||||||
|
for (const doc of documents) {
|
||||||
|
const key = groupKey(doc);
|
||||||
|
if (!key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
groups.set(key, [...(groups.get(key) ?? []), doc]);
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidates: ContradictionCandidate[] = [];
|
||||||
|
for (const docs of groups.values()) {
|
||||||
|
for (let index = 1; index < docs.length; index += 1) {
|
||||||
|
const previous = docs[index - 1];
|
||||||
|
const current = docs[index];
|
||||||
|
if (!previous || !current) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const similarity = textSimilarity(previous.text, current.text);
|
||||||
|
if (similarity < 0.7 || hasCorrectionKeyword(current.text)) {
|
||||||
|
candidates.push({
|
||||||
|
existingId: previous.id,
|
||||||
|
newId: current.id,
|
||||||
|
reason: hasCorrectionKeyword(current.text) ? "keyword" : "text-distance",
|
||||||
|
similarity,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidates;
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import type { HandoffReason } from "@agent/types";
|
||||||
|
import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue";
|
||||||
|
|
||||||
|
export interface UserEscalationRecord {
|
||||||
|
queuedAt: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
reviewId: string;
|
||||||
|
target: string;
|
||||||
|
resolution?: {
|
||||||
|
resolvedAt: string;
|
||||||
|
action: "dismiss" | "manual-reply" | "reroute";
|
||||||
|
note?: string | undefined;
|
||||||
|
} | undefined;
|
||||||
|
packetSummary: string;
|
||||||
|
sourceChain: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UserEscalationHistory {
|
||||||
|
userId: string;
|
||||||
|
chatroomId?: string | undefined;
|
||||||
|
escalations: UserEscalationRecord[];
|
||||||
|
totalEscalations: number;
|
||||||
|
lastEscalationAt: string;
|
||||||
|
topReasons: Array<{ reason: HandoffReason; count: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EscalationHistoryService {
|
||||||
|
load(userId: string): Promise<UserEscalationHistory | null>;
|
||||||
|
recordEscalation(record: {
|
||||||
|
userId: string;
|
||||||
|
chatroomId?: string | undefined;
|
||||||
|
queuedAt: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
reviewId: string;
|
||||||
|
target: string;
|
||||||
|
packetSummary: string;
|
||||||
|
sourceChain: string[];
|
||||||
|
}): Promise<void>;
|
||||||
|
recordResolution(record: {
|
||||||
|
userId: string;
|
||||||
|
reviewId: string;
|
||||||
|
resolution: SupervisorReviewResolution;
|
||||||
|
}): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class NoopEscalationHistoryService implements EscalationHistoryService {
|
||||||
|
load(): Promise<UserEscalationHistory | null> {
|
||||||
|
return Promise.resolve(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
recordEscalation(): Promise<void> {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
|
||||||
|
recordResolution(): Promise<void> {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { evaluateDocument, evaluateUserFlags } from "./forgetting-policy.js";
|
||||||
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
|
|
||||||
|
describe("evaluateDocument", () => {
|
||||||
|
it("archives stale low-reference documents", () => {
|
||||||
|
const decision = evaluateDocument(
|
||||||
|
{ category: "rule", id: "doc-1", text: "old rule" },
|
||||||
|
{
|
||||||
|
now: new Date("2026-05-18T00:00:00.000Z"),
|
||||||
|
referenceCount30d: 0,
|
||||||
|
updatedAt: new Date("2025-12-01T00:00:00.000Z"),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(decision.action).toBe("archive");
|
||||||
|
expect(decision.reason).toBe("freshness-stale");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("evaluateUserFlags", () => {
|
||||||
|
it("keeps the newest unique flags within the limit", () => {
|
||||||
|
const profile = {
|
||||||
|
flaggedIssues: ["a", "b", "c"],
|
||||||
|
} as UserProfile;
|
||||||
|
|
||||||
|
expect(evaluateUserFlags(profile, 2)).toEqual({
|
||||||
|
archived: ["a"],
|
||||||
|
kept: ["b", "c"],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
|
import type { KnowledgeDocument } from "./memory-types.js";
|
||||||
|
|
||||||
|
export interface ForgettingDecision {
|
||||||
|
documentId: string;
|
||||||
|
action: "archive" | "delete" | "keep";
|
||||||
|
reason: "contradiction" | "freshness-stale" | "relevance-low" | "ttl-expired";
|
||||||
|
evidence?: string | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KnowledgeForgettingSignals {
|
||||||
|
contradictsId?: string | undefined;
|
||||||
|
lastReferencedAt?: Date | undefined;
|
||||||
|
now?: Date | undefined;
|
||||||
|
referenceCount30d: number;
|
||||||
|
updatedAt?: Date | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function daysSince(date: Date | undefined, now: Date): number {
|
||||||
|
if (!date) {
|
||||||
|
return Number.POSITIVE_INFINITY;
|
||||||
|
}
|
||||||
|
return Math.max(0, now.getTime() - date.getTime()) / (24 * 60 * 60 * 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateDocument(
|
||||||
|
doc: KnowledgeDocument,
|
||||||
|
signals: KnowledgeForgettingSignals,
|
||||||
|
): ForgettingDecision {
|
||||||
|
const now = signals.now ?? new Date();
|
||||||
|
const documentId = doc.id ?? "unknown";
|
||||||
|
|
||||||
|
if (signals.contradictsId) {
|
||||||
|
return {
|
||||||
|
action: "archive",
|
||||||
|
documentId,
|
||||||
|
evidence: `contradicts ${signals.contradictsId}`,
|
||||||
|
reason: "contradiction",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const staleDays = daysSince(signals.updatedAt, now);
|
||||||
|
const referenceDays = daysSince(signals.lastReferencedAt, now);
|
||||||
|
if (staleDays >= 90 && signals.referenceCount30d < 3) {
|
||||||
|
return {
|
||||||
|
action: "archive",
|
||||||
|
documentId,
|
||||||
|
evidence: `updated ${Math.round(staleDays)}d ago, refs30d=${signals.referenceCount30d}`,
|
||||||
|
reason: "freshness-stale",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (referenceDays >= 30 && signals.referenceCount30d < 3) {
|
||||||
|
return {
|
||||||
|
action: "archive",
|
||||||
|
documentId,
|
||||||
|
evidence: `last referenced ${Math.round(referenceDays)}d ago, refs30d=${signals.referenceCount30d}`,
|
||||||
|
reason: "relevance-low",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
action: "keep",
|
||||||
|
documentId,
|
||||||
|
reason: "relevance-low",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateUserFlags(
|
||||||
|
profile: UserProfile,
|
||||||
|
maxFlags: number,
|
||||||
|
): { archived: string[]; kept: string[] } {
|
||||||
|
const unique = [...new Set(profile.flaggedIssues.map((issue) => issue.trim()).filter(Boolean))];
|
||||||
|
if (unique.length <= maxFlags) {
|
||||||
|
return { archived: [], kept: unique };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
archived: unique.slice(0, Math.max(0, unique.length - maxFlags)),
|
||||||
|
kept: unique.slice(-maxFlags),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import { getLongTermSummaryEnabled } from "../config.js";
|
||||||
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
|
|
||||||
|
export interface LongTermSummary {
|
||||||
|
text: string;
|
||||||
|
updatedAt: string;
|
||||||
|
turnSnapshot: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SummaryPayload {
|
||||||
|
text?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
function daysBetween(leftIso: string, rightIso: string): number {
|
||||||
|
const left = Date.parse(leftIso);
|
||||||
|
const right = Date.parse(rightIso);
|
||||||
|
if (!Number.isFinite(left) || !Number.isFinite(right)) {
|
||||||
|
return Number.POSITIVE_INFINITY;
|
||||||
|
}
|
||||||
|
return Math.abs(right - left) / (24 * 60 * 60 * 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldRefreshLongTermSummary(
|
||||||
|
profile: UserProfile,
|
||||||
|
now = new Date().toISOString(),
|
||||||
|
): boolean {
|
||||||
|
if (!getLongTermSummaryEnabled()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (profile.totalTurns < 30) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const summary = profile.longTermSummary;
|
||||||
|
if (!summary) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
profile.totalTurns - summary.turnSnapshot >= 30 ||
|
||||||
|
daysBetween(summary.updatedAt, now) >= 7
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function buildLongTermSummary(
|
||||||
|
llmClient: Pick<LlmClient, "chatCompletion">,
|
||||||
|
profile: UserProfile,
|
||||||
|
conversationSummary: string | null,
|
||||||
|
escalationPattern?: string,
|
||||||
|
): Promise<LongTermSummary> {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const result = await llmClient.chatCompletion(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
role: "system",
|
||||||
|
content: [
|
||||||
|
"너는 사용자별 장기 메모를 작성한다.",
|
||||||
|
"운영자가 다음 응답에서 참고할 수 있게 3-5문장으로 요약한다.",
|
||||||
|
"개인정보나 원문 식별자는 넣지 않는다.",
|
||||||
|
'JSON으로만 응답한다: {"text":"요약"}',
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
`[totalTurns] ${profile.totalTurns}`,
|
||||||
|
`[preferredStyle] ${profile.preferredStyle}`,
|
||||||
|
`[pastIntents] ${profile.pastIntents.map((item) => `${item.kind}:${item.count}`).join(", ") || "-"}`,
|
||||||
|
`[flaggedIssues] ${profile.flaggedIssues.slice(-10).join(", ") || "-"}`,
|
||||||
|
`[conversationSummary] ${conversationSummary ?? "-"}`,
|
||||||
|
`[escalationPattern] ${escalationPattern ?? "-"}`,
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{
|
||||||
|
jsonMode: true,
|
||||||
|
maxTokens: 300,
|
||||||
|
temperature: 0.3,
|
||||||
|
tier: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!result?.content.trim()) {
|
||||||
|
throw new Error("Long-term summary LLM returned no content");
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = JSON.parse(result.content) as SummaryPayload;
|
||||||
|
if (typeof parsed.text !== "string" || parsed.text.trim().length === 0) {
|
||||||
|
throw new Error("Long-term summary payload missing text");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
text: parsed.text.trim(),
|
||||||
|
updatedAt: now,
|
||||||
|
turnSnapshot: profile.totalTurns,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import type mongoose from "mongoose";
|
import type mongoose from "mongoose";
|
||||||
import type { GarnetClient } from "@common/garnet";
|
import type { GarnetClient } from "@common/garnet";
|
||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import type { Blackboard } from "@skills/onboarding/types";
|
import type { Blackboard } from "@skills/onboarding/types";
|
||||||
import { createEmbeddingClient } from "./embedding-client.js";
|
import { createEmbeddingClient } from "./embedding-client.js";
|
||||||
import { GarnetSessionMemory } from "./garnet-session-memory.js";
|
import { GarnetSessionMemory } from "./garnet-session-memory.js";
|
||||||
@@ -7,6 +8,7 @@ import {
|
|||||||
MongoLongTermMemory,
|
MongoLongTermMemory,
|
||||||
NoopLongTermMemory,
|
NoopLongTermMemory,
|
||||||
} from "./mongo-long-term-memory.js";
|
} from "./mongo-long-term-memory.js";
|
||||||
|
import { LlmReranker } from "./reranker.js";
|
||||||
import type {
|
import type {
|
||||||
LongTermMemory,
|
LongTermMemory,
|
||||||
MemoryService,
|
MemoryService,
|
||||||
@@ -17,6 +19,7 @@ import type {
|
|||||||
|
|
||||||
interface CreateMemoryServiceOptions {
|
interface CreateMemoryServiceOptions {
|
||||||
garnetClient: GarnetClient;
|
garnetClient: GarnetClient;
|
||||||
|
llmClient?: Pick<LlmClient, "chatCompletion"> | undefined;
|
||||||
mongooseConnection?: mongoose.Connection | undefined;
|
mongooseConnection?: mongoose.Connection | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,6 +91,7 @@ export function createMemoryService(
|
|||||||
options.garnetClient,
|
options.garnetClient,
|
||||||
options.mongooseConnection,
|
options.mongooseConnection,
|
||||||
createEmbeddingClient(),
|
createEmbeddingClient(),
|
||||||
|
options.llmClient ? new LlmReranker(options.llmClient) : undefined,
|
||||||
)
|
)
|
||||||
: new NoopLongTermMemory(options.garnetClient);
|
: new NoopLongTermMemory(options.garnetClient);
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,10 @@ export interface SessionMemory {
|
|||||||
export interface SearchOptions {
|
export interface SearchOptions {
|
||||||
skill?: string | undefined;
|
skill?: string | undefined;
|
||||||
category?: string | undefined;
|
category?: string | undefined;
|
||||||
|
candidatePoolSize?: number | undefined;
|
||||||
|
hybrid?: boolean | undefined;
|
||||||
|
rerank?: boolean | undefined;
|
||||||
|
rerankTopK?: number | undefined;
|
||||||
topK?: number | undefined;
|
topK?: number | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,173 @@
|
|||||||
|
import type mongoose from "mongoose";
|
||||||
|
import { Schema } from "mongoose";
|
||||||
|
import { anonymizeIdentifier } from "@eval/anonymize";
|
||||||
|
import type { HandoffReason } from "@agent/types";
|
||||||
|
import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue";
|
||||||
|
import {
|
||||||
|
NoopEscalationHistoryService,
|
||||||
|
type EscalationHistoryService,
|
||||||
|
type UserEscalationHistory,
|
||||||
|
type UserEscalationRecord,
|
||||||
|
} from "./escalation-history.js";
|
||||||
|
|
||||||
|
export const ESCALATION_HISTORY_COLLECTION = "user_escalation_history";
|
||||||
|
|
||||||
|
interface EscalationHistoryRecord {
|
||||||
|
_id: string;
|
||||||
|
userId: string;
|
||||||
|
chatroomId?: string | undefined;
|
||||||
|
escalations: UserEscalationRecord[];
|
||||||
|
totalEscalations: number;
|
||||||
|
lastEscalationAt: string;
|
||||||
|
topReasons: Array<{ reason: HandoffReason; count: number }>;
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
const EscalationHistorySchema = new Schema<EscalationHistoryRecord>(
|
||||||
|
{
|
||||||
|
_id: { type: String, required: true },
|
||||||
|
userId: { type: String, required: true, index: true },
|
||||||
|
chatroomId: { type: String, index: true },
|
||||||
|
escalations: [
|
||||||
|
{
|
||||||
|
queuedAt: String,
|
||||||
|
reason: String,
|
||||||
|
reviewId: String,
|
||||||
|
target: String,
|
||||||
|
resolution: {
|
||||||
|
resolvedAt: String,
|
||||||
|
action: String,
|
||||||
|
note: String,
|
||||||
|
},
|
||||||
|
packetSummary: String,
|
||||||
|
sourceChain: [String],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
totalEscalations: { type: Number, default: 0 },
|
||||||
|
lastEscalationAt: String,
|
||||||
|
topReasons: [{ reason: String, count: Number }],
|
||||||
|
updatedAt: { type: Date, default: Date.now, index: true },
|
||||||
|
},
|
||||||
|
{ versionKey: false },
|
||||||
|
);
|
||||||
|
|
||||||
|
function userKey(userId: string): string {
|
||||||
|
return anonymizeIdentifier("user", userId) ?? "user_unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeTopReasons(records: UserEscalationRecord[]): Array<{ reason: HandoffReason; count: number }> {
|
||||||
|
const counts = new Map<HandoffReason, number>();
|
||||||
|
for (const record of records) {
|
||||||
|
counts.set(record.reason, (counts.get(record.reason) ?? 0) + 1);
|
||||||
|
}
|
||||||
|
return [...counts.entries()]
|
||||||
|
.map(([reason, count]) => ({ reason, count }))
|
||||||
|
.sort((left, right) => right.count - left.count || left.reason.localeCompare(right.reason))
|
||||||
|
.slice(0, 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
export class MongoEscalationHistoryService implements EscalationHistoryService {
|
||||||
|
private readonly model: mongoose.Model<EscalationHistoryRecord>;
|
||||||
|
|
||||||
|
constructor(connection: mongoose.Connection) {
|
||||||
|
this.model =
|
||||||
|
(connection.models["EscalationHistory"] as mongoose.Model<EscalationHistoryRecord> | undefined) ??
|
||||||
|
connection.model<EscalationHistoryRecord>(
|
||||||
|
"EscalationHistory",
|
||||||
|
EscalationHistorySchema,
|
||||||
|
ESCALATION_HISTORY_COLLECTION,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async load(userId: string): Promise<UserEscalationHistory | null> {
|
||||||
|
const doc = await this.model.findById(userKey(userId)).lean().exec();
|
||||||
|
if (!doc) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
userId: doc.userId,
|
||||||
|
...(doc.chatroomId ? { chatroomId: doc.chatroomId } : {}),
|
||||||
|
escalations: doc.escalations.slice(-10),
|
||||||
|
lastEscalationAt: doc.lastEscalationAt,
|
||||||
|
topReasons: doc.topReasons,
|
||||||
|
totalEscalations: doc.totalEscalations,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async recordEscalation(record: {
|
||||||
|
userId: string;
|
||||||
|
chatroomId?: string | undefined;
|
||||||
|
queuedAt: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
reviewId: string;
|
||||||
|
target: string;
|
||||||
|
packetSummary: string;
|
||||||
|
sourceChain: string[];
|
||||||
|
}): Promise<void> {
|
||||||
|
const id = userKey(record.userId);
|
||||||
|
const item: UserEscalationRecord = {
|
||||||
|
packetSummary: record.packetSummary,
|
||||||
|
queuedAt: record.queuedAt,
|
||||||
|
reason: record.reason,
|
||||||
|
reviewId: record.reviewId,
|
||||||
|
sourceChain: record.sourceChain,
|
||||||
|
target: record.target,
|
||||||
|
};
|
||||||
|
await this.model.updateOne(
|
||||||
|
{ _id: id },
|
||||||
|
{
|
||||||
|
$inc: { totalEscalations: 1 },
|
||||||
|
$push: { escalations: { $each: [item], $slice: -50 } },
|
||||||
|
$set: {
|
||||||
|
...(record.chatroomId ? { chatroomId: record.chatroomId } : {}),
|
||||||
|
lastEscalationAt: record.queuedAt,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
userId: id,
|
||||||
|
},
|
||||||
|
$setOnInsert: { _id: id },
|
||||||
|
},
|
||||||
|
{ upsert: true },
|
||||||
|
).exec();
|
||||||
|
await this.recomputeTopReasons(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
async recordResolution(record: {
|
||||||
|
userId: string;
|
||||||
|
reviewId: string;
|
||||||
|
resolution: SupervisorReviewResolution;
|
||||||
|
}): Promise<void> {
|
||||||
|
const id = userKey(record.userId);
|
||||||
|
await this.model.updateOne(
|
||||||
|
{ _id: id, "escalations.reviewId": record.reviewId },
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
"escalations.$.resolution": {
|
||||||
|
action: record.resolution.action,
|
||||||
|
resolvedAt: record.resolution.resolvedAt,
|
||||||
|
...(record.resolution.note ? { note: record.resolution.note } : {}),
|
||||||
|
},
|
||||||
|
updatedAt: new Date(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
).exec();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async recomputeTopReasons(id: string): Promise<void> {
|
||||||
|
const doc = await this.model.findById(id).lean().exec();
|
||||||
|
if (!doc) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.model.updateOne(
|
||||||
|
{ _id: id },
|
||||||
|
{ $set: { topReasons: computeTopReasons(doc.escalations) } },
|
||||||
|
).exec();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createEscalationHistoryService(
|
||||||
|
connection?: mongoose.Connection,
|
||||||
|
): EscalationHistoryService {
|
||||||
|
return connection
|
||||||
|
? new MongoEscalationHistoryService(connection)
|
||||||
|
: new NoopEscalationHistoryService();
|
||||||
|
}
|
||||||
@@ -3,11 +3,19 @@ import type mongoose from "mongoose";
|
|||||||
import { Schema } from "mongoose";
|
import { Schema } from "mongoose";
|
||||||
import { anonymizeIdentifier } from "@eval/anonymize";
|
import { anonymizeIdentifier } from "@eval/anonymize";
|
||||||
import { withSpan } from "@obs/spans";
|
import { withSpan } from "@obs/spans";
|
||||||
import { getAgentUserMemoryTtlDays, getKnowledgeSearchCandidateLimit } from "../config.js";
|
import {
|
||||||
|
getAgentUserMemoryTtlDays,
|
||||||
|
getKnowledgeSearchCandidateLimit,
|
||||||
|
getRagCandidatePoolSize,
|
||||||
|
getRagHybridEnabled,
|
||||||
|
getRagRerankerEnabled,
|
||||||
|
getRagRerankerTopK,
|
||||||
|
} from "../config.js";
|
||||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory";
|
import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory";
|
||||||
import type { GarnetClient } from "@common/garnet";
|
import type { GarnetClient } from "@common/garnet";
|
||||||
import { rankByCosineSimilarity } from "./cosine.js";
|
import { rankByCosineSimilarity } from "./cosine.js";
|
||||||
|
import { buildBM25Index } from "./bm25.js";
|
||||||
import type {
|
import type {
|
||||||
KnowledgeDocument,
|
KnowledgeDocument,
|
||||||
LongTermMemory,
|
LongTermMemory,
|
||||||
@@ -15,6 +23,7 @@ import type {
|
|||||||
SearchOptions,
|
SearchOptions,
|
||||||
} from "./memory-types.js";
|
} from "./memory-types.js";
|
||||||
import type { EmbeddingClient } from "./embedding-client.js";
|
import type { EmbeddingClient } from "./embedding-client.js";
|
||||||
|
import type { Reranker } from "./reranker.js";
|
||||||
|
|
||||||
export const KNOWLEDGE_COLLECTION = "knowledge_documents";
|
export const KNOWLEDGE_COLLECTION = "knowledge_documents";
|
||||||
|
|
||||||
@@ -26,6 +35,8 @@ interface KnowledgeDocumentRecord {
|
|||||||
embedding: number[];
|
embedding: number[];
|
||||||
embeddingModel: string;
|
embeddingModel: string;
|
||||||
metadata?: Record<string, unknown> | undefined;
|
metadata?: Record<string, unknown> | undefined;
|
||||||
|
lastReferencedAt?: Date | undefined;
|
||||||
|
references?: Date[] | undefined;
|
||||||
createdAt: Date;
|
createdAt: Date;
|
||||||
updatedAt: Date;
|
updatedAt: Date;
|
||||||
}
|
}
|
||||||
@@ -39,6 +50,8 @@ const KnowledgeSchema = new Schema<KnowledgeDocumentRecord>(
|
|||||||
embedding: { type: [Number], required: true },
|
embedding: { type: [Number], required: true },
|
||||||
embeddingModel: { type: String, required: true },
|
embeddingModel: { type: String, required: true },
|
||||||
metadata: { type: Schema.Types.Mixed },
|
metadata: { type: Schema.Types.Mixed },
|
||||||
|
lastReferencedAt: { type: Date, index: true },
|
||||||
|
references: { type: [Date], default: [] },
|
||||||
createdAt: { type: Date, default: Date.now },
|
createdAt: { type: Date, default: Date.now },
|
||||||
updatedAt: { type: Date, default: Date.now, index: true },
|
updatedAt: { type: Date, default: Date.now, index: true },
|
||||||
},
|
},
|
||||||
@@ -83,7 +96,9 @@ function stableDocumentId(doc: KnowledgeDocument): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildFilter(options: SearchOptions): Record<string, unknown> {
|
function buildFilter(options: SearchOptions): Record<string, unknown> {
|
||||||
const filter: Record<string, unknown> = {};
|
const filter: Record<string, unknown> = {
|
||||||
|
"metadata.archived": { $ne: true },
|
||||||
|
};
|
||||||
if (options.skill?.trim()) {
|
if (options.skill?.trim()) {
|
||||||
filter["skill"] = options.skill.trim();
|
filter["skill"] = options.skill.trim();
|
||||||
}
|
}
|
||||||
@@ -93,6 +108,34 @@ function buildFilter(options: SearchOptions): Record<string, unknown> {
|
|||||||
return filter;
|
return filter;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeScores(scores: Map<string, number>): Map<string, number> {
|
||||||
|
const values = [...scores.values()].filter((value) => Number.isFinite(value));
|
||||||
|
if (values.length === 0) {
|
||||||
|
return new Map();
|
||||||
|
}
|
||||||
|
const min = Math.min(...values);
|
||||||
|
const max = Math.max(...values);
|
||||||
|
if (max === min) {
|
||||||
|
return new Map([...scores.keys()].map((id) => [id, max > 0 ? 1 : 0]));
|
||||||
|
}
|
||||||
|
return new Map(
|
||||||
|
[...scores.entries()].map(([id, score]) => [id, (score - min) / (max - min)]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function toRetrievedDocument(
|
||||||
|
doc: KnowledgeDocumentRecord & { score: number },
|
||||||
|
): RetrievedDocument {
|
||||||
|
return {
|
||||||
|
id: doc._id,
|
||||||
|
category: doc.category,
|
||||||
|
metadata: doc.metadata,
|
||||||
|
score: doc.score,
|
||||||
|
skill: doc.skill,
|
||||||
|
text: doc.text,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export class MongoLongTermMemory implements LongTermMemory {
|
export class MongoLongTermMemory implements LongTermMemory {
|
||||||
private readonly model: mongoose.Model<KnowledgeDocumentRecord>;
|
private readonly model: mongoose.Model<KnowledgeDocumentRecord>;
|
||||||
|
|
||||||
@@ -100,6 +143,7 @@ export class MongoLongTermMemory implements LongTermMemory {
|
|||||||
private readonly garnetClient: Pick<GarnetClient, "getJson" | "setJson">,
|
private readonly garnetClient: Pick<GarnetClient, "getJson" | "setJson">,
|
||||||
connection: mongoose.Connection,
|
connection: mongoose.Connection,
|
||||||
private readonly embeddingClient: EmbeddingClient,
|
private readonly embeddingClient: EmbeddingClient,
|
||||||
|
private readonly reranker?: Reranker | undefined,
|
||||||
) {
|
) {
|
||||||
this.model =
|
this.model =
|
||||||
(connection.models["KnowledgeDocument"] as mongoose.Model<KnowledgeDocumentRecord> | undefined) ??
|
(connection.models["KnowledgeDocument"] as mongoose.Model<KnowledgeDocumentRecord> | undefined) ??
|
||||||
@@ -162,33 +206,93 @@ export class MongoLongTermMemory implements LongTermMemory {
|
|||||||
.limit(getKnowledgeSearchCandidateLimit())
|
.limit(getKnowledgeSearchCandidateLimit())
|
||||||
.lean()
|
.lean()
|
||||||
.exec()) as KnowledgeDocumentRecord[];
|
.exec()) as KnowledgeDocumentRecord[];
|
||||||
const ranked = rankByCosineSimilarity(
|
const candidatePoolSize = options.candidatePoolSize ?? getRagCandidatePoolSize();
|
||||||
|
const topK = options.topK ?? 5;
|
||||||
|
const shouldHybrid = options.hybrid ?? getRagHybridEnabled();
|
||||||
|
const shouldRerank = Boolean(
|
||||||
|
this.reranker && (options.rerank ?? getRagRerankerEnabled()),
|
||||||
|
);
|
||||||
|
const cosineRanked = rankByCosineSimilarity(
|
||||||
queryEmbedding,
|
queryEmbedding,
|
||||||
docs.filter((doc) =>
|
docs.filter((doc) =>
|
||||||
Array.isArray(doc.embedding) &&
|
Array.isArray(doc.embedding) &&
|
||||||
doc.embedding.length === queryEmbedding.length
|
doc.embedding.length === queryEmbedding.length
|
||||||
),
|
),
|
||||||
options.topK ?? 5,
|
Math.max(candidatePoolSize, topK),
|
||||||
);
|
);
|
||||||
|
const ranked = shouldHybrid
|
||||||
|
? this.rankHybrid(trimmed, docs, cosineRanked, Math.max(candidatePoolSize, topK))
|
||||||
|
: cosineRanked.slice(0, Math.max(candidatePoolSize, topK));
|
||||||
|
const pool = ranked.slice(0, shouldRerank ? candidatePoolSize : topK);
|
||||||
|
const results =
|
||||||
|
shouldRerank && this.reranker
|
||||||
|
? await this.reranker.rerank(
|
||||||
|
trimmed,
|
||||||
|
pool.map(toRetrievedDocument),
|
||||||
|
options.rerankTopK ?? getRagRerankerTopK(),
|
||||||
|
)
|
||||||
|
: pool.slice(0, topK).map(toRetrievedDocument);
|
||||||
|
|
||||||
|
await this.markReferenced(results.map((doc) => doc.id));
|
||||||
|
|
||||||
span.setAttributes({
|
span.setAttributes({
|
||||||
"memory.retrieval.candidate_count": docs.length,
|
"memory.retrieval.candidate_count": docs.length,
|
||||||
"memory.retrieval.duration_ms": Math.max(0, Date.now() - startedAt),
|
"memory.retrieval.duration_ms": Math.max(0, Date.now() - startedAt),
|
||||||
"memory.retrieval.top_score": ranked[0]?.score,
|
"memory.retrieval.hybrid": shouldHybrid,
|
||||||
|
"memory.retrieval.rerank": shouldRerank,
|
||||||
|
"memory.retrieval.top_score": results[0]?.score,
|
||||||
});
|
});
|
||||||
|
|
||||||
return ranked.map((doc) => ({
|
return results.slice(0, topK);
|
||||||
id: doc._id,
|
|
||||||
category: doc.category,
|
|
||||||
metadata: doc.metadata,
|
|
||||||
score: doc.score,
|
|
||||||
skill: doc.skill,
|
|
||||||
text: doc.text,
|
|
||||||
}));
|
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private rankHybrid(
|
||||||
|
query: string,
|
||||||
|
docs: KnowledgeDocumentRecord[],
|
||||||
|
cosineRanked: Array<KnowledgeDocumentRecord & { score: number }>,
|
||||||
|
limit: number,
|
||||||
|
): Array<KnowledgeDocumentRecord & { score: number }> {
|
||||||
|
const cosineScores = normalizeScores(
|
||||||
|
new Map(cosineRanked.map((doc) => [doc._id, doc.score])),
|
||||||
|
);
|
||||||
|
const bm25Scores = normalizeScores(
|
||||||
|
new Map(
|
||||||
|
buildBM25Index(docs.map((doc) => ({ id: doc._id, text: doc.text })))
|
||||||
|
.rank(query, docs.length)
|
||||||
|
.map((item) => [item.id, item.score]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const byId = new Map(docs.map((doc) => [doc._id, doc]));
|
||||||
|
|
||||||
|
return [...byId.keys()]
|
||||||
|
.map((id) => ({
|
||||||
|
doc: byId.get(id),
|
||||||
|
score: ((cosineScores.get(id) ?? 0) + (bm25Scores.get(id) ?? 0)) / 2,
|
||||||
|
}))
|
||||||
|
.filter((item): item is { doc: KnowledgeDocumentRecord; score: number } =>
|
||||||
|
Boolean(item.doc) && item.score > 0
|
||||||
|
)
|
||||||
|
.map((item) => ({ ...item.doc, score: item.score }))
|
||||||
|
.sort((left, right) => right.score - left.score)
|
||||||
|
.slice(0, limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async markReferenced(ids: string[]): Promise<void> {
|
||||||
|
if (ids.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const now = new Date();
|
||||||
|
await this.model.updateMany(
|
||||||
|
{ _id: { $in: ids } },
|
||||||
|
{
|
||||||
|
$set: { lastReferencedAt: now },
|
||||||
|
$push: { references: { $each: [now], $slice: -100 } },
|
||||||
|
},
|
||||||
|
).exec();
|
||||||
|
}
|
||||||
|
|
||||||
async upsertDocument(doc: KnowledgeDocument): Promise<string> {
|
async upsertDocument(doc: KnowledgeDocument): Promise<string> {
|
||||||
const id = doc.id?.trim() || stableDocumentId(doc);
|
const id = doc.id?.trim() || stableDocumentId(doc);
|
||||||
const embedding = doc.embedding ?? await this.embeddingClient.embed(doc.text);
|
const embedding = doc.embedding ?? await this.embeddingClient.embed(doc.text);
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { deriveProfilePromptSnippets } from "./profile-update-policy.js";
|
||||||
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
|
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||||
|
|
||||||
|
const profile: UserProfile = {
|
||||||
|
firstSeenAt: "2026-05-01T00:00:00.000Z",
|
||||||
|
flaggedIssues: ["닉네임 형식 오류", "닉네임 형식 오류", "닉네임 형식 오류"],
|
||||||
|
lastSeenAt: "2026-05-18T00:00:00.000Z",
|
||||||
|
pastIntents: [{ kind: "nickname-question", count: 5 }],
|
||||||
|
preferredStyle: "concise",
|
||||||
|
totalTurns: 10,
|
||||||
|
userId: "user_x",
|
||||||
|
};
|
||||||
|
|
||||||
|
const derived = {
|
||||||
|
intent: { kind: "nickname-question" },
|
||||||
|
} as DerivedState;
|
||||||
|
|
||||||
|
describe("deriveProfilePromptSnippets", () => {
|
||||||
|
it("derives frequent intent, repeated issue, and preferred style snippets", () => {
|
||||||
|
const snippets = deriveProfilePromptSnippets(profile, derived);
|
||||||
|
|
||||||
|
expect(snippets.map((snippet) => snippet.reason)).toEqual([
|
||||||
|
"frequent-intent",
|
||||||
|
"repeated-issue",
|
||||||
|
"preferred-style",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||||
|
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||||
|
|
||||||
|
export interface ProfilePromptSnippet {
|
||||||
|
block: string;
|
||||||
|
reason: "frequent-intent" | "preferred-style" | "repeated-issue";
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeIssue(issue: string): string {
|
||||||
|
return issue
|
||||||
|
.normalize("NFKC")
|
||||||
|
.replace(/\[[^\]]+\]/gu, "")
|
||||||
|
.replace(/[0-9]/gu, "")
|
||||||
|
.replace(/\s+/gu, " ")
|
||||||
|
.trim()
|
||||||
|
.slice(0, 32);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function deriveProfilePromptSnippets(
|
||||||
|
profile: UserProfile,
|
||||||
|
derived: DerivedState,
|
||||||
|
): ProfilePromptSnippet[] {
|
||||||
|
const snippets: ProfilePromptSnippet[] = [];
|
||||||
|
const frequentIntent = profile.pastIntents.find(
|
||||||
|
(intent) => intent.kind === derived.intent.kind && intent.count >= 5,
|
||||||
|
);
|
||||||
|
if (frequentIntent) {
|
||||||
|
snippets.push({
|
||||||
|
reason: "frequent-intent",
|
||||||
|
block: `사용자는 ${frequentIntent.kind} 유형을 자주 물어봤어요. 핵심 답부터 짧게 말하고, 질문 범위를 넘기지 마세요.`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const issueCounts = new Map<string, number>();
|
||||||
|
for (const issue of profile.flaggedIssues) {
|
||||||
|
const normalized = normalizeIssue(issue);
|
||||||
|
if (normalized) {
|
||||||
|
issueCounts.set(normalized, (issueCounts.get(normalized) ?? 0) + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const repeatedIssue = [...issueCounts.entries()].find(([, count]) => count >= 3);
|
||||||
|
if (repeatedIssue) {
|
||||||
|
snippets.push({
|
||||||
|
reason: "repeated-issue",
|
||||||
|
block: `반복된 이슈가 있었어요: ${repeatedIssue[0]}. 같은 실수가 이어지지 않도록 정확한 형식 예시를 먼저 확인하세요.`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (profile.preferredStyle === "concise") {
|
||||||
|
snippets.push({
|
||||||
|
reason: "preferred-style",
|
||||||
|
block: "사용자는 짧고 바로 실행 가능한 답변을 선호해요.",
|
||||||
|
});
|
||||||
|
} else if (profile.preferredStyle === "warm") {
|
||||||
|
snippets.push({
|
||||||
|
reason: "preferred-style",
|
||||||
|
block: "사용자는 따뜻한 톤에 잘 반응했어요. 다만 불필요한 절차 설명은 붙이지 마세요.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return snippets;
|
||||||
|
}
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import { getRequestLogger } from "@obs/context";
|
||||||
|
import { withSpan } from "@obs/spans";
|
||||||
|
import type { RetrievedDocument } from "./memory-types.js";
|
||||||
|
|
||||||
|
export interface Reranker {
|
||||||
|
rerank(
|
||||||
|
query: string,
|
||||||
|
candidates: RetrievedDocument[],
|
||||||
|
topN: number,
|
||||||
|
): Promise<RetrievedDocument[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RankingItem {
|
||||||
|
id?: unknown;
|
||||||
|
score?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RerankerPayload {
|
||||||
|
rankings?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncate(text: string, maxLength: number): string {
|
||||||
|
return text.length <= maxLength ? text : `${text.slice(0, maxLength)}...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRankings(raw: string): Array<{ id: string; score: number }> {
|
||||||
|
const parsed = JSON.parse(raw) as RerankerPayload;
|
||||||
|
if (!Array.isArray(parsed.rankings)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed.rankings.flatMap((item: RankingItem) => {
|
||||||
|
if (typeof item.id !== "string") {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const score =
|
||||||
|
typeof item.score === "number" && Number.isFinite(item.score)
|
||||||
|
? Math.max(0, Math.min(1, item.score))
|
||||||
|
: 0;
|
||||||
|
return [{ id: item.id, score }];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export class LlmReranker implements Reranker {
|
||||||
|
constructor(private readonly llmClient: Pick<LlmClient, "chatCompletion">) {}
|
||||||
|
|
||||||
|
async rerank(
|
||||||
|
query: string,
|
||||||
|
candidates: RetrievedDocument[],
|
||||||
|
topN: number,
|
||||||
|
): Promise<RetrievedDocument[]> {
|
||||||
|
if (candidates.length <= 1) {
|
||||||
|
return candidates.slice(0, topN);
|
||||||
|
}
|
||||||
|
|
||||||
|
return withSpan(
|
||||||
|
"memory.reranker",
|
||||||
|
{
|
||||||
|
"memory.reranker.candidates_count": candidates.length,
|
||||||
|
"memory.reranker.top_n": topN,
|
||||||
|
},
|
||||||
|
async (span) => {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
try {
|
||||||
|
const result = await this.llmClient.chatCompletion(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
role: "system",
|
||||||
|
content: [
|
||||||
|
"너는 검색 결과 reranker다.",
|
||||||
|
"질문과 후보 문서를 보고 관련도가 높은 순서로만 정렬한다.",
|
||||||
|
'JSON으로만 응답한다: {"rankings":[{"id":"문서 id","score":0.0-1.0}]}',
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
`[query]\n${query}`,
|
||||||
|
"",
|
||||||
|
"[candidates]",
|
||||||
|
...candidates.map((candidate, index) =>
|
||||||
|
[
|
||||||
|
`#${index + 1} id=${candidate.id} score=${candidate.score.toFixed(3)}`,
|
||||||
|
truncate(candidate.text, 600),
|
||||||
|
].join("\n"),
|
||||||
|
),
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{
|
||||||
|
jsonMode: true,
|
||||||
|
maxTokens: 200,
|
||||||
|
temperature: 0,
|
||||||
|
tier: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!result?.content.trim()) {
|
||||||
|
return candidates.slice(0, topN);
|
||||||
|
}
|
||||||
|
|
||||||
|
const byId = new Map(candidates.map((candidate) => [candidate.id, candidate]));
|
||||||
|
const rankings = parseRankings(result.content);
|
||||||
|
const ranked = rankings
|
||||||
|
.flatMap((ranking) => {
|
||||||
|
const candidate = byId.get(ranking.id);
|
||||||
|
return candidate ? [{ ...candidate, score: ranking.score }] : [];
|
||||||
|
})
|
||||||
|
.slice(0, topN);
|
||||||
|
const seen = new Set(ranked.map((candidate) => candidate.id));
|
||||||
|
const next = [
|
||||||
|
...ranked,
|
||||||
|
...candidates.filter((candidate) => !seen.has(candidate.id)),
|
||||||
|
].slice(0, topN);
|
||||||
|
|
||||||
|
span.setAttributes({
|
||||||
|
"memory.reranker.changed_order": next[0]?.id !== candidates[0]?.id,
|
||||||
|
"memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt),
|
||||||
|
});
|
||||||
|
return next;
|
||||||
|
} catch (error) {
|
||||||
|
getRequestLogger().warn(
|
||||||
|
{ err: error },
|
||||||
|
"[Memory] Reranker failed; using original retrieval order",
|
||||||
|
);
|
||||||
|
span.setAttributes({
|
||||||
|
"memory.reranker.changed_order": false,
|
||||||
|
"memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt),
|
||||||
|
});
|
||||||
|
return candidates.slice(0, topN);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,7 +10,11 @@ import { getRequestLogger } from "@obs/context";
|
|||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||||
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||||
import { getRagEnabled } from "../../config.js";
|
import {
|
||||||
|
getRagEnabled,
|
||||||
|
getRagHybridEnabled,
|
||||||
|
getRagRerankerEnabled,
|
||||||
|
} from "../../config.js";
|
||||||
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
||||||
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
||||||
import { createModerationToolRegistry } from "./tools.js";
|
import { createModerationToolRegistry } from "./tools.js";
|
||||||
@@ -150,6 +154,8 @@ async function lookupReferenceText(
|
|||||||
try {
|
try {
|
||||||
const docs = await memoryService.longTerm.search(message, {
|
const docs = await memoryService.longTerm.search(message, {
|
||||||
category: "rule",
|
category: "rule",
|
||||||
|
hybrid: getRagHybridEnabled(),
|
||||||
|
rerank: getRagRerankerEnabled(),
|
||||||
skill: "moderation",
|
skill: "moderation",
|
||||||
topK: 3,
|
topK: 3,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -66,6 +66,8 @@ export const ONBOARDING_PROMPT_DATA = {
|
|||||||
"[답변 원칙]",
|
"[답변 원칙]",
|
||||||
"- 마지막 메시지에서 직접 물은 내용에만 정확히 답해.",
|
"- 마지막 메시지에서 직접 물은 내용에만 정확히 답해.",
|
||||||
"- 사용자가 요청하지 않은 다음 단계, 방법, 위치, 예시는 먼저 붙이지 마.",
|
"- 사용자가 요청하지 않은 다음 단계, 방법, 위치, 예시는 먼저 붙이지 마.",
|
||||||
|
"- [user_context] 블록이 있으면 그 안의 정보를 응답 톤과 우선순위에 반영해.",
|
||||||
|
"- [long_term_memory] 블록이 있으면 사용자의 과거 패턴을 참고하되, 사용자에게 직접 언급하지 마.",
|
||||||
"- 마크다운 문법, 링크, URL은 쓰지 마.",
|
"- 마크다운 문법, 링크, URL은 쓰지 마.",
|
||||||
'- 온보딩과 무관한 잡담이거나 다른 사람에게 하는 말이면 "[NO_RESPONSE]"만 출력해.',
|
'- 온보딩과 무관한 잡담이거나 다른 사람에게 하는 말이면 "[NO_RESPONSE]"만 출력해.',
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -144,6 +144,28 @@ function formatRetrievedKnowledge(context: PromptContext): string[] {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatProfileSnippets(context: PromptContext): string[] {
|
||||||
|
const snippets = context.profileSnippets ?? [];
|
||||||
|
if (snippets.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
'',
|
||||||
|
'[user_context]',
|
||||||
|
snippets.map((snippet) => `- ${snippet.block}`).join('\n'),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatLongTermMemory(context: PromptContext): string[] {
|
||||||
|
const summary = context.userProfile?.longTermSummary?.text?.trim();
|
||||||
|
if (!summary) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return ['', '[long_term_memory]', summary];
|
||||||
|
}
|
||||||
|
|
||||||
export function personaLayer(): string {
|
export function personaLayer(): string {
|
||||||
return ONBOARDING_PROMPT_DATA.persona;
|
return ONBOARDING_PROMPT_DATA.persona;
|
||||||
}
|
}
|
||||||
@@ -222,6 +244,8 @@ export function contextLayer(context: PromptContext): string {
|
|||||||
formatUserProfile(context.userProfile),
|
formatUserProfile(context.userProfile),
|
||||||
]
|
]
|
||||||
: []),
|
: []),
|
||||||
|
...formatProfileSnippets(context),
|
||||||
|
...formatLongTermMemory(context),
|
||||||
...formatRetrievedKnowledge(context),
|
...formatRetrievedKnowledge(context),
|
||||||
].join('\n');
|
].join('\n');
|
||||||
}
|
}
|
||||||
@@ -3,6 +3,7 @@ import type { GarnetClient } from '@common/garnet';
|
|||||||
import { getConversationSummaryKey } from '@conv/conversation-store';
|
import { getConversationSummaryKey } from '@conv/conversation-store';
|
||||||
import type { ChatMessage } from '@llm/types';
|
import type { ChatMessage } from '@llm/types';
|
||||||
import type { MemoryService } from '../../../memory/memory-types.js';
|
import type { MemoryService } from '../../../memory/memory-types.js';
|
||||||
|
import { deriveProfilePromptSnippets } from '../../../memory/profile-update-policy.js';
|
||||||
import { getRequestLogger } from '@obs/context';
|
import { getRequestLogger } from '@obs/context';
|
||||||
import type { RoomContextService } from '@room/room-context-service';
|
import type { RoomContextService } from '@room/room-context-service';
|
||||||
import { meetsTrustTier } from '../../../eval/governance/trust.js';
|
import { meetsTrustTier } from '../../../eval/governance/trust.js';
|
||||||
@@ -10,6 +11,8 @@ import {
|
|||||||
getAgentTrustMinTier,
|
getAgentTrustMinTier,
|
||||||
getAgentUserMemoryEnabled,
|
getAgentUserMemoryEnabled,
|
||||||
getRagEnabled,
|
getRagEnabled,
|
||||||
|
getRagHybridEnabled,
|
||||||
|
getRagRerankerEnabled,
|
||||||
getPromptVariantId,
|
getPromptVariantId,
|
||||||
} from '../../../config.js';
|
} from '../../../config.js';
|
||||||
import {
|
import {
|
||||||
@@ -127,6 +130,8 @@ export async function buildLlmMessages(
|
|||||||
query: buildRetrievalQuery(request, derived),
|
query: buildRetrievalQuery(request, derived),
|
||||||
options: {
|
options: {
|
||||||
category: 'onboarding-step',
|
category: 'onboarding-step',
|
||||||
|
hybrid: getRagHybridEnabled(),
|
||||||
|
rerank: getRagRerankerEnabled(),
|
||||||
skill: 'onboarding',
|
skill: 'onboarding',
|
||||||
topK: 2,
|
topK: 2,
|
||||||
},
|
},
|
||||||
@@ -173,6 +178,7 @@ export async function buildLlmMessages(
|
|||||||
throw new Error('Failed to load onboarding blackboard');
|
throw new Error('Failed to load onboarding blackboard');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const userProfile = workingMemory?.userProfile ?? legacyUserProfile;
|
||||||
const context: PromptContext = {
|
const context: PromptContext = {
|
||||||
blackboard,
|
blackboard,
|
||||||
conversationHistory:
|
conversationHistory:
|
||||||
@@ -180,10 +186,13 @@ export async function buildLlmMessages(
|
|||||||
? workingMemory.conversationHistory
|
? workingMemory.conversationHistory
|
||||||
: conversationHistory,
|
: conversationHistory,
|
||||||
roomContext,
|
roomContext,
|
||||||
|
profileSnippets: userProfile
|
||||||
|
? deriveProfilePromptSnippets(userProfile, derived)
|
||||||
|
: undefined,
|
||||||
retrievedKnowledge: workingMemory?.retrievedKnowledge,
|
retrievedKnowledge: workingMemory?.retrievedKnowledge,
|
||||||
summary: workingMemory?.summary ?? legacySummary,
|
summary: workingMemory?.summary ?? legacySummary,
|
||||||
userMessage: buildPromptUserMessage(request, derived),
|
userMessage: buildPromptUserMessage(request, derived),
|
||||||
userProfile: workingMemory?.userProfile ?? legacyUserProfile,
|
userProfile,
|
||||||
};
|
};
|
||||||
|
|
||||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||||
|
|||||||
@@ -2,6 +2,10 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/ty
|
|||||||
import type { GarnetClient } from '@common/garnet';
|
import type { GarnetClient } from '@common/garnet';
|
||||||
import type { NatsClient } from '@common/nats';
|
import type { NatsClient } from '@common/nats';
|
||||||
import type { MemoryService } from '../../memory/memory-types.js';
|
import type { MemoryService } from '../../memory/memory-types.js';
|
||||||
|
import {
|
||||||
|
buildLongTermSummary,
|
||||||
|
shouldRefreshLongTermSummary,
|
||||||
|
} from '../../memory/long-term-summarizer.js';
|
||||||
import { getContext, getRequestLogger } from '@obs/context';
|
import { getContext, getRequestLogger } from '@obs/context';
|
||||||
import type { LlmClient } from '@llm/llm-client';
|
import type { LlmClient } from '@llm/llm-client';
|
||||||
import type { RoomContextService } from '@room/room-context-service';
|
import type { RoomContextService } from '@room/room-context-service';
|
||||||
@@ -427,12 +431,43 @@ export class OnboardingSkill implements Skill {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
||||||
flaggedIssues:
|
flaggedIssues:
|
||||||
derived.phase.kind === 'nickname-rejected'
|
derived.phase.kind === 'nickname-rejected'
|
||||||
? derived.phase.issues.map((issue) => issue.message)
|
? derived.phase.issues.map((issue) => issue.message)
|
||||||
: [],
|
: [],
|
||||||
});
|
});
|
||||||
|
if (!shouldRefreshLongTermSummary(profile)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
void this.refreshLongTermSummary(request, profile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async refreshLongTermSummary(
|
||||||
|
request: AgentRequest,
|
||||||
|
profile: Awaited<ReturnType<UserMemoryStore['observeTurn']>>,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
const conversationSummary =
|
||||||
|
(await this.memoryService?.session.getSummary(
|
||||||
|
request.chatroomId,
|
||||||
|
request.userId,
|
||||||
|
)) ?? null;
|
||||||
|
const longTermSummary = await buildLongTermSummary(
|
||||||
|
this.llmClient,
|
||||||
|
profile,
|
||||||
|
conversationSummary,
|
||||||
|
);
|
||||||
|
await this.userMemoryStore.updateProfile(request.userId, {
|
||||||
|
longTermSummary,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
getRequestLogger().warn(
|
||||||
|
{ err: error },
|
||||||
|
'[OnboardingSkill] Failed to refresh long-term user summary',
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private makeTemplateFallbackResponse(
|
private makeTemplateFallbackResponse(
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import {
|
|||||||
anonymizeText,
|
anonymizeText,
|
||||||
} from "@eval/anonymize";
|
} from "@eval/anonymize";
|
||||||
import { getAgentUserMemoryTtlDays } from "../../../config.js";
|
import { getAgentUserMemoryTtlDays } from "../../../config.js";
|
||||||
|
import { evaluateUserFlags } from "../../../memory/forgetting-policy.js";
|
||||||
|
import type { LongTermSummary } from "../../../memory/long-term-summarizer.js";
|
||||||
import type { LongTermMemory } from "../../../memory/memory-types.js";
|
import type { LongTermMemory } from "../../../memory/memory-types.js";
|
||||||
import type { DerivedState, Intent } from "./onboarding-state.js";
|
import type { DerivedState, Intent } from "./onboarding-state.js";
|
||||||
|
|
||||||
@@ -11,9 +13,11 @@ export type PreferredStyle = "concise" | "neutral" | "warm";
|
|||||||
|
|
||||||
export interface UserProfile {
|
export interface UserProfile {
|
||||||
completedOnboardingAt?: string | undefined;
|
completedOnboardingAt?: string | undefined;
|
||||||
|
archivedFlaggedIssues?: string[] | undefined;
|
||||||
firstSeenAt: string;
|
firstSeenAt: string;
|
||||||
flaggedIssues: string[];
|
flaggedIssues: string[];
|
||||||
lastSeenAt: string;
|
lastSeenAt: string;
|
||||||
|
longTermSummary?: LongTermSummary | undefined;
|
||||||
pastIntents: Array<{ count: number; kind: Intent["kind"] }>;
|
pastIntents: Array<{ count: number; kind: Intent["kind"] }>;
|
||||||
preferredStyle: PreferredStyle;
|
preferredStyle: PreferredStyle;
|
||||||
totalTurns: number;
|
totalTurns: number;
|
||||||
@@ -115,14 +119,32 @@ export class UserMemoryStore {
|
|||||||
derived.phase.kind === "completed"
|
derived.phase.kind === "completed"
|
||||||
? current.completedOnboardingAt ?? now
|
? current.completedOnboardingAt ?? now
|
||||||
: current.completedOnboardingAt;
|
: current.completedOnboardingAt;
|
||||||
const next: UserProfile = {
|
const flagDecision = evaluateUserFlags(
|
||||||
...(completedOnboardingAt ? { completedOnboardingAt } : {}),
|
{
|
||||||
firstSeenAt: current.firstSeenAt,
|
...current,
|
||||||
flaggedIssues: sanitizeIssues([
|
flaggedIssues: sanitizeIssues([
|
||||||
...current.flaggedIssues,
|
...current.flaggedIssues,
|
||||||
...(options?.flaggedIssues ?? []),
|
...(options?.flaggedIssues ?? []),
|
||||||
]),
|
]),
|
||||||
|
},
|
||||||
|
10,
|
||||||
|
);
|
||||||
|
const next: UserProfile = {
|
||||||
|
...(current.archivedFlaggedIssues || flagDecision.archived.length > 0
|
||||||
|
? {
|
||||||
|
archivedFlaggedIssues: [
|
||||||
|
...(current.archivedFlaggedIssues ?? []),
|
||||||
|
...flagDecision.archived,
|
||||||
|
].slice(-50),
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
...(completedOnboardingAt ? { completedOnboardingAt } : {}),
|
||||||
|
firstSeenAt: current.firstSeenAt,
|
||||||
|
flaggedIssues: flagDecision.kept,
|
||||||
lastSeenAt: now,
|
lastSeenAt: now,
|
||||||
|
...(current.longTermSummary
|
||||||
|
? { longTermSummary: current.longTermSummary }
|
||||||
|
: {}),
|
||||||
pastIntents: mergeIntent(current.pastIntents, derived.intent.kind),
|
pastIntents: mergeIntent(current.pastIntents, derived.intent.kind),
|
||||||
preferredStyle: inferPreferredStyle(
|
preferredStyle: inferPreferredStyle(
|
||||||
current.preferredStyle,
|
current.preferredStyle,
|
||||||
@@ -140,4 +162,29 @@ export class UserMemoryStore {
|
|||||||
}
|
}
|
||||||
return next;
|
return next;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async updateProfile(
|
||||||
|
userId: string,
|
||||||
|
patch: Partial<UserProfile>,
|
||||||
|
): Promise<UserProfile> {
|
||||||
|
const current = (await this.load(userId)) ?? emptyProfile(userId);
|
||||||
|
const next: UserProfile = {
|
||||||
|
...current,
|
||||||
|
...patch,
|
||||||
|
firstSeenAt: patch.firstSeenAt ?? current.firstSeenAt,
|
||||||
|
flaggedIssues: patch.flaggedIssues ?? current.flaggedIssues,
|
||||||
|
lastSeenAt: patch.lastSeenAt ?? nowIso(),
|
||||||
|
pastIntents: patch.pastIntents ?? current.pastIntents,
|
||||||
|
preferredStyle: patch.preferredStyle ?? current.preferredStyle,
|
||||||
|
totalTurns: patch.totalTurns ?? current.totalTurns,
|
||||||
|
userId: current.userId,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (this.longTermMemory) {
|
||||||
|
await this.longTermMemory.updateUserProfile(userId, next);
|
||||||
|
} else {
|
||||||
|
await this.garnetClient.setJson(profileKey(userId), next, ttlSeconds());
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import type { ConversationMessage } from "@agent/types";
|
import type { ConversationMessage } from "@agent/types";
|
||||||
|
import type { ProfilePromptSnippet } from "../../memory/profile-update-policy.js";
|
||||||
import type { RetrievedDocument } from "../../memory/memory-types.js";
|
import type { RetrievedDocument } from "../../memory/memory-types.js";
|
||||||
import type { RoomMessage } from "@room/types";
|
import type { RoomMessage } from "@room/types";
|
||||||
import type { UserProfile } from "./state/user-memory.js";
|
import type { UserProfile } from "./state/user-memory.js";
|
||||||
@@ -27,6 +28,7 @@ export interface Example {
|
|||||||
export interface PromptContext {
|
export interface PromptContext {
|
||||||
blackboard: Blackboard;
|
blackboard: Blackboard;
|
||||||
conversationHistory: ConversationMessage[];
|
conversationHistory: ConversationMessage[];
|
||||||
|
profileSnippets?: ProfilePromptSnippet[] | undefined;
|
||||||
roomContext?: RoomMessage[] | undefined;
|
roomContext?: RoomMessage[] | undefined;
|
||||||
retrievedKnowledge?: RetrievedDocument[] | undefined;
|
retrievedKnowledge?: RetrievedDocument[] | undefined;
|
||||||
summary?: string | undefined;
|
summary?: string | undefined;
|
||||||
|
|||||||
@@ -2,12 +2,14 @@ import type { GarnetClient } from "@common/garnet";
|
|||||||
import type { NatsClient } from "@common/nats";
|
import type { NatsClient } from "@common/nats";
|
||||||
import type { ShadowEvalBus } from "@eval/shadow-bus";
|
import type { ShadowEvalBus } from "@eval/shadow-bus";
|
||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import type { EscalationHistoryService } from "../memory/escalation-history.js";
|
||||||
import type { MemoryService } from "../memory/memory-types.js";
|
import type { MemoryService } from "../memory/memory-types.js";
|
||||||
import type { RoomContextService } from "@room/room-context-service";
|
import type { RoomContextService } from "@room/room-context-service";
|
||||||
import type { Skill } from "./skill.js";
|
import type { Skill } from "./skill.js";
|
||||||
import type { SkillRegistry } from "./skill-loader.js";
|
import type { SkillRegistry } from "./skill-loader.js";
|
||||||
|
|
||||||
export interface SkillDeps {
|
export interface SkillDeps {
|
||||||
|
escalationHistoryService?: EscalationHistoryService | undefined;
|
||||||
garnetClient: GarnetClient;
|
garnetClient: GarnetClient;
|
||||||
llmClient: LlmClient;
|
llmClient: LlmClient;
|
||||||
memoryService?: MemoryService | undefined;
|
memoryService?: MemoryService | undefined;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||||
|
import type { UserEscalationHistory } from "../../memory/escalation-history.js";
|
||||||
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
|
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
|
||||||
|
|
||||||
function formatSkill(manifest: SkillManifest): string {
|
function formatSkill(manifest: SkillManifest): string {
|
||||||
@@ -14,16 +15,35 @@ function formatSkill(manifest: SkillManifest): string {
|
|||||||
export function buildSupervisorSystemPrompt(
|
export function buildSupervisorSystemPrompt(
|
||||||
registry: SkillRegistry,
|
registry: SkillRegistry,
|
||||||
_packet?: HandoffPacket,
|
_packet?: HandoffPacket,
|
||||||
|
history?: UserEscalationHistory | null,
|
||||||
): string {
|
): string {
|
||||||
const skills = registry
|
const skills = registry
|
||||||
.list()
|
.list()
|
||||||
.filter((manifest) => manifest.name !== "supervisor");
|
.filter((manifest) => manifest.name !== "supervisor");
|
||||||
|
const historyBlock = history
|
||||||
|
? [
|
||||||
|
"",
|
||||||
|
"[user_escalation_history]",
|
||||||
|
`총 escalation: ${history.totalEscalations}`,
|
||||||
|
`마지막 escalation: ${history.lastEscalationAt}`,
|
||||||
|
`top reasons: ${history.topReasons.map((item) => `${item.reason}(${item.count})`).join(", ") || "-"}`,
|
||||||
|
`recent resolutions: ${
|
||||||
|
history.escalations
|
||||||
|
.slice(-3)
|
||||||
|
.map((item) => item.resolution?.action)
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(", ") || "-"
|
||||||
|
}`,
|
||||||
|
"- 이 블록은 라우팅 참고용이다. 사용자에게 직접 언급하지 않는다.",
|
||||||
|
]
|
||||||
|
: [];
|
||||||
return [
|
return [
|
||||||
"너는 multi-agent 시스템의 supervisor다.",
|
"너는 multi-agent 시스템의 supervisor다.",
|
||||||
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
|
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
|
||||||
"",
|
"",
|
||||||
"[사용 가능한 specialist]",
|
"[사용 가능한 specialist]",
|
||||||
...skills.map(formatSkill),
|
...skills.map(formatSkill),
|
||||||
|
...historyBlock,
|
||||||
"",
|
"",
|
||||||
"[응답 형식]",
|
"[응답 형식]",
|
||||||
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}',
|
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}',
|
||||||
|
|||||||
@@ -10,5 +10,6 @@ export function createSupervisorSkill(deps: SkillDeps): Skill {
|
|||||||
deps.natsClient,
|
deps.natsClient,
|
||||||
deps.llmClient,
|
deps.llmClient,
|
||||||
deps.skillRegistry,
|
deps.skillRegistry,
|
||||||
|
deps.escalationHistoryService,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||||
import type { HandoffReason } from "@agent/types";
|
import type { HandoffReason } from "@agent/types";
|
||||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||||
|
import type { EscalationHistoryService } from "../../memory/escalation-history.js";
|
||||||
import {
|
import {
|
||||||
getSupervisorResolutionsStreamMaxLen,
|
getSupervisorResolutionsStreamMaxLen,
|
||||||
getSupervisorReviewQueueMaxLen,
|
getSupervisorReviewQueueMaxLen,
|
||||||
@@ -152,13 +153,13 @@ function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean
|
|||||||
export async function appendSupervisorReview(
|
export async function appendSupervisorReview(
|
||||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||||
fields: Record<string, string>,
|
fields: Record<string, string>,
|
||||||
): Promise<void> {
|
): Promise<string> {
|
||||||
await garnetClient.appendStream(
|
return garnetClient.appendStream(
|
||||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||||
{
|
{
|
||||||
...fields,
|
...fields,
|
||||||
type: "supervisor-handoff",
|
type: "supervisor-handoff",
|
||||||
queuedAt: new Date().toISOString(),
|
queuedAt: fields["queuedAt"] ?? new Date().toISOString(),
|
||||||
},
|
},
|
||||||
{ maxLen: getSupervisorReviewQueueMaxLen() },
|
{ maxLen: getSupervisorReviewQueueMaxLen() },
|
||||||
);
|
);
|
||||||
@@ -275,6 +276,10 @@ export async function resolveSupervisorReview(
|
|||||||
garnetClient: SupervisorReviewResolveGarnetClient,
|
garnetClient: SupervisorReviewResolveGarnetClient,
|
||||||
id: string,
|
id: string,
|
||||||
resolution: SupervisorReviewResolution,
|
resolution: SupervisorReviewResolution,
|
||||||
|
options?: {
|
||||||
|
escalationHistoryService?: EscalationHistoryService | undefined;
|
||||||
|
userId?: string | undefined;
|
||||||
|
},
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const fields: Record<string, string> = {
|
const fields: Record<string, string> = {
|
||||||
action: resolution.action,
|
action: resolution.action,
|
||||||
@@ -297,6 +302,13 @@ export async function resolveSupervisorReview(
|
|||||||
},
|
},
|
||||||
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
|
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
|
||||||
);
|
);
|
||||||
|
if (options?.escalationHistoryService && options.userId) {
|
||||||
|
await options.escalationHistoryService.recordResolution({
|
||||||
|
userId: options.userId,
|
||||||
|
reviewId: id,
|
||||||
|
resolution,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getResolutionHistory(
|
export async function getResolutionHistory(
|
||||||
|
|||||||
@@ -8,6 +8,10 @@ import type { HandoffPacket } from "@agent/handoff-packet";
|
|||||||
import type { GarnetClient } from "@common/garnet";
|
import type { GarnetClient } from "@common/garnet";
|
||||||
import type { NatsClient } from "@common/nats";
|
import type { NatsClient } from "@common/nats";
|
||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import type {
|
||||||
|
EscalationHistoryService,
|
||||||
|
UserEscalationHistory,
|
||||||
|
} from "../../memory/escalation-history.js";
|
||||||
import { AlertService } from "@obs/alert";
|
import { AlertService } from "@obs/alert";
|
||||||
import { metrics } from "@obs/metrics";
|
import { metrics } from "@obs/metrics";
|
||||||
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
|
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
|
||||||
@@ -87,6 +91,7 @@ export class SupervisorSkill implements Skill {
|
|||||||
natsClient: Pick<NatsClient, "publish">,
|
natsClient: Pick<NatsClient, "publish">,
|
||||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||||
|
private readonly escalationHistoryService?: EscalationHistoryService | undefined,
|
||||||
) {
|
) {
|
||||||
this.alertService = new AlertService(
|
this.alertService = new AlertService(
|
||||||
garnetClient,
|
garnetClient,
|
||||||
@@ -108,7 +113,8 @@ export class SupervisorSkill implements Skill {
|
|||||||
_conversationHistory: ConversationMessage[],
|
_conversationHistory: ConversationMessage[],
|
||||||
): Promise<AgentResponse> {
|
): Promise<AgentResponse> {
|
||||||
const packet = readHandoffPacket(request);
|
const packet = readHandoffPacket(request);
|
||||||
const llmDecision = await this.routeByLlm(packet);
|
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
||||||
|
const llmDecision = await this.routeByLlm(packet, escalationHistory);
|
||||||
const useLlmRoute =
|
const useLlmRoute =
|
||||||
llmDecision !== null &&
|
llmDecision !== null &&
|
||||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||||
@@ -117,14 +123,26 @@ export class SupervisorSkill implements Skill {
|
|||||||
: routeSupervisorTarget(packet);
|
: routeSupervisorTarget(packet);
|
||||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||||
|
|
||||||
await appendSupervisorReview(this.garnetClient, {
|
const queuedAt = new Date().toISOString();
|
||||||
|
const reviewId = await appendSupervisorReview(this.garnetClient, {
|
||||||
chatroomId: request.chatroomId,
|
chatroomId: request.chatroomId,
|
||||||
packet: JSON.stringify(packet ?? null),
|
packet: JSON.stringify(packet ?? null),
|
||||||
|
queuedAt,
|
||||||
reason,
|
reason,
|
||||||
requestId: request.requestId,
|
requestId: request.requestId,
|
||||||
target: target ?? "human-review",
|
target: target ?? "human-review",
|
||||||
userId: request.userId,
|
userId: request.userId,
|
||||||
});
|
});
|
||||||
|
await this.escalationHistoryService?.recordEscalation({
|
||||||
|
chatroomId: request.chatroomId,
|
||||||
|
packetSummary: packet?.userGoal ?? packet?.summary ?? "",
|
||||||
|
queuedAt,
|
||||||
|
reason,
|
||||||
|
reviewId,
|
||||||
|
sourceChain: packet?.sourceChain ?? [],
|
||||||
|
target: target ?? "human-review",
|
||||||
|
userId: request.userId,
|
||||||
|
});
|
||||||
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
|
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
|
||||||
|
|
||||||
await this.alertService.sendAlert(
|
await this.alertService.sendAlert(
|
||||||
@@ -154,6 +172,7 @@ export class SupervisorSkill implements Skill {
|
|||||||
routedBy: useLlmRoute ? "llm" : "rules",
|
routedBy: useLlmRoute ? "llm" : "rules",
|
||||||
source: userMessage.source,
|
source: userMessage.source,
|
||||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||||
|
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
|
||||||
suggestedTarget: target,
|
suggestedTarget: target,
|
||||||
},
|
},
|
||||||
requestId: request.requestId,
|
requestId: request.requestId,
|
||||||
@@ -164,6 +183,7 @@ export class SupervisorSkill implements Skill {
|
|||||||
|
|
||||||
private async routeByLlm(
|
private async routeByLlm(
|
||||||
packet?: HandoffPacket,
|
packet?: HandoffPacket,
|
||||||
|
escalationHistory?: UserEscalationHistory | null,
|
||||||
): Promise<SupervisorRoutingDecision | null> {
|
): Promise<SupervisorRoutingDecision | null> {
|
||||||
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
||||||
return null;
|
return null;
|
||||||
@@ -173,7 +193,11 @@ export class SupervisorSkill implements Skill {
|
|||||||
const result = await this.llmClient.chatCompletion(
|
const result = await this.llmClient.chatCompletion(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
|
content: buildSupervisorSystemPrompt(
|
||||||
|
this.skillRegistry,
|
||||||
|
packet,
|
||||||
|
escalationHistory,
|
||||||
|
),
|
||||||
role: "system",
|
role: "system",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -213,4 +237,19 @@ export class SupervisorSkill implements Skill {
|
|||||||
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async loadEscalationHistory(userId: string) {
|
||||||
|
if (!this.escalationHistoryService) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await this.escalationHistoryService.load(userId);
|
||||||
|
} catch (error) {
|
||||||
|
getLogger().warn(
|
||||||
|
{ err: error },
|
||||||
|
"[Supervisor] Failed to load escalation history",
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,11 @@
|
|||||||
import { readFileSync } from "node:fs";
|
import { readFileSync } from "node:fs";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { getRagEnabled } from "../../config.js";
|
import {
|
||||||
|
getRagEnabled,
|
||||||
|
getRagHybridEnabled,
|
||||||
|
getRagRerankerEnabled,
|
||||||
|
} from "../../config.js";
|
||||||
import { getRequestLogger } from "../../observability/context.js";
|
import { getRequestLogger } from "../../observability/context.js";
|
||||||
import type { Tool } from "../tool.js";
|
import type { Tool } from "../tool.js";
|
||||||
|
|
||||||
@@ -96,6 +100,8 @@ export const lookupRuleTool: Tool<
|
|||||||
try {
|
try {
|
||||||
const docs = await ctx.memoryService.longTerm.search(args.topic, {
|
const docs = await ctx.memoryService.longTerm.search(args.topic, {
|
||||||
category: "rule",
|
category: "rule",
|
||||||
|
hybrid: getRagHybridEnabled(),
|
||||||
|
rerank: getRagRerankerEnabled(),
|
||||||
skill: "moderation",
|
skill: "moderation",
|
||||||
topK: args.topK ?? 3,
|
topK: args.topK ?? 3,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { getRagEnabled } from "../../config.js";
|
import {
|
||||||
|
getRagEnabled,
|
||||||
|
getRagHybridEnabled,
|
||||||
|
getRagRerankerEnabled,
|
||||||
|
} from "../../config.js";
|
||||||
import { getRequestLogger } from "../../observability/context.js";
|
import { getRequestLogger } from "../../observability/context.js";
|
||||||
import {
|
import {
|
||||||
DEADLINE_NEAR_THRESHOLD_MIN,
|
DEADLINE_NEAR_THRESHOLD_MIN,
|
||||||
@@ -93,6 +97,8 @@ export const lookupCommunityRuleTool: Tool<
|
|||||||
try {
|
try {
|
||||||
const docs = await ctx.memoryService.longTerm.search(args.topic, {
|
const docs = await ctx.memoryService.longTerm.search(args.topic, {
|
||||||
category: "onboarding-step",
|
category: "onboarding-step",
|
||||||
|
hybrid: getRagHybridEnabled(),
|
||||||
|
rerank: getRagRerankerEnabled(),
|
||||||
skill: "onboarding",
|
skill: "onboarding",
|
||||||
topK: args.topK ?? 3,
|
topK: args.topK ?? 3,
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in new issue
Block a user