Add hybrid memory retrieval and escalation history
This commit is contained in:
1 parent
1a86bded0b
commit
211de18476
38 files changed
+1602
-37
No files matched your search
@@ -7,8 +7,13 @@ import {
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getEmbeddingModel,
|
||||
getEmbeddingProvider,
|
||||
getLongTermSummaryEnabled,
|
||||
getMemoryEvictionEnabled,
|
||||
getPhoenixLogPrompts,
|
||||
getPhoenixOtlpEndpoint,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
getRagRerankerTopK,
|
||||
getRagEnabled,
|
||||
getSlackAlertWebhookUrl,
|
||||
getSlackWebhookTimeoutMs,
|
||||
@@ -25,6 +30,11 @@ const ENV_KEYS = [
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
||||
"AGENT_RAG_ENABLED",
|
||||
"AGENT_RAG_HYBRID_ENABLED",
|
||||
"AGENT_RAG_RERANKER_ENABLED",
|
||||
"AGENT_RAG_RERANKER_TOP_K",
|
||||
"AGENT_LONG_TERM_SUMMARY_ENABLED",
|
||||
"AGENT_MEMORY_EVICTION_ENABLED",
|
||||
"EMBEDDING_MODEL",
|
||||
"EMBEDDING_PROVIDER",
|
||||
"PHOENIX_LOG_PROMPTS",
|
||||
@@ -85,14 +95,29 @@ describe("config", () => {
|
||||
|
||||
it("reads RAG and embedding configuration", () => {
|
||||
expect(getRagEnabled()).toBe(true);
|
||||
expect(getRagHybridEnabled()).toBe(true);
|
||||
expect(getRagRerankerEnabled()).toBe(true);
|
||||
expect(getRagRerankerTopK()).toBe(5);
|
||||
expect(getLongTermSummaryEnabled()).toBe(true);
|
||||
expect(getMemoryEvictionEnabled()).toBe(false);
|
||||
expect(getEmbeddingProvider()).toBe("openrouter");
|
||||
expect(getEmbeddingModel()).toBe("openai/text-embedding-3-small");
|
||||
|
||||
vi.stubEnv("AGENT_RAG_ENABLED", "false");
|
||||
vi.stubEnv("AGENT_RAG_HYBRID_ENABLED", "false");
|
||||
vi.stubEnv("AGENT_RAG_RERANKER_ENABLED", "false");
|
||||
vi.stubEnv("AGENT_RAG_RERANKER_TOP_K", "2");
|
||||
vi.stubEnv("AGENT_LONG_TERM_SUMMARY_ENABLED", "false");
|
||||
vi.stubEnv("AGENT_MEMORY_EVICTION_ENABLED", "true");
|
||||
vi.stubEnv("EMBEDDING_PROVIDER", "openai");
|
||||
vi.stubEnv("EMBEDDING_MODEL", "custom-embedding");
|
||||
|
||||
expect(getRagEnabled()).toBe(false);
|
||||
expect(getRagHybridEnabled()).toBe(false);
|
||||
expect(getRagRerankerEnabled()).toBe(false);
|
||||
expect(getRagRerankerTopK()).toBe(2);
|
||||
expect(getLongTermSummaryEnabled()).toBe(false);
|
||||
expect(getMemoryEvictionEnabled()).toBe(true);
|
||||
expect(getEmbeddingProvider()).toBe("openai");
|
||||
expect(getEmbeddingModel()).toBe("custom-embedding");
|
||||
});
|
||||
|
||||
@@ -133,6 +133,30 @@ export function getRagEnabled(): boolean {
|
||||
return getBoolean("AGENT_RAG_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getRagHybridEnabled(): boolean {
|
||||
return getBoolean("AGENT_RAG_HYBRID_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getRagRerankerEnabled(): boolean {
|
||||
return getBoolean("AGENT_RAG_RERANKER_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getRagRerankerTopK(): number {
|
||||
return getPositiveNumber("AGENT_RAG_RERANKER_TOP_K", 5);
|
||||
}
|
||||
|
||||
export function getRagCandidatePoolSize(): number {
|
||||
return getPositiveNumber("AGENT_RAG_CANDIDATE_POOL_SIZE", 50);
|
||||
}
|
||||
|
||||
export function getLongTermSummaryEnabled(): boolean {
|
||||
return getBoolean("AGENT_LONG_TERM_SUMMARY_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getMemoryEvictionEnabled(): boolean {
|
||||
return getBoolean("AGENT_MEMORY_EVICTION_ENABLED", false);
|
||||
}
|
||||
|
||||
export function getEmbeddingProvider(): "openrouter" | "openai" {
|
||||
const raw = process.env["EMBEDDING_PROVIDER"]?.trim().toLowerCase();
|
||||
return raw === "openai" ? "openai" : "openrouter";
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{"id":"memory-retrieval-accurate","name":"retrieval-accurate","input":{"request":{"requestId":"mem_mod_ads","skill":"moderation","chatroomId":"memory_room","userId":"memory_user_1","nickname":"tester","trigger":"user-message","userMessage":"광고 도배 기준이 뭐예요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["광고","도배"]}}
|
||||
{"id":"memory-long-range-understanding","name":"long-range-understanding","input":{"request":{"requestId":"mem_onboarding_recall","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_2","nickname":"LolName_KR/95/남/D","trigger":"user-message","userMessage":"내 닉네임 뭐라고 했지?"},"conversationHistory":[{"role":"user","content":"디스코드 닉네임은 LolName_KR입니다","timestamp":"2026-05-01T00:00:00.000Z"},{"role":"assistant","content":"확인했어요.","timestamp":"2026-05-01T00:00:01.000Z"}],"summary":"사용자가 디스코드 닉네임은 LolName_KR이라고 말했다."},"expected":{"mustIncludeAny":["LolName_KR"],"expectedMemory":{"expectedRecalledFact":"LolName_KR"}}}
|
||||
{"id":"memory-test-time-learning","name":"test-time-learning","input":{"request":{"requestId":"mem_onboarding_hint","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_3","nickname":"낭만/00/남D","trigger":"nickname-invalid"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":["닉네임 형식 오류","닉네임 형식 오류","닉네임 형식 오류"],"pastIntents":[{"kind":"nickname-question","count":5}],"preferredStyle":"concise","totalTurns":12,"userId":"user_memory"}},"expected":{"mustIncludeAny":["예시","형식"],"expectedMemory":{"expectedProphylacticHint":true}}}
|
||||
{"id":"memory-selective-forgetting","name":"selective-forgetting","input":{"request":{"requestId":"mem_forgetting","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_4","nickname":"tester","trigger":"user-message","userMessage":"닉네임 규칙 최신 기준 알려줘"},"summary":"정정된 최신 규칙만 답해야 한다."},"expected":{"mustNotInclude":["예전 기준"],"expectedMemory":{"expectedArchivedDocPattern":"예전 기준"}}}
|
||||
{"id":"memory-escalation-history","name":"escalation-history","input":{"request":{"requestId":"mem_supervisor_history","skill":"supervisor","chatroomId":"memory_room","userId":"memory_user_5","nickname":"tester","trigger":"user-message","context":{"handoffPacket":{"constraints":[],"currentState":{},"evidence":[],"expectedOutputSchema":"text","handoffReason":"out-of-scope","sourceChain":["onboarding"],"summary":"광고성 메시지로 보임","userGoal":"광고 홍보 가능 여부 확인"}}}},"expected":{"expectedMemory":{"expectedSupervisorTarget":"moderation"}}}
|
||||
{"id":"memory-contradiction","name":"memory-contradiction","input":{"request":{"requestId":"mem_contradiction","skill":"onboarding","chatroomId":"memory_room","userId":"memory_user_6","nickname":"새닉/95/남/D","trigger":"user-message","userMessage":"닉네임 새닉으로 변경했어요"},"userProfile":{"firstSeenAt":"2026-05-01T00:00:00.000Z","lastSeenAt":"2026-05-18T00:00:00.000Z","flaggedIssues":[],"pastIntents":[{"kind":"nickname-question","count":4}],"preferredStyle":"neutral","totalTurns":8,"userId":"user_memory","longTermSummary":{"text":"사용자는 이전 닉네임에서 새닉으로 변경했다고 말했다.","updatedAt":"2026-05-18T00:00:00.000Z","turnSnapshot":8}}},"expected":{"mustIncludeAny":["새닉"],"expectedMemory":{"expectedContradictionResolved":true,"expectedRecalledFact":"새닉"}}}
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { AgentResponse } from "@agent/types";
|
||||
import type { RetrievedDocument } from "../../memory/memory-types.js";
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
|
||||
export interface MemoryJudgeExpected {
|
||||
expectedArchivedDocPattern?: string | undefined;
|
||||
expectedContradictionResolved?: boolean | undefined;
|
||||
expectedProphylacticHint?: boolean | undefined;
|
||||
expectedRecalledFact?: string | undefined;
|
||||
expectedRetrievedDocIds?: string[] | undefined;
|
||||
expectedSupervisorTarget?: string | undefined;
|
||||
}
|
||||
|
||||
export interface MemoryJudgeResult {
|
||||
checks: Array<{
|
||||
detail?: string | undefined;
|
||||
name: keyof MemoryJudgeExpected;
|
||||
passed: boolean;
|
||||
skipped: boolean;
|
||||
}>;
|
||||
passed: boolean;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export function judgeMemory(
|
||||
expected: MemoryJudgeExpected | undefined,
|
||||
response: AgentResponse,
|
||||
metadata?: {
|
||||
profile?: UserProfile | undefined;
|
||||
retrievedDocs?: RetrievedDocument[] | undefined;
|
||||
},
|
||||
): MemoryJudgeResult {
|
||||
if (!expected || Object.keys(expected).length === 0) {
|
||||
return { checks: [], passed: true, score: 1 };
|
||||
}
|
||||
|
||||
const checks: MemoryJudgeResult["checks"] = [];
|
||||
if (expected.expectedRecalledFact) {
|
||||
checks.push({
|
||||
name: "expectedRecalledFact",
|
||||
passed: response.text.includes(expected.expectedRecalledFact),
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
if (expected.expectedProphylacticHint !== undefined) {
|
||||
const hasHint = /(예시|형식|가이드|주의|먼저)/u.test(response.text);
|
||||
checks.push({
|
||||
name: "expectedProphylacticHint",
|
||||
passed: expected.expectedProphylacticHint ? hasHint : !hasHint,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
if (expected.expectedSupervisorTarget) {
|
||||
const suggestedTarget = response.metadata?.["suggestedTarget"];
|
||||
checks.push({
|
||||
detail: typeof suggestedTarget === "string" ? suggestedTarget : "",
|
||||
name: "expectedSupervisorTarget",
|
||||
passed: suggestedTarget === expected.expectedSupervisorTarget,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
if (expected.expectedRetrievedDocIds) {
|
||||
const ids = new Set(metadata?.retrievedDocs?.map((doc) => doc.id) ?? []);
|
||||
checks.push({
|
||||
name: "expectedRetrievedDocIds",
|
||||
passed: expected.expectedRetrievedDocIds.every((id) => ids.has(id)),
|
||||
skipped: ids.size === 0,
|
||||
});
|
||||
}
|
||||
if (expected.expectedArchivedDocPattern) {
|
||||
checks.push({
|
||||
name: "expectedArchivedDocPattern",
|
||||
passed: !response.text.includes(expected.expectedArchivedDocPattern),
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
if (expected.expectedContradictionResolved !== undefined) {
|
||||
checks.push({
|
||||
name: "expectedContradictionResolved",
|
||||
passed: expected.expectedContradictionResolved
|
||||
? !/(예전|이전|기존)/u.test(response.text)
|
||||
: true,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
|
||||
const activeChecks = checks.filter((check) => !check.skipped);
|
||||
const passedCount = activeChecks.filter((check) => check.passed).length;
|
||||
const score = activeChecks.length === 0 ? 1 : passedCount / activeChecks.length;
|
||||
return {
|
||||
checks,
|
||||
passed: score === 1,
|
||||
score,
|
||||
};
|
||||
}
|
||||
+53
-9
@@ -7,6 +7,7 @@ import { GarnetClient, type StreamEntry } from "@common/garnet";
|
||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||
import { NatsClient } from "@common/nats";
|
||||
import { getConversationSummaryKey } from "@conv/conversation-store";
|
||||
import { anonymizeIdentifier } from "@eval/anonymize";
|
||||
import { LlmClient } from "@llm/llm-client";
|
||||
import type { ProviderName } from "@llm/types";
|
||||
import { runWithContext } from "@obs/context";
|
||||
@@ -34,6 +35,7 @@ import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js
|
||||
import { runEvalDispatch } from "./handoff-dispatch.js";
|
||||
import { judgeHandoff } from "./judges/handoff-judge.js";
|
||||
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeMemory } from "./judges/memory-judge.js";
|
||||
import { judgeProcessRule } from "./judges/process-judge.js";
|
||||
import { judgeRuleCase } from "./judges/rule-judge.js";
|
||||
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||
@@ -50,6 +52,7 @@ import type {
|
||||
EvalSummaryDelta,
|
||||
} from "./types.js";
|
||||
import { createMemoryService } from "../memory/memory-service.js";
|
||||
import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory";
|
||||
|
||||
const DEFAULT_REPORTS_DIR = "eval-reports";
|
||||
const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
|
||||
@@ -71,7 +74,7 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
|
||||
moderation: (deps) =>
|
||||
new ModerationSkill(
|
||||
deps.llm,
|
||||
createMemoryService({ garnetClient: deps.garnet }),
|
||||
createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }),
|
||||
),
|
||||
onboarding: (deps) =>
|
||||
new OnboardingSkill(
|
||||
@@ -80,7 +83,7 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
|
||||
deps.room,
|
||||
noopShadowEvalPublisher,
|
||||
undefined,
|
||||
createMemoryService({ garnetClient: deps.garnet }),
|
||||
createMemoryService({ garnetClient: deps.garnet, llmClient: deps.llm }),
|
||||
),
|
||||
supervisor: (deps) =>
|
||||
new SupervisorSkill(deps.garnet, {
|
||||
@@ -451,6 +454,9 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
|
||||
if (mustNotInclude) {
|
||||
expected.mustNotInclude = mustNotInclude;
|
||||
}
|
||||
if (isRecord(raw["expectedMemory"])) {
|
||||
expected.expectedMemory = raw["expectedMemory"] as EvalCaseExpected["expectedMemory"];
|
||||
}
|
||||
|
||||
const processExpectedRaw = raw["processExpected"];
|
||||
if (isRecord(processExpectedRaw)) {
|
||||
@@ -569,6 +575,10 @@ function parseEvalCaseLine(
|
||||
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
|
||||
}
|
||||
|
||||
if (raw.input["userProfile"] === null || isRecord(raw.input["userProfile"])) {
|
||||
input.userProfile = raw.input["userProfile"] as EvalCaseInput["userProfile"];
|
||||
}
|
||||
|
||||
return {
|
||||
id:
|
||||
typeof raw.id === "string" && raw.id.trim()
|
||||
@@ -646,6 +656,15 @@ async function hydrateEvalState(
|
||||
input.summary,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.userProfile !== undefined) {
|
||||
await garnetClient.setJson(
|
||||
`${USER_MEMORY_KEY_PREFIX}:${
|
||||
anonymizeIdentifier("user", userId) ?? "unknown"
|
||||
}`,
|
||||
input.userProfile,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
||||
@@ -688,7 +707,11 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
||||
}
|
||||
|
||||
function isCasePassed(result: EvalCaseResult): boolean {
|
||||
return result.ruleJudge.passed && (result.handoffJudge?.passed ?? true);
|
||||
return (
|
||||
result.ruleJudge.passed &&
|
||||
(result.handoffJudge?.passed ?? true) &&
|
||||
(result.memoryJudge?.passed ?? true)
|
||||
);
|
||||
}
|
||||
|
||||
function createBaselineDelta(
|
||||
@@ -943,6 +966,13 @@ async function evaluateOnboardingCase(
|
||||
processTraceSnapshot,
|
||||
judgeConfig.toolRegistryFactory(),
|
||||
);
|
||||
const memoryJudge = judgeMemory(
|
||||
evalCase.expected.expectedMemory,
|
||||
response,
|
||||
{
|
||||
profile: input.userProfile ?? undefined,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
@@ -955,6 +985,7 @@ async function evaluateOnboardingCase(
|
||||
response,
|
||||
ruleJudge,
|
||||
llmJudge,
|
||||
memoryJudge,
|
||||
processJudge,
|
||||
processTrace: processTraceSnapshot,
|
||||
};
|
||||
@@ -1028,13 +1059,17 @@ async function evaluateHandoffCase(
|
||||
skills,
|
||||
});
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const handoffJudge = judgeHandoff(
|
||||
evalCase.expected,
|
||||
result.response,
|
||||
result.finalSkill,
|
||||
);
|
||||
const handoffJudge = judgeHandoff(
|
||||
evalCase.expected,
|
||||
result.response,
|
||||
result.finalSkill,
|
||||
);
|
||||
const memoryJudge = judgeMemory(
|
||||
evalCase.expected.expectedMemory,
|
||||
result.response,
|
||||
);
|
||||
|
||||
return {
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
name: evalCase.name,
|
||||
@@ -1060,6 +1095,7 @@ async function evaluateHandoffCase(
|
||||
reason: "handoff-eval",
|
||||
},
|
||||
handoffJudge,
|
||||
memoryJudge,
|
||||
};
|
||||
} catch (error) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
@@ -1129,6 +1165,13 @@ async function evaluateGenericSkillCase(
|
||||
input.conversationHistory ?? [],
|
||||
);
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const memoryJudge = judgeMemory(
|
||||
evalCase.expected.expectedMemory,
|
||||
response,
|
||||
{
|
||||
profile: input.userProfile ?? undefined,
|
||||
},
|
||||
);
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
@@ -1160,6 +1203,7 @@ async function evaluateGenericSkillCase(
|
||||
skipped: true,
|
||||
reason: "disabled",
|
||||
},
|
||||
memoryJudge,
|
||||
};
|
||||
} catch (error) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
|
||||
@@ -5,12 +5,17 @@ import type {
|
||||
HandoffReason,
|
||||
} from "@agent/types";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import type {
|
||||
ProcessTraceSnapshot,
|
||||
ProcessTraceStepName,
|
||||
} from "@skills/onboarding/runtime/trace";
|
||||
import type { PrmFeatureVector } from "./judges/prm.js";
|
||||
import type {
|
||||
MemoryJudgeExpected,
|
||||
MemoryJudgeResult,
|
||||
} from "./judges/memory-judge.js";
|
||||
|
||||
export type EvalProcessTraceSnapshot = ProcessTraceSnapshot;
|
||||
|
||||
@@ -28,6 +33,7 @@ export interface EvalCaseInput {
|
||||
session?: OnboardingSession | null | undefined;
|
||||
blackboard?: Blackboard | null | undefined;
|
||||
summary?: string | undefined;
|
||||
userProfile?: UserProfile | null | undefined;
|
||||
}
|
||||
|
||||
export interface EvalCaseExpected {
|
||||
@@ -37,6 +43,7 @@ export interface EvalCaseExpected {
|
||||
expectedTarget?: string | undefined;
|
||||
shouldHandoff?: boolean | undefined;
|
||||
} | undefined;
|
||||
expectedMemory?: MemoryJudgeExpected | undefined;
|
||||
expectedSourceChain?: {
|
||||
chain: string[];
|
||||
mode: "contains" | "exact" | "prefix";
|
||||
@@ -183,6 +190,7 @@ export interface EvalCaseResult {
|
||||
ruleJudge: RuleJudgeResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
handoffJudge?: HandoffJudgeResult | undefined;
|
||||
memoryJudge?: MemoryJudgeResult | undefined;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
processTrace?: ProcessTraceSnapshot | undefined;
|
||||
error?: string | undefined;
|
||||
|
||||
@@ -31,6 +31,9 @@ const { LlmClient } = await import("./llm/index.js");
|
||||
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
|
||||
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
|
||||
const { createMemoryService } = await import("./memory/memory-service.js");
|
||||
const { createEscalationHistoryService } = await import(
|
||||
"./memory/mongo-escalation-history.js"
|
||||
);
|
||||
const { RoomContextService } = await import("./room-context/index.js");
|
||||
const { startMetricsServer, stopMetricsServer } = await import(
|
||||
"./observability/metrics-server.js"
|
||||
@@ -226,8 +229,11 @@ async function bootstrap(): Promise<void> {
|
||||
|
||||
const memoryService = createMemoryService({
|
||||
garnetClient,
|
||||
llmClient,
|
||||
mongooseConnection: mongoConnection,
|
||||
});
|
||||
const escalationHistoryService =
|
||||
createEscalationHistoryService(mongoConnection);
|
||||
|
||||
// Agent setup
|
||||
const conversationStore = new ConversationStore(garnetClient);
|
||||
@@ -265,6 +271,7 @@ async function bootstrap(): Promise<void> {
|
||||
agent.registerSkill(
|
||||
factory({
|
||||
garnetClient,
|
||||
escalationHistoryService,
|
||||
llmClient,
|
||||
memoryService,
|
||||
natsClient,
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildBM25Index } from "./bm25.js";
|
||||
|
||||
describe("buildBM25Index", () => {
|
||||
it("ranks Korean keyword matches above unrelated documents", () => {
|
||||
const index = buildBM25Index([
|
||||
{ id: "ads", text: "광고와 도배 홍보 링크는 제재 대상입니다." },
|
||||
{ id: "age", text: "가입 가능 연령은 27세 이상입니다." },
|
||||
]);
|
||||
|
||||
const [top] = index.rank("광고 도배 기준", 2);
|
||||
|
||||
expect(top?.id).toBe("ads");
|
||||
expect(top?.score).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
export interface BM25Document {
|
||||
id: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface BM25Index {
|
||||
rank(query: string, topK: number): Array<{ id: string; score: number }>;
|
||||
}
|
||||
|
||||
const K1 = 1.5;
|
||||
const B = 0.75;
|
||||
|
||||
function normalize(text: string): string {
|
||||
return text
|
||||
.normalize("NFKC")
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{Letter}\p{Number}\s]/gu, " ");
|
||||
}
|
||||
|
||||
function tokenize(text: string): string[] {
|
||||
const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0);
|
||||
const compact = normalize(text).replace(/\s+/gu, "");
|
||||
const grams: string[] = [];
|
||||
for (let i = 0; i < compact.length - 1; i += 1) {
|
||||
grams.push(compact.slice(i, i + 2));
|
||||
}
|
||||
return [...new Set([...words, ...grams])];
|
||||
}
|
||||
|
||||
export function buildBM25Index(documents: BM25Document[]): BM25Index {
|
||||
const termFrequencies = new Map<string, Map<string, number>>();
|
||||
const documentLengths = new Map<string, number>();
|
||||
const documentFrequency = new Map<string, number>();
|
||||
let totalLength = 0;
|
||||
|
||||
for (const document of documents) {
|
||||
const tokens = tokenize(document.text);
|
||||
documentLengths.set(document.id, tokens.length);
|
||||
totalLength += tokens.length;
|
||||
|
||||
const frequencies = new Map<string, number>();
|
||||
for (const token of tokens) {
|
||||
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
|
||||
}
|
||||
termFrequencies.set(document.id, frequencies);
|
||||
|
||||
for (const token of frequencies.keys()) {
|
||||
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const averageLength = documents.length === 0 ? 0 : totalLength / documents.length;
|
||||
|
||||
return {
|
||||
rank(query: string, topK: number): Array<{ id: string; score: number }> {
|
||||
const queryTokens = tokenize(query);
|
||||
if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return documents
|
||||
.map((document) => {
|
||||
const frequencies = termFrequencies.get(document.id) ?? new Map<string, number>();
|
||||
const documentLength = documentLengths.get(document.id) ?? 0;
|
||||
const score = queryTokens.reduce((sum, token) => {
|
||||
const tf = frequencies.get(token) ?? 0;
|
||||
if (tf === 0) {
|
||||
return sum;
|
||||
}
|
||||
const df = documentFrequency.get(token) ?? 0;
|
||||
const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5));
|
||||
const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength));
|
||||
return sum + idf * ((tf * (K1 + 1)) / denominator);
|
||||
}, 0);
|
||||
return { id: document.id, score };
|
||||
})
|
||||
.filter((item) => item.score > 0)
|
||||
.sort((left, right) => right.score - left.score)
|
||||
.slice(0, Math.max(1, topK));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { detectContradictions, textSimilarity } from "./contradiction-detector.js";
|
||||
|
||||
describe("textSimilarity", () => {
|
||||
it("returns lower scores for very different text", () => {
|
||||
expect(textSimilarity("닉네임 규칙", "광고 제재 기준")).toBeLessThan(0.7);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectContradictions", () => {
|
||||
it("flags corrections in the same source section", () => {
|
||||
const [candidate] = detectContradictions([
|
||||
{
|
||||
id: "old",
|
||||
metadata: { sourceFile: "rules.md", sectionTitle: "nickname" },
|
||||
text: "닉네임은 기존 형식을 사용합니다.",
|
||||
},
|
||||
{
|
||||
id: "new",
|
||||
metadata: { sourceFile: "rules.md", sectionTitle: "nickname" },
|
||||
text: "정정: 닉네임은 새 형식을 사용합니다.",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(candidate).toMatchObject({
|
||||
existingId: "old",
|
||||
newId: "new",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
export interface ContradictionCandidate {
|
||||
existingId: string;
|
||||
newId: string;
|
||||
reason: "keyword" | "text-distance";
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
export interface ContradictionDocument {
|
||||
id: string;
|
||||
metadata?: Record<string, unknown> | undefined;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function groupKey(doc: ContradictionDocument): string | null {
|
||||
const sourceFile = doc.metadata?.["sourceFile"];
|
||||
const sectionTitle = doc.metadata?.["sectionTitle"];
|
||||
if (typeof sourceFile !== "string" || typeof sectionTitle !== "string") {
|
||||
return null;
|
||||
}
|
||||
return `${sourceFile}#${sectionTitle}`;
|
||||
}
|
||||
|
||||
function bigrams(text: string): Set<string> {
|
||||
const compact = text
|
||||
.normalize("NFKC")
|
||||
.toLowerCase()
|
||||
.replace(/\s+/gu, "");
|
||||
const grams = new Set<string>();
|
||||
for (let i = 0; i < compact.length - 1; i += 1) {
|
||||
grams.add(compact.slice(i, i + 2));
|
||||
}
|
||||
return grams;
|
||||
}
|
||||
|
||||
export function textSimilarity(left: string, right: string): number {
|
||||
const leftGrams = bigrams(left);
|
||||
const rightGrams = bigrams(right);
|
||||
if (leftGrams.size === 0 && rightGrams.size === 0) {
|
||||
return 1;
|
||||
}
|
||||
const intersection = [...leftGrams].filter((gram) => rightGrams.has(gram)).length;
|
||||
const union = new Set([...leftGrams, ...rightGrams]).size;
|
||||
return union === 0 ? 0 : intersection / union;
|
||||
}
|
||||
|
||||
function hasCorrectionKeyword(text: string): boolean {
|
||||
return /(변경|정정|수정|폐지|삭제|바뀜|업데이트)/u.test(text);
|
||||
}
|
||||
|
||||
export function detectContradictions(
|
||||
documents: ContradictionDocument[],
|
||||
): ContradictionCandidate[] {
|
||||
const groups = new Map<string, ContradictionDocument[]>();
|
||||
for (const doc of documents) {
|
||||
const key = groupKey(doc);
|
||||
if (!key) {
|
||||
continue;
|
||||
}
|
||||
groups.set(key, [...(groups.get(key) ?? []), doc]);
|
||||
}
|
||||
|
||||
const candidates: ContradictionCandidate[] = [];
|
||||
for (const docs of groups.values()) {
|
||||
for (let index = 1; index < docs.length; index += 1) {
|
||||
const previous = docs[index - 1];
|
||||
const current = docs[index];
|
||||
if (!previous || !current) {
|
||||
continue;
|
||||
}
|
||||
const similarity = textSimilarity(previous.text, current.text);
|
||||
if (similarity < 0.7 || hasCorrectionKeyword(current.text)) {
|
||||
candidates.push({
|
||||
existingId: previous.id,
|
||||
newId: current.id,
|
||||
reason: hasCorrectionKeyword(current.text) ? "keyword" : "text-distance",
|
||||
similarity,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue";
|
||||
|
||||
export interface UserEscalationRecord {
|
||||
queuedAt: string;
|
||||
reason: HandoffReason;
|
||||
reviewId: string;
|
||||
target: string;
|
||||
resolution?: {
|
||||
resolvedAt: string;
|
||||
action: "dismiss" | "manual-reply" | "reroute";
|
||||
note?: string | undefined;
|
||||
} | undefined;
|
||||
packetSummary: string;
|
||||
sourceChain: string[];
|
||||
}
|
||||
|
||||
export interface UserEscalationHistory {
|
||||
userId: string;
|
||||
chatroomId?: string | undefined;
|
||||
escalations: UserEscalationRecord[];
|
||||
totalEscalations: number;
|
||||
lastEscalationAt: string;
|
||||
topReasons: Array<{ reason: HandoffReason; count: number }>;
|
||||
}
|
||||
|
||||
export interface EscalationHistoryService {
|
||||
load(userId: string): Promise<UserEscalationHistory | null>;
|
||||
recordEscalation(record: {
|
||||
userId: string;
|
||||
chatroomId?: string | undefined;
|
||||
queuedAt: string;
|
||||
reason: HandoffReason;
|
||||
reviewId: string;
|
||||
target: string;
|
||||
packetSummary: string;
|
||||
sourceChain: string[];
|
||||
}): Promise<void>;
|
||||
recordResolution(record: {
|
||||
userId: string;
|
||||
reviewId: string;
|
||||
resolution: SupervisorReviewResolution;
|
||||
}): Promise<void>;
|
||||
}
|
||||
|
||||
export class NoopEscalationHistoryService implements EscalationHistoryService {
|
||||
load(): Promise<UserEscalationHistory | null> {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
recordEscalation(): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
recordResolution(): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { evaluateDocument, evaluateUserFlags } from "./forgetting-policy.js";
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
|
||||
describe("evaluateDocument", () => {
|
||||
it("archives stale low-reference documents", () => {
|
||||
const decision = evaluateDocument(
|
||||
{ category: "rule", id: "doc-1", text: "old rule" },
|
||||
{
|
||||
now: new Date("2026-05-18T00:00:00.000Z"),
|
||||
referenceCount30d: 0,
|
||||
updatedAt: new Date("2025-12-01T00:00:00.000Z"),
|
||||
},
|
||||
);
|
||||
|
||||
expect(decision.action).toBe("archive");
|
||||
expect(decision.reason).toBe("freshness-stale");
|
||||
});
|
||||
});
|
||||
|
||||
describe("evaluateUserFlags", () => {
|
||||
it("keeps the newest unique flags within the limit", () => {
|
||||
const profile = {
|
||||
flaggedIssues: ["a", "b", "c"],
|
||||
} as UserProfile;
|
||||
|
||||
expect(evaluateUserFlags(profile, 2)).toEqual({
|
||||
archived: ["a"],
|
||||
kept: ["b", "c"],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
import type { KnowledgeDocument } from "./memory-types.js";
|
||||
|
||||
export interface ForgettingDecision {
|
||||
documentId: string;
|
||||
action: "archive" | "delete" | "keep";
|
||||
reason: "contradiction" | "freshness-stale" | "relevance-low" | "ttl-expired";
|
||||
evidence?: string | undefined;
|
||||
}
|
||||
|
||||
export interface KnowledgeForgettingSignals {
|
||||
contradictsId?: string | undefined;
|
||||
lastReferencedAt?: Date | undefined;
|
||||
now?: Date | undefined;
|
||||
referenceCount30d: number;
|
||||
updatedAt?: Date | undefined;
|
||||
}
|
||||
|
||||
function daysSince(date: Date | undefined, now: Date): number {
|
||||
if (!date) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
return Math.max(0, now.getTime() - date.getTime()) / (24 * 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
export function evaluateDocument(
|
||||
doc: KnowledgeDocument,
|
||||
signals: KnowledgeForgettingSignals,
|
||||
): ForgettingDecision {
|
||||
const now = signals.now ?? new Date();
|
||||
const documentId = doc.id ?? "unknown";
|
||||
|
||||
if (signals.contradictsId) {
|
||||
return {
|
||||
action: "archive",
|
||||
documentId,
|
||||
evidence: `contradicts ${signals.contradictsId}`,
|
||||
reason: "contradiction",
|
||||
};
|
||||
}
|
||||
|
||||
const staleDays = daysSince(signals.updatedAt, now);
|
||||
const referenceDays = daysSince(signals.lastReferencedAt, now);
|
||||
if (staleDays >= 90 && signals.referenceCount30d < 3) {
|
||||
return {
|
||||
action: "archive",
|
||||
documentId,
|
||||
evidence: `updated ${Math.round(staleDays)}d ago, refs30d=${signals.referenceCount30d}`,
|
||||
reason: "freshness-stale",
|
||||
};
|
||||
}
|
||||
|
||||
if (referenceDays >= 30 && signals.referenceCount30d < 3) {
|
||||
return {
|
||||
action: "archive",
|
||||
documentId,
|
||||
evidence: `last referenced ${Math.round(referenceDays)}d ago, refs30d=${signals.referenceCount30d}`,
|
||||
reason: "relevance-low",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
action: "keep",
|
||||
documentId,
|
||||
reason: "relevance-low",
|
||||
};
|
||||
}
|
||||
|
||||
export function evaluateUserFlags(
|
||||
profile: UserProfile,
|
||||
maxFlags: number,
|
||||
): { archived: string[]; kept: string[] } {
|
||||
const unique = [...new Set(profile.flaggedIssues.map((issue) => issue.trim()).filter(Boolean))];
|
||||
if (unique.length <= maxFlags) {
|
||||
return { archived: [], kept: unique };
|
||||
}
|
||||
return {
|
||||
archived: unique.slice(0, Math.max(0, unique.length - maxFlags)),
|
||||
kept: unique.slice(-maxFlags),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLongTermSummaryEnabled } from "../config.js";
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
|
||||
export interface LongTermSummary {
|
||||
text: string;
|
||||
updatedAt: string;
|
||||
turnSnapshot: number;
|
||||
}
|
||||
|
||||
interface SummaryPayload {
|
||||
text?: unknown;
|
||||
}
|
||||
|
||||
function daysBetween(leftIso: string, rightIso: string): number {
|
||||
const left = Date.parse(leftIso);
|
||||
const right = Date.parse(rightIso);
|
||||
if (!Number.isFinite(left) || !Number.isFinite(right)) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
return Math.abs(right - left) / (24 * 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
export function shouldRefreshLongTermSummary(
|
||||
profile: UserProfile,
|
||||
now = new Date().toISOString(),
|
||||
): boolean {
|
||||
if (!getLongTermSummaryEnabled()) {
|
||||
return false;
|
||||
}
|
||||
if (profile.totalTurns < 30) {
|
||||
return false;
|
||||
}
|
||||
const summary = profile.longTermSummary;
|
||||
if (!summary) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
profile.totalTurns - summary.turnSnapshot >= 30 ||
|
||||
daysBetween(summary.updatedAt, now) >= 7
|
||||
);
|
||||
}
|
||||
|
||||
export async function buildLongTermSummary(
|
||||
llmClient: Pick<LlmClient, "chatCompletion">,
|
||||
profile: UserProfile,
|
||||
conversationSummary: string | null,
|
||||
escalationPattern?: string,
|
||||
): Promise<LongTermSummary> {
|
||||
const now = new Date().toISOString();
|
||||
const result = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"너는 사용자별 장기 메모를 작성한다.",
|
||||
"운영자가 다음 응답에서 참고할 수 있게 3-5문장으로 요약한다.",
|
||||
"개인정보나 원문 식별자는 넣지 않는다.",
|
||||
'JSON으로만 응답한다: {"text":"요약"}',
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`[totalTurns] ${profile.totalTurns}`,
|
||||
`[preferredStyle] ${profile.preferredStyle}`,
|
||||
`[pastIntents] ${profile.pastIntents.map((item) => `${item.kind}:${item.count}`).join(", ") || "-"}`,
|
||||
`[flaggedIssues] ${profile.flaggedIssues.slice(-10).join(", ") || "-"}`,
|
||||
`[conversationSummary] ${conversationSummary ?? "-"}`,
|
||||
`[escalationPattern] ${escalationPattern ?? "-"}`,
|
||||
].join("\n"),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: 300,
|
||||
temperature: 0.3,
|
||||
tier: 0,
|
||||
},
|
||||
);
|
||||
|
||||
if (!result?.content.trim()) {
|
||||
throw new Error("Long-term summary LLM returned no content");
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(result.content) as SummaryPayload;
|
||||
if (typeof parsed.text !== "string" || parsed.text.trim().length === 0) {
|
||||
throw new Error("Long-term summary payload missing text");
|
||||
}
|
||||
|
||||
return {
|
||||
text: parsed.text.trim(),
|
||||
updatedAt: now,
|
||||
turnSnapshot: profile.totalTurns,
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import type mongoose from "mongoose";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { Blackboard } from "@skills/onboarding/types";
|
||||
import { createEmbeddingClient } from "./embedding-client.js";
|
||||
import { GarnetSessionMemory } from "./garnet-session-memory.js";
|
||||
@@ -7,6 +8,7 @@ import {
|
||||
MongoLongTermMemory,
|
||||
NoopLongTermMemory,
|
||||
} from "./mongo-long-term-memory.js";
|
||||
import { LlmReranker } from "./reranker.js";
|
||||
import type {
|
||||
LongTermMemory,
|
||||
MemoryService,
|
||||
@@ -17,6 +19,7 @@ import type {
|
||||
|
||||
interface CreateMemoryServiceOptions {
|
||||
garnetClient: GarnetClient;
|
||||
llmClient?: Pick<LlmClient, "chatCompletion"> | undefined;
|
||||
mongooseConnection?: mongoose.Connection | undefined;
|
||||
}
|
||||
|
||||
@@ -88,6 +91,7 @@ export function createMemoryService(
|
||||
options.garnetClient,
|
||||
options.mongooseConnection,
|
||||
createEmbeddingClient(),
|
||||
options.llmClient ? new LlmReranker(options.llmClient) : undefined,
|
||||
)
|
||||
: new NoopLongTermMemory(options.garnetClient);
|
||||
|
||||
|
||||
@@ -46,6 +46,10 @@ export interface SessionMemory {
|
||||
export interface SearchOptions {
|
||||
skill?: string | undefined;
|
||||
category?: string | undefined;
|
||||
candidatePoolSize?: number | undefined;
|
||||
hybrid?: boolean | undefined;
|
||||
rerank?: boolean | undefined;
|
||||
rerankTopK?: number | undefined;
|
||||
topK?: number | undefined;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
import type mongoose from "mongoose";
|
||||
import { Schema } from "mongoose";
|
||||
import { anonymizeIdentifier } from "@eval/anonymize";
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { SupervisorReviewResolution } from "@skills/supervisor/review-queue";
|
||||
import {
|
||||
NoopEscalationHistoryService,
|
||||
type EscalationHistoryService,
|
||||
type UserEscalationHistory,
|
||||
type UserEscalationRecord,
|
||||
} from "./escalation-history.js";
|
||||
|
||||
export const ESCALATION_HISTORY_COLLECTION = "user_escalation_history";
|
||||
|
||||
interface EscalationHistoryRecord {
|
||||
_id: string;
|
||||
userId: string;
|
||||
chatroomId?: string | undefined;
|
||||
escalations: UserEscalationRecord[];
|
||||
totalEscalations: number;
|
||||
lastEscalationAt: string;
|
||||
topReasons: Array<{ reason: HandoffReason; count: number }>;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
const EscalationHistorySchema = new Schema<EscalationHistoryRecord>(
|
||||
{
|
||||
_id: { type: String, required: true },
|
||||
userId: { type: String, required: true, index: true },
|
||||
chatroomId: { type: String, index: true },
|
||||
escalations: [
|
||||
{
|
||||
queuedAt: String,
|
||||
reason: String,
|
||||
reviewId: String,
|
||||
target: String,
|
||||
resolution: {
|
||||
resolvedAt: String,
|
||||
action: String,
|
||||
note: String,
|
||||
},
|
||||
packetSummary: String,
|
||||
sourceChain: [String],
|
||||
},
|
||||
],
|
||||
totalEscalations: { type: Number, default: 0 },
|
||||
lastEscalationAt: String,
|
||||
topReasons: [{ reason: String, count: Number }],
|
||||
updatedAt: { type: Date, default: Date.now, index: true },
|
||||
},
|
||||
{ versionKey: false },
|
||||
);
|
||||
|
||||
function userKey(userId: string): string {
|
||||
return anonymizeIdentifier("user", userId) ?? "user_unknown";
|
||||
}
|
||||
|
||||
function computeTopReasons(records: UserEscalationRecord[]): Array<{ reason: HandoffReason; count: number }> {
|
||||
const counts = new Map<HandoffReason, number>();
|
||||
for (const record of records) {
|
||||
counts.set(record.reason, (counts.get(record.reason) ?? 0) + 1);
|
||||
}
|
||||
return [...counts.entries()]
|
||||
.map(([reason, count]) => ({ reason, count }))
|
||||
.sort((left, right) => right.count - left.count || left.reason.localeCompare(right.reason))
|
||||
.slice(0, 5);
|
||||
}
|
||||
|
||||
export class MongoEscalationHistoryService implements EscalationHistoryService {
|
||||
private readonly model: mongoose.Model<EscalationHistoryRecord>;
|
||||
|
||||
constructor(connection: mongoose.Connection) {
|
||||
this.model =
|
||||
(connection.models["EscalationHistory"] as mongoose.Model<EscalationHistoryRecord> | undefined) ??
|
||||
connection.model<EscalationHistoryRecord>(
|
||||
"EscalationHistory",
|
||||
EscalationHistorySchema,
|
||||
ESCALATION_HISTORY_COLLECTION,
|
||||
);
|
||||
}
|
||||
|
||||
async load(userId: string): Promise<UserEscalationHistory | null> {
|
||||
const doc = await this.model.findById(userKey(userId)).lean().exec();
|
||||
if (!doc) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
userId: doc.userId,
|
||||
...(doc.chatroomId ? { chatroomId: doc.chatroomId } : {}),
|
||||
escalations: doc.escalations.slice(-10),
|
||||
lastEscalationAt: doc.lastEscalationAt,
|
||||
topReasons: doc.topReasons,
|
||||
totalEscalations: doc.totalEscalations,
|
||||
};
|
||||
}
|
||||
|
||||
async recordEscalation(record: {
|
||||
userId: string;
|
||||
chatroomId?: string | undefined;
|
||||
queuedAt: string;
|
||||
reason: HandoffReason;
|
||||
reviewId: string;
|
||||
target: string;
|
||||
packetSummary: string;
|
||||
sourceChain: string[];
|
||||
}): Promise<void> {
|
||||
const id = userKey(record.userId);
|
||||
const item: UserEscalationRecord = {
|
||||
packetSummary: record.packetSummary,
|
||||
queuedAt: record.queuedAt,
|
||||
reason: record.reason,
|
||||
reviewId: record.reviewId,
|
||||
sourceChain: record.sourceChain,
|
||||
target: record.target,
|
||||
};
|
||||
await this.model.updateOne(
|
||||
{ _id: id },
|
||||
{
|
||||
$inc: { totalEscalations: 1 },
|
||||
$push: { escalations: { $each: [item], $slice: -50 } },
|
||||
$set: {
|
||||
...(record.chatroomId ? { chatroomId: record.chatroomId } : {}),
|
||||
lastEscalationAt: record.queuedAt,
|
||||
updatedAt: new Date(),
|
||||
userId: id,
|
||||
},
|
||||
$setOnInsert: { _id: id },
|
||||
},
|
||||
{ upsert: true },
|
||||
).exec();
|
||||
await this.recomputeTopReasons(id);
|
||||
}
|
||||
|
||||
async recordResolution(record: {
|
||||
userId: string;
|
||||
reviewId: string;
|
||||
resolution: SupervisorReviewResolution;
|
||||
}): Promise<void> {
|
||||
const id = userKey(record.userId);
|
||||
await this.model.updateOne(
|
||||
{ _id: id, "escalations.reviewId": record.reviewId },
|
||||
{
|
||||
$set: {
|
||||
"escalations.$.resolution": {
|
||||
action: record.resolution.action,
|
||||
resolvedAt: record.resolution.resolvedAt,
|
||||
...(record.resolution.note ? { note: record.resolution.note } : {}),
|
||||
},
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
}
|
||||
|
||||
private async recomputeTopReasons(id: string): Promise<void> {
|
||||
const doc = await this.model.findById(id).lean().exec();
|
||||
if (!doc) {
|
||||
return;
|
||||
}
|
||||
await this.model.updateOne(
|
||||
{ _id: id },
|
||||
{ $set: { topReasons: computeTopReasons(doc.escalations) } },
|
||||
).exec();
|
||||
}
|
||||
}
|
||||
|
||||
export function createEscalationHistoryService(
|
||||
connection?: mongoose.Connection,
|
||||
): EscalationHistoryService {
|
||||
return connection
|
||||
? new MongoEscalationHistoryService(connection)
|
||||
: new NoopEscalationHistoryService();
|
||||
}
|
||||
@@ -3,11 +3,19 @@ import type mongoose from "mongoose";
|
||||
import { Schema } from "mongoose";
|
||||
import { anonymizeIdentifier } from "@eval/anonymize";
|
||||
import { withSpan } from "@obs/spans";
|
||||
import { getAgentUserMemoryTtlDays, getKnowledgeSearchCandidateLimit } from "../config.js";
|
||||
import {
|
||||
getAgentUserMemoryTtlDays,
|
||||
getKnowledgeSearchCandidateLimit,
|
||||
getRagCandidatePoolSize,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
getRagRerankerTopK,
|
||||
} from "../config.js";
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
import { USER_MEMORY_KEY_PREFIX } from "@skills/onboarding/state/user-memory";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { rankByCosineSimilarity } from "./cosine.js";
|
||||
import { buildBM25Index } from "./bm25.js";
|
||||
import type {
|
||||
KnowledgeDocument,
|
||||
LongTermMemory,
|
||||
@@ -15,6 +23,7 @@ import type {
|
||||
SearchOptions,
|
||||
} from "./memory-types.js";
|
||||
import type { EmbeddingClient } from "./embedding-client.js";
|
||||
import type { Reranker } from "./reranker.js";
|
||||
|
||||
export const KNOWLEDGE_COLLECTION = "knowledge_documents";
|
||||
|
||||
@@ -26,6 +35,8 @@ interface KnowledgeDocumentRecord {
|
||||
embedding: number[];
|
||||
embeddingModel: string;
|
||||
metadata?: Record<string, unknown> | undefined;
|
||||
lastReferencedAt?: Date | undefined;
|
||||
references?: Date[] | undefined;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -39,6 +50,8 @@ const KnowledgeSchema = new Schema<KnowledgeDocumentRecord>(
|
||||
embedding: { type: [Number], required: true },
|
||||
embeddingModel: { type: String, required: true },
|
||||
metadata: { type: Schema.Types.Mixed },
|
||||
lastReferencedAt: { type: Date, index: true },
|
||||
references: { type: [Date], default: [] },
|
||||
createdAt: { type: Date, default: Date.now },
|
||||
updatedAt: { type: Date, default: Date.now, index: true },
|
||||
},
|
||||
@@ -83,7 +96,9 @@ function stableDocumentId(doc: KnowledgeDocument): string {
|
||||
}
|
||||
|
||||
function buildFilter(options: SearchOptions): Record<string, unknown> {
|
||||
const filter: Record<string, unknown> = {};
|
||||
const filter: Record<string, unknown> = {
|
||||
"metadata.archived": { $ne: true },
|
||||
};
|
||||
if (options.skill?.trim()) {
|
||||
filter["skill"] = options.skill.trim();
|
||||
}
|
||||
@@ -93,6 +108,34 @@ function buildFilter(options: SearchOptions): Record<string, unknown> {
|
||||
return filter;
|
||||
}
|
||||
|
||||
function normalizeScores(scores: Map<string, number>): Map<string, number> {
|
||||
const values = [...scores.values()].filter((value) => Number.isFinite(value));
|
||||
if (values.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
const min = Math.min(...values);
|
||||
const max = Math.max(...values);
|
||||
if (max === min) {
|
||||
return new Map([...scores.keys()].map((id) => [id, max > 0 ? 1 : 0]));
|
||||
}
|
||||
return new Map(
|
||||
[...scores.entries()].map(([id, score]) => [id, (score - min) / (max - min)]),
|
||||
);
|
||||
}
|
||||
|
||||
function toRetrievedDocument(
|
||||
doc: KnowledgeDocumentRecord & { score: number },
|
||||
): RetrievedDocument {
|
||||
return {
|
||||
id: doc._id,
|
||||
category: doc.category,
|
||||
metadata: doc.metadata,
|
||||
score: doc.score,
|
||||
skill: doc.skill,
|
||||
text: doc.text,
|
||||
};
|
||||
}
|
||||
|
||||
export class MongoLongTermMemory implements LongTermMemory {
|
||||
private readonly model: mongoose.Model<KnowledgeDocumentRecord>;
|
||||
|
||||
@@ -100,6 +143,7 @@ export class MongoLongTermMemory implements LongTermMemory {
|
||||
private readonly garnetClient: Pick<GarnetClient, "getJson" | "setJson">,
|
||||
connection: mongoose.Connection,
|
||||
private readonly embeddingClient: EmbeddingClient,
|
||||
private readonly reranker?: Reranker | undefined,
|
||||
) {
|
||||
this.model =
|
||||
(connection.models["KnowledgeDocument"] as mongoose.Model<KnowledgeDocumentRecord> | undefined) ??
|
||||
@@ -162,33 +206,93 @@ export class MongoLongTermMemory implements LongTermMemory {
|
||||
.limit(getKnowledgeSearchCandidateLimit())
|
||||
.lean()
|
||||
.exec()) as KnowledgeDocumentRecord[];
|
||||
const ranked = rankByCosineSimilarity(
|
||||
const candidatePoolSize = options.candidatePoolSize ?? getRagCandidatePoolSize();
|
||||
const topK = options.topK ?? 5;
|
||||
const shouldHybrid = options.hybrid ?? getRagHybridEnabled();
|
||||
const shouldRerank = Boolean(
|
||||
this.reranker && (options.rerank ?? getRagRerankerEnabled()),
|
||||
);
|
||||
const cosineRanked = rankByCosineSimilarity(
|
||||
queryEmbedding,
|
||||
docs.filter((doc) =>
|
||||
Array.isArray(doc.embedding) &&
|
||||
doc.embedding.length === queryEmbedding.length
|
||||
),
|
||||
options.topK ?? 5,
|
||||
Math.max(candidatePoolSize, topK),
|
||||
);
|
||||
const ranked = shouldHybrid
|
||||
? this.rankHybrid(trimmed, docs, cosineRanked, Math.max(candidatePoolSize, topK))
|
||||
: cosineRanked.slice(0, Math.max(candidatePoolSize, topK));
|
||||
const pool = ranked.slice(0, shouldRerank ? candidatePoolSize : topK);
|
||||
const results =
|
||||
shouldRerank && this.reranker
|
||||
? await this.reranker.rerank(
|
||||
trimmed,
|
||||
pool.map(toRetrievedDocument),
|
||||
options.rerankTopK ?? getRagRerankerTopK(),
|
||||
)
|
||||
: pool.slice(0, topK).map(toRetrievedDocument);
|
||||
|
||||
await this.markReferenced(results.map((doc) => doc.id));
|
||||
|
||||
span.setAttributes({
|
||||
"memory.retrieval.candidate_count": docs.length,
|
||||
"memory.retrieval.duration_ms": Math.max(0, Date.now() - startedAt),
|
||||
"memory.retrieval.top_score": ranked[0]?.score,
|
||||
"memory.retrieval.hybrid": shouldHybrid,
|
||||
"memory.retrieval.rerank": shouldRerank,
|
||||
"memory.retrieval.top_score": results[0]?.score,
|
||||
});
|
||||
|
||||
return ranked.map((doc) => ({
|
||||
id: doc._id,
|
||||
category: doc.category,
|
||||
metadata: doc.metadata,
|
||||
score: doc.score,
|
||||
skill: doc.skill,
|
||||
text: doc.text,
|
||||
}));
|
||||
return results.slice(0, topK);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private rankHybrid(
|
||||
query: string,
|
||||
docs: KnowledgeDocumentRecord[],
|
||||
cosineRanked: Array<KnowledgeDocumentRecord & { score: number }>,
|
||||
limit: number,
|
||||
): Array<KnowledgeDocumentRecord & { score: number }> {
|
||||
const cosineScores = normalizeScores(
|
||||
new Map(cosineRanked.map((doc) => [doc._id, doc.score])),
|
||||
);
|
||||
const bm25Scores = normalizeScores(
|
||||
new Map(
|
||||
buildBM25Index(docs.map((doc) => ({ id: doc._id, text: doc.text })))
|
||||
.rank(query, docs.length)
|
||||
.map((item) => [item.id, item.score]),
|
||||
),
|
||||
);
|
||||
const byId = new Map(docs.map((doc) => [doc._id, doc]));
|
||||
|
||||
return [...byId.keys()]
|
||||
.map((id) => ({
|
||||
doc: byId.get(id),
|
||||
score: ((cosineScores.get(id) ?? 0) + (bm25Scores.get(id) ?? 0)) / 2,
|
||||
}))
|
||||
.filter((item): item is { doc: KnowledgeDocumentRecord; score: number } =>
|
||||
Boolean(item.doc) && item.score > 0
|
||||
)
|
||||
.map((item) => ({ ...item.doc, score: item.score }))
|
||||
.sort((left, right) => right.score - left.score)
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
||||
private async markReferenced(ids: string[]): Promise<void> {
|
||||
if (ids.length === 0) {
|
||||
return;
|
||||
}
|
||||
const now = new Date();
|
||||
await this.model.updateMany(
|
||||
{ _id: { $in: ids } },
|
||||
{
|
||||
$set: { lastReferencedAt: now },
|
||||
$push: { references: { $each: [now], $slice: -100 } },
|
||||
},
|
||||
).exec();
|
||||
}
|
||||
|
||||
async upsertDocument(doc: KnowledgeDocument): Promise<string> {
|
||||
const id = doc.id?.trim() || stableDocumentId(doc);
|
||||
const embedding = doc.embedding ?? await this.embeddingClient.embed(doc.text);
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { deriveProfilePromptSnippets } from "./profile-update-policy.js";
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
const profile: UserProfile = {
|
||||
firstSeenAt: "2026-05-01T00:00:00.000Z",
|
||||
flaggedIssues: ["닉네임 형식 오류", "닉네임 형식 오류", "닉네임 형식 오류"],
|
||||
lastSeenAt: "2026-05-18T00:00:00.000Z",
|
||||
pastIntents: [{ kind: "nickname-question", count: 5 }],
|
||||
preferredStyle: "concise",
|
||||
totalTurns: 10,
|
||||
userId: "user_x",
|
||||
};
|
||||
|
||||
const derived = {
|
||||
intent: { kind: "nickname-question" },
|
||||
} as DerivedState;
|
||||
|
||||
describe("deriveProfilePromptSnippets", () => {
|
||||
it("derives frequent intent, repeated issue, and preferred style snippets", () => {
|
||||
const snippets = deriveProfilePromptSnippets(profile, derived);
|
||||
|
||||
expect(snippets.map((snippet) => snippet.reason)).toEqual([
|
||||
"frequent-intent",
|
||||
"repeated-issue",
|
||||
"preferred-style",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { UserProfile } from "@skills/onboarding/state/user-memory";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
export interface ProfilePromptSnippet {
|
||||
block: string;
|
||||
reason: "frequent-intent" | "preferred-style" | "repeated-issue";
|
||||
}
|
||||
|
||||
function normalizeIssue(issue: string): string {
|
||||
return issue
|
||||
.normalize("NFKC")
|
||||
.replace(/\[[^\]]+\]/gu, "")
|
||||
.replace(/[0-9]/gu, "")
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim()
|
||||
.slice(0, 32);
|
||||
}
|
||||
|
||||
export function deriveProfilePromptSnippets(
|
||||
profile: UserProfile,
|
||||
derived: DerivedState,
|
||||
): ProfilePromptSnippet[] {
|
||||
const snippets: ProfilePromptSnippet[] = [];
|
||||
const frequentIntent = profile.pastIntents.find(
|
||||
(intent) => intent.kind === derived.intent.kind && intent.count >= 5,
|
||||
);
|
||||
if (frequentIntent) {
|
||||
snippets.push({
|
||||
reason: "frequent-intent",
|
||||
block: `사용자는 ${frequentIntent.kind} 유형을 자주 물어봤어요. 핵심 답부터 짧게 말하고, 질문 범위를 넘기지 마세요.`,
|
||||
});
|
||||
}
|
||||
|
||||
const issueCounts = new Map<string, number>();
|
||||
for (const issue of profile.flaggedIssues) {
|
||||
const normalized = normalizeIssue(issue);
|
||||
if (normalized) {
|
||||
issueCounts.set(normalized, (issueCounts.get(normalized) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
const repeatedIssue = [...issueCounts.entries()].find(([, count]) => count >= 3);
|
||||
if (repeatedIssue) {
|
||||
snippets.push({
|
||||
reason: "repeated-issue",
|
||||
block: `반복된 이슈가 있었어요: ${repeatedIssue[0]}. 같은 실수가 이어지지 않도록 정확한 형식 예시를 먼저 확인하세요.`,
|
||||
});
|
||||
}
|
||||
|
||||
if (profile.preferredStyle === "concise") {
|
||||
snippets.push({
|
||||
reason: "preferred-style",
|
||||
block: "사용자는 짧고 바로 실행 가능한 답변을 선호해요.",
|
||||
});
|
||||
} else if (profile.preferredStyle === "warm") {
|
||||
snippets.push({
|
||||
reason: "preferred-style",
|
||||
block: "사용자는 따뜻한 톤에 잘 반응했어요. 다만 불필요한 절차 설명은 붙이지 마세요.",
|
||||
});
|
||||
}
|
||||
|
||||
return snippets;
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import { withSpan } from "@obs/spans";
|
||||
import type { RetrievedDocument } from "./memory-types.js";
|
||||
|
||||
export interface Reranker {
|
||||
rerank(
|
||||
query: string,
|
||||
candidates: RetrievedDocument[],
|
||||
topN: number,
|
||||
): Promise<RetrievedDocument[]>;
|
||||
}
|
||||
|
||||
interface RankingItem {
|
||||
id?: unknown;
|
||||
score?: unknown;
|
||||
}
|
||||
|
||||
interface RerankerPayload {
|
||||
rankings?: unknown;
|
||||
}
|
||||
|
||||
function truncate(text: string, maxLength: number): string {
|
||||
return text.length <= maxLength ? text : `${text.slice(0, maxLength)}...`;
|
||||
}
|
||||
|
||||
function parseRankings(raw: string): Array<{ id: string; score: number }> {
|
||||
const parsed = JSON.parse(raw) as RerankerPayload;
|
||||
if (!Array.isArray(parsed.rankings)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return parsed.rankings.flatMap((item: RankingItem) => {
|
||||
if (typeof item.id !== "string") {
|
||||
return [];
|
||||
}
|
||||
const score =
|
||||
typeof item.score === "number" && Number.isFinite(item.score)
|
||||
? Math.max(0, Math.min(1, item.score))
|
||||
: 0;
|
||||
return [{ id: item.id, score }];
|
||||
});
|
||||
}
|
||||
|
||||
export class LlmReranker implements Reranker {
|
||||
constructor(private readonly llmClient: Pick<LlmClient, "chatCompletion">) {}
|
||||
|
||||
async rerank(
|
||||
query: string,
|
||||
candidates: RetrievedDocument[],
|
||||
topN: number,
|
||||
): Promise<RetrievedDocument[]> {
|
||||
if (candidates.length <= 1) {
|
||||
return candidates.slice(0, topN);
|
||||
}
|
||||
|
||||
return withSpan(
|
||||
"memory.reranker",
|
||||
{
|
||||
"memory.reranker.candidates_count": candidates.length,
|
||||
"memory.reranker.top_n": topN,
|
||||
},
|
||||
async (span) => {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const result = await this.llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"너는 검색 결과 reranker다.",
|
||||
"질문과 후보 문서를 보고 관련도가 높은 순서로만 정렬한다.",
|
||||
'JSON으로만 응답한다: {"rankings":[{"id":"문서 id","score":0.0-1.0}]}',
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`[query]\n${query}`,
|
||||
"",
|
||||
"[candidates]",
|
||||
...candidates.map((candidate, index) =>
|
||||
[
|
||||
`#${index + 1} id=${candidate.id} score=${candidate.score.toFixed(3)}`,
|
||||
truncate(candidate.text, 600),
|
||||
].join("\n"),
|
||||
),
|
||||
].join("\n"),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: 200,
|
||||
temperature: 0,
|
||||
tier: 0,
|
||||
},
|
||||
);
|
||||
|
||||
if (!result?.content.trim()) {
|
||||
return candidates.slice(0, topN);
|
||||
}
|
||||
|
||||
const byId = new Map(candidates.map((candidate) => [candidate.id, candidate]));
|
||||
const rankings = parseRankings(result.content);
|
||||
const ranked = rankings
|
||||
.flatMap((ranking) => {
|
||||
const candidate = byId.get(ranking.id);
|
||||
return candidate ? [{ ...candidate, score: ranking.score }] : [];
|
||||
})
|
||||
.slice(0, topN);
|
||||
const seen = new Set(ranked.map((candidate) => candidate.id));
|
||||
const next = [
|
||||
...ranked,
|
||||
...candidates.filter((candidate) => !seen.has(candidate.id)),
|
||||
].slice(0, topN);
|
||||
|
||||
span.setAttributes({
|
||||
"memory.reranker.changed_order": next[0]?.id !== candidates[0]?.id,
|
||||
"memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt),
|
||||
});
|
||||
return next;
|
||||
} catch (error) {
|
||||
getRequestLogger().warn(
|
||||
{ err: error },
|
||||
"[Memory] Reranker failed; using original retrieval order",
|
||||
);
|
||||
span.setAttributes({
|
||||
"memory.reranker.changed_order": false,
|
||||
"memory.reranker.duration_ms": Math.max(0, Date.now() - startedAt),
|
||||
});
|
||||
return candidates.slice(0, topN);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,11 @@ import { getRequestLogger } from "@obs/context";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||
import { getRagEnabled } from "../../config.js";
|
||||
import {
|
||||
getRagEnabled,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
} from "../../config.js";
|
||||
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
||||
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
||||
import { createModerationToolRegistry } from "./tools.js";
|
||||
@@ -150,6 +154,8 @@ async function lookupReferenceText(
|
||||
try {
|
||||
const docs = await memoryService.longTerm.search(message, {
|
||||
category: "rule",
|
||||
hybrid: getRagHybridEnabled(),
|
||||
rerank: getRagRerankerEnabled(),
|
||||
skill: "moderation",
|
||||
topK: 3,
|
||||
});
|
||||
|
||||
@@ -66,6 +66,8 @@ export const ONBOARDING_PROMPT_DATA = {
|
||||
"[답변 원칙]",
|
||||
"- 마지막 메시지에서 직접 물은 내용에만 정확히 답해.",
|
||||
"- 사용자가 요청하지 않은 다음 단계, 방법, 위치, 예시는 먼저 붙이지 마.",
|
||||
"- [user_context] 블록이 있으면 그 안의 정보를 응답 톤과 우선순위에 반영해.",
|
||||
"- [long_term_memory] 블록이 있으면 사용자의 과거 패턴을 참고하되, 사용자에게 직접 언급하지 마.",
|
||||
"- 마크다운 문법, 링크, URL은 쓰지 마.",
|
||||
'- 온보딩과 무관한 잡담이거나 다른 사람에게 하는 말이면 "[NO_RESPONSE]"만 출력해.',
|
||||
],
|
||||
|
||||
@@ -144,6 +144,28 @@ function formatRetrievedKnowledge(context: PromptContext): string[] {
|
||||
];
|
||||
}
|
||||
|
||||
function formatProfileSnippets(context: PromptContext): string[] {
|
||||
const snippets = context.profileSnippets ?? [];
|
||||
if (snippets.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
'',
|
||||
'[user_context]',
|
||||
snippets.map((snippet) => `- ${snippet.block}`).join('\n'),
|
||||
];
|
||||
}
|
||||
|
||||
function formatLongTermMemory(context: PromptContext): string[] {
|
||||
const summary = context.userProfile?.longTermSummary?.text?.trim();
|
||||
if (!summary) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return ['', '[long_term_memory]', summary];
|
||||
}
|
||||
|
||||
export function personaLayer(): string {
|
||||
return ONBOARDING_PROMPT_DATA.persona;
|
||||
}
|
||||
@@ -222,6 +244,8 @@ export function contextLayer(context: PromptContext): string {
|
||||
formatUserProfile(context.userProfile),
|
||||
]
|
||||
: []),
|
||||
...formatProfileSnippets(context),
|
||||
...formatLongTermMemory(context),
|
||||
...formatRetrievedKnowledge(context),
|
||||
].join('\n');
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import type { GarnetClient } from '@common/garnet';
|
||||
import { getConversationSummaryKey } from '@conv/conversation-store';
|
||||
import type { ChatMessage } from '@llm/types';
|
||||
import type { MemoryService } from '../../../memory/memory-types.js';
|
||||
import { deriveProfilePromptSnippets } from '../../../memory/profile-update-policy.js';
|
||||
import { getRequestLogger } from '@obs/context';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
import { meetsTrustTier } from '../../../eval/governance/trust.js';
|
||||
@@ -10,6 +11,8 @@ import {
|
||||
getAgentTrustMinTier,
|
||||
getAgentUserMemoryEnabled,
|
||||
getRagEnabled,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
getPromptVariantId,
|
||||
} from '../../../config.js';
|
||||
import {
|
||||
@@ -127,6 +130,8 @@ export async function buildLlmMessages(
|
||||
query: buildRetrievalQuery(request, derived),
|
||||
options: {
|
||||
category: 'onboarding-step',
|
||||
hybrid: getRagHybridEnabled(),
|
||||
rerank: getRagRerankerEnabled(),
|
||||
skill: 'onboarding',
|
||||
topK: 2,
|
||||
},
|
||||
@@ -173,6 +178,7 @@ export async function buildLlmMessages(
|
||||
throw new Error('Failed to load onboarding blackboard');
|
||||
}
|
||||
|
||||
const userProfile = workingMemory?.userProfile ?? legacyUserProfile;
|
||||
const context: PromptContext = {
|
||||
blackboard,
|
||||
conversationHistory:
|
||||
@@ -180,10 +186,13 @@ export async function buildLlmMessages(
|
||||
? workingMemory.conversationHistory
|
||||
: conversationHistory,
|
||||
roomContext,
|
||||
profileSnippets: userProfile
|
||||
? deriveProfilePromptSnippets(userProfile, derived)
|
||||
: undefined,
|
||||
retrievedKnowledge: workingMemory?.retrievedKnowledge,
|
||||
summary: workingMemory?.summary ?? legacySummary,
|
||||
userMessage: buildPromptUserMessage(request, derived),
|
||||
userProfile: workingMemory?.userProfile ?? legacyUserProfile,
|
||||
userProfile,
|
||||
};
|
||||
|
||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||
|
||||
@@ -2,6 +2,10 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/ty
|
||||
import type { GarnetClient } from '@common/garnet';
|
||||
import type { NatsClient } from '@common/nats';
|
||||
import type { MemoryService } from '../../memory/memory-types.js';
|
||||
import {
|
||||
buildLongTermSummary,
|
||||
shouldRefreshLongTermSummary,
|
||||
} from '../../memory/long-term-summarizer.js';
|
||||
import { getContext, getRequestLogger } from '@obs/context';
|
||||
import type { LlmClient } from '@llm/llm-client';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
@@ -427,12 +431,43 @@ export class OnboardingSkill implements Skill {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
||||
const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
||||
flaggedIssues:
|
||||
derived.phase.kind === 'nickname-rejected'
|
||||
? derived.phase.issues.map((issue) => issue.message)
|
||||
: [],
|
||||
});
|
||||
if (!shouldRefreshLongTermSummary(profile)) {
|
||||
return;
|
||||
}
|
||||
|
||||
void this.refreshLongTermSummary(request, profile);
|
||||
}
|
||||
|
||||
private async refreshLongTermSummary(
|
||||
request: AgentRequest,
|
||||
profile: Awaited<ReturnType<UserMemoryStore['observeTurn']>>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const conversationSummary =
|
||||
(await this.memoryService?.session.getSummary(
|
||||
request.chatroomId,
|
||||
request.userId,
|
||||
)) ?? null;
|
||||
const longTermSummary = await buildLongTermSummary(
|
||||
this.llmClient,
|
||||
profile,
|
||||
conversationSummary,
|
||||
);
|
||||
await this.userMemoryStore.updateProfile(request.userId, {
|
||||
longTermSummary,
|
||||
});
|
||||
} catch (error) {
|
||||
getRequestLogger().warn(
|
||||
{ err: error },
|
||||
'[OnboardingSkill] Failed to refresh long-term user summary',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private makeTemplateFallbackResponse(
|
||||
|
||||
@@ -4,6 +4,8 @@ import {
|
||||
anonymizeText,
|
||||
} from "@eval/anonymize";
|
||||
import { getAgentUserMemoryTtlDays } from "../../../config.js";
|
||||
import { evaluateUserFlags } from "../../../memory/forgetting-policy.js";
|
||||
import type { LongTermSummary } from "../../../memory/long-term-summarizer.js";
|
||||
import type { LongTermMemory } from "../../../memory/memory-types.js";
|
||||
import type { DerivedState, Intent } from "./onboarding-state.js";
|
||||
|
||||
@@ -11,9 +13,11 @@ export type PreferredStyle = "concise" | "neutral" | "warm";
|
||||
|
||||
export interface UserProfile {
|
||||
completedOnboardingAt?: string | undefined;
|
||||
archivedFlaggedIssues?: string[] | undefined;
|
||||
firstSeenAt: string;
|
||||
flaggedIssues: string[];
|
||||
lastSeenAt: string;
|
||||
longTermSummary?: LongTermSummary | undefined;
|
||||
pastIntents: Array<{ count: number; kind: Intent["kind"] }>;
|
||||
preferredStyle: PreferredStyle;
|
||||
totalTurns: number;
|
||||
@@ -115,14 +119,32 @@ export class UserMemoryStore {
|
||||
derived.phase.kind === "completed"
|
||||
? current.completedOnboardingAt ?? now
|
||||
: current.completedOnboardingAt;
|
||||
const flagDecision = evaluateUserFlags(
|
||||
{
|
||||
...current,
|
||||
flaggedIssues: sanitizeIssues([
|
||||
...current.flaggedIssues,
|
||||
...(options?.flaggedIssues ?? []),
|
||||
]),
|
||||
},
|
||||
10,
|
||||
);
|
||||
const next: UserProfile = {
|
||||
...(current.archivedFlaggedIssues || flagDecision.archived.length > 0
|
||||
? {
|
||||
archivedFlaggedIssues: [
|
||||
...(current.archivedFlaggedIssues ?? []),
|
||||
...flagDecision.archived,
|
||||
].slice(-50),
|
||||
}
|
||||
: {}),
|
||||
...(completedOnboardingAt ? { completedOnboardingAt } : {}),
|
||||
firstSeenAt: current.firstSeenAt,
|
||||
flaggedIssues: sanitizeIssues([
|
||||
...current.flaggedIssues,
|
||||
...(options?.flaggedIssues ?? []),
|
||||
]),
|
||||
flaggedIssues: flagDecision.kept,
|
||||
lastSeenAt: now,
|
||||
...(current.longTermSummary
|
||||
? { longTermSummary: current.longTermSummary }
|
||||
: {}),
|
||||
pastIntents: mergeIntent(current.pastIntents, derived.intent.kind),
|
||||
preferredStyle: inferPreferredStyle(
|
||||
current.preferredStyle,
|
||||
@@ -140,4 +162,29 @@ export class UserMemoryStore {
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
async updateProfile(
|
||||
userId: string,
|
||||
patch: Partial<UserProfile>,
|
||||
): Promise<UserProfile> {
|
||||
const current = (await this.load(userId)) ?? emptyProfile(userId);
|
||||
const next: UserProfile = {
|
||||
...current,
|
||||
...patch,
|
||||
firstSeenAt: patch.firstSeenAt ?? current.firstSeenAt,
|
||||
flaggedIssues: patch.flaggedIssues ?? current.flaggedIssues,
|
||||
lastSeenAt: patch.lastSeenAt ?? nowIso(),
|
||||
pastIntents: patch.pastIntents ?? current.pastIntents,
|
||||
preferredStyle: patch.preferredStyle ?? current.preferredStyle,
|
||||
totalTurns: patch.totalTurns ?? current.totalTurns,
|
||||
userId: current.userId,
|
||||
};
|
||||
|
||||
if (this.longTermMemory) {
|
||||
await this.longTermMemory.updateUserProfile(userId, next);
|
||||
} else {
|
||||
await this.garnetClient.setJson(profileKey(userId), next, ttlSeconds());
|
||||
}
|
||||
return next;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { ConversationMessage } from "@agent/types";
|
||||
import type { ProfilePromptSnippet } from "../../memory/profile-update-policy.js";
|
||||
import type { RetrievedDocument } from "../../memory/memory-types.js";
|
||||
import type { RoomMessage } from "@room/types";
|
||||
import type { UserProfile } from "./state/user-memory.js";
|
||||
@@ -27,6 +28,7 @@ export interface Example {
|
||||
export interface PromptContext {
|
||||
blackboard: Blackboard;
|
||||
conversationHistory: ConversationMessage[];
|
||||
profileSnippets?: ProfilePromptSnippet[] | undefined;
|
||||
roomContext?: RoomMessage[] | undefined;
|
||||
retrievedKnowledge?: RetrievedDocument[] | undefined;
|
||||
summary?: string | undefined;
|
||||
|
||||
@@ -2,12 +2,14 @@ import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { ShadowEvalBus } from "@eval/shadow-bus";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { EscalationHistoryService } from "../memory/escalation-history.js";
|
||||
import type { MemoryService } from "../memory/memory-types.js";
|
||||
import type { RoomContextService } from "@room/room-context-service";
|
||||
import type { Skill } from "./skill.js";
|
||||
import type { SkillRegistry } from "./skill-loader.js";
|
||||
|
||||
export interface SkillDeps {
|
||||
escalationHistoryService?: EscalationHistoryService | undefined;
|
||||
garnetClient: GarnetClient;
|
||||
llmClient: LlmClient;
|
||||
memoryService?: MemoryService | undefined;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { UserEscalationHistory } from "../../memory/escalation-history.js";
|
||||
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
|
||||
|
||||
function formatSkill(manifest: SkillManifest): string {
|
||||
@@ -14,16 +15,35 @@ function formatSkill(manifest: SkillManifest): string {
|
||||
export function buildSupervisorSystemPrompt(
|
||||
registry: SkillRegistry,
|
||||
_packet?: HandoffPacket,
|
||||
history?: UserEscalationHistory | null,
|
||||
): string {
|
||||
const skills = registry
|
||||
.list()
|
||||
.filter((manifest) => manifest.name !== "supervisor");
|
||||
const historyBlock = history
|
||||
? [
|
||||
"",
|
||||
"[user_escalation_history]",
|
||||
`총 escalation: ${history.totalEscalations}`,
|
||||
`마지막 escalation: ${history.lastEscalationAt}`,
|
||||
`top reasons: ${history.topReasons.map((item) => `${item.reason}(${item.count})`).join(", ") || "-"}`,
|
||||
`recent resolutions: ${
|
||||
history.escalations
|
||||
.slice(-3)
|
||||
.map((item) => item.resolution?.action)
|
||||
.filter(Boolean)
|
||||
.join(", ") || "-"
|
||||
}`,
|
||||
"- 이 블록은 라우팅 참고용이다. 사용자에게 직접 언급하지 않는다.",
|
||||
]
|
||||
: [];
|
||||
return [
|
||||
"너는 multi-agent 시스템의 supervisor다.",
|
||||
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
|
||||
"",
|
||||
"[사용 가능한 specialist]",
|
||||
...skills.map(formatSkill),
|
||||
...historyBlock,
|
||||
"",
|
||||
"[응답 형식]",
|
||||
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}',
|
||||
|
||||
@@ -10,5 +10,6 @@ export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||
deps.natsClient,
|
||||
deps.llmClient,
|
||||
deps.skillRegistry,
|
||||
deps.escalationHistoryService,
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import type { EscalationHistoryService } from "../../memory/escalation-history.js";
|
||||
import {
|
||||
getSupervisorResolutionsStreamMaxLen,
|
||||
getSupervisorReviewQueueMaxLen,
|
||||
@@ -152,13 +153,13 @@ function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean
|
||||
export async function appendSupervisorReview(
|
||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await garnetClient.appendStream(
|
||||
): Promise<string> {
|
||||
return garnetClient.appendStream(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
{
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
queuedAt: fields["queuedAt"] ?? new Date().toISOString(),
|
||||
},
|
||||
{ maxLen: getSupervisorReviewQueueMaxLen() },
|
||||
);
|
||||
@@ -275,6 +276,10 @@ export async function resolveSupervisorReview(
|
||||
garnetClient: SupervisorReviewResolveGarnetClient,
|
||||
id: string,
|
||||
resolution: SupervisorReviewResolution,
|
||||
options?: {
|
||||
escalationHistoryService?: EscalationHistoryService | undefined;
|
||||
userId?: string | undefined;
|
||||
},
|
||||
): Promise<void> {
|
||||
const fields: Record<string, string> = {
|
||||
action: resolution.action,
|
||||
@@ -297,6 +302,13 @@ export async function resolveSupervisorReview(
|
||||
},
|
||||
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
|
||||
);
|
||||
if (options?.escalationHistoryService && options.userId) {
|
||||
await options.escalationHistoryService.recordResolution({
|
||||
userId: options.userId,
|
||||
reviewId: id,
|
||||
resolution,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function getResolutionHistory(
|
||||
|
||||
@@ -8,6 +8,10 @@ import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type {
|
||||
EscalationHistoryService,
|
||||
UserEscalationHistory,
|
||||
} from "../../memory/escalation-history.js";
|
||||
import { AlertService } from "@obs/alert";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
|
||||
@@ -87,6 +91,7 @@ export class SupervisorSkill implements Skill {
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||
private readonly escalationHistoryService?: EscalationHistoryService | undefined,
|
||||
) {
|
||||
this.alertService = new AlertService(
|
||||
garnetClient,
|
||||
@@ -108,7 +113,8 @@ export class SupervisorSkill implements Skill {
|
||||
_conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const packet = readHandoffPacket(request);
|
||||
const llmDecision = await this.routeByLlm(packet);
|
||||
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
||||
const llmDecision = await this.routeByLlm(packet, escalationHistory);
|
||||
const useLlmRoute =
|
||||
llmDecision !== null &&
|
||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||
@@ -117,14 +123,26 @@ export class SupervisorSkill implements Skill {
|
||||
: routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
const queuedAt = new Date().toISOString();
|
||||
const reviewId = await appendSupervisorReview(this.garnetClient, {
|
||||
chatroomId: request.chatroomId,
|
||||
packet: JSON.stringify(packet ?? null),
|
||||
queuedAt,
|
||||
reason,
|
||||
requestId: request.requestId,
|
||||
target: target ?? "human-review",
|
||||
userId: request.userId,
|
||||
});
|
||||
await this.escalationHistoryService?.recordEscalation({
|
||||
chatroomId: request.chatroomId,
|
||||
packetSummary: packet?.userGoal ?? packet?.summary ?? "",
|
||||
queuedAt,
|
||||
reason,
|
||||
reviewId,
|
||||
sourceChain: packet?.sourceChain ?? [],
|
||||
target: target ?? "human-review",
|
||||
userId: request.userId,
|
||||
});
|
||||
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
|
||||
|
||||
await this.alertService.sendAlert(
|
||||
@@ -154,6 +172,7 @@ export class SupervisorSkill implements Skill {
|
||||
routedBy: useLlmRoute ? "llm" : "rules",
|
||||
source: userMessage.source,
|
||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
@@ -164,6 +183,7 @@ export class SupervisorSkill implements Skill {
|
||||
|
||||
private async routeByLlm(
|
||||
packet?: HandoffPacket,
|
||||
escalationHistory?: UserEscalationHistory | null,
|
||||
): Promise<SupervisorRoutingDecision | null> {
|
||||
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
||||
return null;
|
||||
@@ -173,7 +193,11 @@ export class SupervisorSkill implements Skill {
|
||||
const result = await this.llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
|
||||
content: buildSupervisorSystemPrompt(
|
||||
this.skillRegistry,
|
||||
packet,
|
||||
escalationHistory,
|
||||
),
|
||||
role: "system",
|
||||
},
|
||||
{
|
||||
@@ -213,4 +237,19 @@ export class SupervisorSkill implements Skill {
|
||||
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
||||
);
|
||||
}
|
||||
|
||||
private async loadEscalationHistory(userId: string) {
|
||||
if (!this.escalationHistoryService) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return await this.escalationHistoryService.load(userId);
|
||||
} catch (error) {
|
||||
getLogger().warn(
|
||||
{ err: error },
|
||||
"[Supervisor] Failed to load escalation history",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,11 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { z } from "zod";
|
||||
import { getRagEnabled } from "../../config.js";
|
||||
import {
|
||||
getRagEnabled,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
} from "../../config.js";
|
||||
import { getRequestLogger } from "../../observability/context.js";
|
||||
import type { Tool } from "../tool.js";
|
||||
|
||||
@@ -96,6 +100,8 @@ export const lookupRuleTool: Tool<
|
||||
try {
|
||||
const docs = await ctx.memoryService.longTerm.search(args.topic, {
|
||||
category: "rule",
|
||||
hybrid: getRagHybridEnabled(),
|
||||
rerank: getRagRerankerEnabled(),
|
||||
skill: "moderation",
|
||||
topK: args.topK ?? 3,
|
||||
});
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { z } from "zod";
|
||||
import { getRagEnabled } from "../../config.js";
|
||||
import {
|
||||
getRagEnabled,
|
||||
getRagHybridEnabled,
|
||||
getRagRerankerEnabled,
|
||||
} from "../../config.js";
|
||||
import { getRequestLogger } from "../../observability/context.js";
|
||||
import {
|
||||
DEADLINE_NEAR_THRESHOLD_MIN,
|
||||
@@ -93,6 +97,8 @@ export const lookupCommunityRuleTool: Tool<
|
||||
try {
|
||||
const docs = await ctx.memoryService.longTerm.search(args.topic, {
|
||||
category: "onboarding-step",
|
||||
hybrid: getRagHybridEnabled(),
|
||||
rerank: getRagRerankerEnabled(),
|
||||
skill: "onboarding",
|
||||
topK: args.topK ?? 3,
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user