From d05de52cd9b8650fcaccd480e370bdca20386375 Mon Sep 17 00:00:00 2001 From: dahyeon Date: Sun, 24 May 2026 23:41:09 -0700 Subject: [PATCH] Improve onboarding verification and routing controls --- docs/service-evaluation-operations.md | 33 +- docs/system-overview.md | 1329 +++++++++++++++++ launch-server.ps1 | 36 +- src/common/env-loader.ts | 11 + src/common/garnet-cli.ts | 5 +- src/common/nats-subjects.ts | 21 + src/config.test.ts | 78 + src/config.ts | 138 +- src/eval/golden/onboarding/candidate.jsonl | 60 + src/eval/judges/llm-judge.test.ts | 23 + src/eval/judges/llm-judge.ts | 11 +- src/eval/proposals/579dcc7d.json | 16 + src/eval/proposals/991bfb18.json | 16 + src/eval/runner.ts | 68 +- src/llm/llm-client.ts | 19 +- src/llm/routing.test.ts | 69 +- src/llm/routing.ts | 63 +- src/skills/onboarding/intents.ts | 75 +- src/skills/onboarding/prompt-data.ts | 8 +- src/skills/onboarding/runtime/critic.ts | 3 - .../onboarding/runtime/orchestrator.test.ts | 311 ++++ src/skills/onboarding/runtime/orchestrator.ts | 256 +++- .../onboarding/runtime/tool-selector.ts | 6 +- .../onboarding/state/classifier.test.ts | 9 + src/tools/moderation/lookup-rule.test.ts | 29 +- src/tools/moderation/lookup-rule.ts | 32 +- .../check-onboarding-progress.test.ts | 4 + .../onboarding/check-onboarding-progress.ts | 4 + .../verify-discord-nickname.test.ts | 132 +- .../onboarding/verify-discord-nickname.ts | 321 +++- 30 files changed, 3024 insertions(+), 162 deletions(-) create mode 100644 docs/system-overview.md create mode 100644 src/eval/proposals/579dcc7d.json create mode 100644 src/eval/proposals/991bfb18.json diff --git a/docs/service-evaluation-operations.md b/docs/service-evaluation-operations.md index 44e5fb6..52fd985 100644 --- a/docs/service-evaluation-operations.md +++ b/docs/service-evaluation-operations.md @@ -257,6 +257,18 @@ npm run review:list -- --reason=low-confidence --limit=10 Resolution history는 stream에 쌓이며, compatibility hash는 sunset 전까지 read fallback으로 유지된다. Sunset 조건과 절차는 [resolution-hash-sunset-procedure.md](resolution-hash-sunset-procedure.md)를 따른다. +### 10.2 Tool Audit + +Tool call은 성공과 실패 모두 `lolympus-agent:tool-calls` Stream에 남긴다. 원본 args는 저장하지 않고, 민감 키를 제거한 뒤 hash만 기록한다. + +```bash +npm run tool-audit:list -- --limit=20 +npm run tool-audit:list -- --skill=onboarding --since=24h +npm run tool-audit:show -- +``` + +Stream은 `AGENT_TOOL_AUDIT_MAX_LEN` 기준으로 trim된다. 기본값은 `100000`이다. + ## 11. Calibration 흐름 Calibration은 사람 점수와 LLM judge 점수의 상관을 계산한다. @@ -460,6 +472,13 @@ npm test | `EVAL_TOURNAMENT_MIN_CASES` | 10 | proposal 최소 archive cases | | `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 | | `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path | +| `AGENT_BUDGET_LLM_COST_ENABLED` | true | LLM token cost를 request budget에 누적 | +| `AGENT_COST_AWARE_TIER_ENABLED` | true | confidence/cost-aware tier selection | +| `AGENT_SUPERVISOR_FAST_PATH_ENABLED` | true | supervisor regex fast-path 사용 | +| `SUPERVISOR_FAST_PATH_CONFIDENCE` | 0.85 | fast-path 최소 규칙 confidence | +| `AGENT_TOOL_SANDBOX_ENABLED` | false | tool context 최소 권한 강제 여부 | +| `AGENT_TOOL_AUDIT_MAX_LEN` | 100000 | tool audit Stream maxlen | +| `AGENT_NATS_ACL_MODE` | log-only | NATS subject ACL 모드 | | `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 | | `PHOENIX_OTLP_ENDPOINT` | unset | Phoenix OTLP trace endpoint | | `PHOENIX_LOG_PROMPTS` | false | LLM prompt/completion content trace 기록 | @@ -519,7 +538,15 @@ ALERT_THRESHOLD_DRIFT_PCT=15 예를 들어 onboarding latency alert와 moderation latency alert는 서로 throttle하지 않는다. skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한다. -### 16.2 Phoenix trace +### 16.2 Secret, Tool Sandbox, NATS ACL + +Secret은 Infisical을 우선으로 로드한다. Infisical credential은 로컬 env 또는 IMDS에서 읽고, 새 secret env를 추가할 때는 `src/common/known-secret-keys.ts`에 함께 등록해야 로그 redaction 대상에 들어간다. LLM, embedding, Phoenix API key도 redaction 대상이다. + +Tool은 `requires` 선언으로 필요한 context만 받는다. `AGENT_TOOL_SANDBOX_ENABLED=false`가 기본이라 운영 초기는 호환성을 유지한다. 1주 정도 audit을 본 뒤 false positive가 없으면 true로 전환한다. + +NATS subject ACL은 `src/common/nats-skill-acl.ts`의 `DEFAULT_NATS_ACL`에서 관리한다. 기본 `AGENT_NATS_ACL_MODE=log-only`는 위반을 warn으로만 남기고 publish/request는 그대로 진행한다. 1개월 운영 후 false positive가 없으면 `enforce`로 전환한다. + +### 16.3 Phoenix trace LLM 호출, skill 실행, tool 실행 span은 OpenTelemetry trace로 남긴다. `PHOENIX_OTLP_ENDPOINT`가 없으면 기존 설정을 따른다. 개발 환경 기본값은 trace exporter 비활성화이고, `OTEL_EXPORTER=stdout`이면 콘솔 exporter를 쓴다. @@ -544,7 +571,7 @@ Phoenix UI에서는 `service.name=lolympus-agent`, `gen_ai.system`, `gen_ai.requ Prompt와 completion 본문은 기본적으로 기록하지 않는다. 디버깅 때문에 필요할 때만 `PHOENIX_LOG_PROMPTS=true`를 켠다. 이 경우 email, 전화번호, token/API key 형태의 값은 span 기록 전에 masking된다. -### 16.3 Prometheus scrape +### 16.4 Prometheus scrape Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다. @@ -596,7 +623,7 @@ sum by (skill, stage) (rate(agent_validator_fail_total[5m])) | eval drift | 15% 이하 | | archive stalled | 3회 연속 실패 0건 | -### 16.4 Slack webhook 설정 +### 16.5 Slack webhook 설정 Slack incoming webhook URL을 env로 설정한다. diff --git a/docs/system-overview.md b/docs/system-overview.md new file mode 100644 index 0000000..99b7ae8 --- /dev/null +++ b/docs/system-overview.md @@ -0,0 +1,1329 @@ +# Lolympus Agent — 시스템 아키텍처 및 운영 흐름 + +> Iter1~12 (2026-05-13 ~ 2026-05-19) 완료 시점 기준의 통합 architecture + operations 문서입니다. +> 신규 기여자/운영자/AI agent의 단일 진입점. + +--- + +## 0. 한눈에 보기 + +**Lolympus Agent**는 카카오톡 오픈채팅 운영을 자동화하는 **NATS 메시지 드리븐 멀티 에이전트 시스템**이다. + +- **도메인**: 카카오톡 오픈채팅 가입 안내(onboarding) + 커뮤니티 규칙 모더레이션(moderation) +- **외부 진입**: NATS subject `lolympus-agent.request` +- **응답 형태**: NATS reply (단일 publish/요청) +- **핵심 자산**: + - Skill SSOT 디렉토리 (`src/skills/{onboarding,moderation,supervisor}/`) + - 통합 dispatch loop (`src/agent/dispatch.ts`, production + eval 공용) + - 3-layer memory + Agentic RAG (`src/memory/`) + - Bounded Self-Escalation Graph (handoff + transition matrix + budget + supervisor fallback) + - Eval 파이프라인 (capture → run → propose → tournament → promote) + - 자체 metrics + Phoenix LLM tracing + - Slack webhook + operations CLI + +### 권장 시작 순서 +1. **§2 아키텍처 다이어그램** — 전체 그림 +2. **§3 메시지 흐름** — 한 요청이 어떻게 흐르는지 +3. **§4~9** — 각 컴포넌트 깊이 있게 +4. **§16 cheat sheet** — 자주 쓰는 명령 + +--- + +## 1. 기술 스택 요약 + +| 영역 | 선택 | 위치 | +|---|---|---| +| 언어 | TypeScript (Node 22+) | `tsconfig.json` | +| 메시지 | NATS | `src/common/nats.ts` | +| 캐시/세션 | Garnet (Redis 호환) | `src/common/garnet.ts` | +| 영속 저장 | MongoDB (mongoose) | `src/room-context/`, `src/memory/mongo-*.ts` | +| LLM provider | OpenRouter / OpenAI / cliproxy / llama (multi-provider) | `src/llm/llm-client.ts`, `src/llm/routing.ts` | +| Embedding | OpenRouter free 우선 → OpenAI fallback | `src/memory/embedding-client.ts` | +| Vector store | MongoDB collection + in-app cosine (Vector Search 마이그레이션 path 예약) | `src/memory/mongo-long-term-memory.ts`, `src/memory/cosine.ts` | +| BM25 | in-app | `src/memory/bm25.ts` | +| 관측 | OTel SDK + Phoenix OTLP + 자체 metrics 모듈 | `src/observability/` | +| 알림 | NATS publish + Slack incoming webhook | `src/observability/alert.ts`, `slack-notifier.ts` | +| Secret | Infisical → IMDS → env | `src/common/secret-manager.ts` | +| Test | Vitest | `*.test.ts` | +| Plan/Eval 데이터 | MongoDB collections + Garnet streams | 각 모듈 | + +새 의존성 도입 시 §15 환경변수 + KNOWN_SECRET_KEYS 갱신 확인. + +--- + +## 2. 아키텍처 다이어그램 + +``` +[NATS subject "lolympus-agent.request"] + │ + ▼ + ┌─────────────────────────────────┐ + │ Agent.handleRequest │ src/agent/agent.ts (~250 lines) + │ - runWithContext (AsyncLocal)│ + │ - withSpan (OTel) │ + │ - budget 생성, NATS reply │ + └────────────┬────────────────────┘ + │ dispatch(input, deps) + ▼ + ┌─────────────────────────────────┐ + │ dispatch loop │ src/agent/dispatch.ts (~580 lines) + │ ├─ selectSkillForRequest │ priority + canHandle 필터 + │ ├─ runSkillOnce │ skill.handle() 호출 + │ ├─ normalizeControlTokens │ [HANDOFF:...] 파싱 + │ ├─ applyRuntimeVerifier │ retry/escalate + │ ├─ applyEscalationTrigger │ confidence/risk → handoff + │ ├─ prepareHandoff │ TransitionMatrix + Budget 검증 + │ └─ recurse (max depth 4) │ 같은 dispatch 재진입 + └────────────┬────────────────────┘ + │ + ┌───────┴───────┐ + ▼ ▼ + [Onboarding] [Moderation] [Supervisor] + ├ intents ├ intents ├ capability-prompt (LLM tier0) + ├ prompt-data ├ prompt-data ├ routing-rules (regex fast-path) + ├ tools (5) ├ tools (1) └ review-queue append + alert + ├ runtime/ ├ skill.ts + └ skill.ts + + ┌─────────────────────────────────┐ + │ MemoryService (Iter10-11) │ src/memory/ + │ ├─ WorkingMemory (조립) │ 매 turn + │ ├─ SessionMemory (12h Garnet)│ blackboard + conv + summary + │ └─ LongTermMemory (30d+) │ + │ ├─ UserProfile (Garnet) │ + │ ├─ Knowledge (Mongo+RAG) │ embedding + cosine + BM25 + reranker + │ └─ EscalationHistory (Mongo)│ user-level review queue rollup + └─────────────────────────────────┘ + + ┌─────────────────────────────────┐ + │ ToolRegistry │ src/tools/ + │ ├─ Tool.requires (sandbox) │ + │ ├─ scoped-context │ least-privileged ctx + │ └─ audit stream │ lolympus-agent:tool-calls + └─────────────────────────────────┘ + + ┌─────────────────────────────────┐ + │ Observability │ src/observability/ + │ ├─ Pino logger (skill child) │ + │ ├─ OTel tracing → Phoenix │ gen_ai.* + memory.* + handoff.* + │ ├─ Self-built metrics │ /metrics endpoint (9464) + │ └─ AlertService │ NATS + Slack webhook + └─────────────────────────────────┘ + + ┌─────────────────────────────────┐ + │ Eval Pipeline │ src/eval/ + │ ├─ Capture (shadow-bus) │ prod traffic 샘플링 + │ ├─ Run (runner) │ golden cases 실행 + │ ├─ Judges (rule/llm/process/ │ + │ │ handoff/memory) │ + │ ├─ Propose (variant 생성) │ + │ ├─ Tournament (A/B) │ + │ └─ Promote (T4→T2) │ + └─────────────────────────────────┘ +``` + +--- + +## 3. 메시지 수신 → 응답 흐름 + +### 3.1 진입점 — `Agent.handleRequest` + +`src/agent/agent.ts`. 흐름: + +1. **NATS raw 수신**: `natsClient.subscribeRaw(getRequestSubject(), this.handleRequest)` (start time) +2. **JSON 파싱 + 검증**: `isValidRequest(parsed)` 통과 안 하면 즉시 empty response +3. **`runWithContext`** 호출 — `src/observability/context.ts`. AsyncLocalStorage로 `RequestContext` 주입: + - `requestId`, `chatroomId`, `userId`, `skill`, `trigger` + - `logger` (skill name child binding) + - `startedAt`, `deadlineAt` + - `budget`, `handoffDepth`, `handoffChain` (Iter4) +4. **`withSpan('agent.handle')`** OTel 외부 span 시작 +5. **`dispatch(input, deps)`** 호출 (실제 비즈니스 로직은 dispatch.ts에 위임 — Iter6 추출) +6. **단일 NATS reply**: `natsClient.respond(msg, response)` (line 144 등) — dispatch 종료 후 정확히 1회 + +### 3.2 dispatch loop — `src/agent/dispatch.ts` + +핵심 재귀 함수 `dispatch(input, deps)`: + +``` +인자: + input: { budget, chain, depth, history, request, skill } + deps: { selectSkillForRequest, skills, transitionMatrix, + shouldApplyRuntimeVerifier, getSkillJudgeConfig, + onHandoff?, onDispatchEnd?, evalPlan? } + +순서: + 1) runSkillOnce(input) → response # skill.handle() + 2) normalizeControlTokens(response) # [HANDOFF:..] → metadata + 3) applyRuntimeVerifier(input, response) # pass/retry/escalate (Iter6) + 4) applyEscalationTrigger(...) # confidence/risk → handoff + 5) withHandoffMetadata(response, chain, depth) # handoffChain 노출 (Iter5) + 6) readHandoffRequest(response) # metadata.handoffRequest + └─ null → 최종 응답 반환 + 7) prepareHandoff(...) # next skill + packet 합성 + ├─ resolveHandoffTarget (TransitionMatrix 검증) + ├─ budget.assertBeforeHandoff (max-hops/loop/cost) + ├─ budget.recordHandoff(target) + └─ buildHandoffPacket (PII 차단, evidence_refs) + 8) dispatch(next) 재귀 +``` + +### 3.3 dispatch 종료 후 + +- `metadata.handoffChain: string[]` 자동 부착 (eval에서 chain 검증 가능) +- `metadata.handoffDepth: number` +- `metadata.handoffRequest`는 마지막 handoff 정보 (없으면 미부착) +- `metadata.routedBy`: supervisor의 경우 `"regex-fast-path" | "llm" | "rules-fallback"` (Iter12) + +### 3.4 단일 publish 보장 + +NATS publish 지점은 `handleRequest` 끝 한 곳뿐. 4 hop 재귀라도 사용자는 응답 한 번만 받음. + +--- + +## 4. Skill 라우팅 + +### 4.1 Skill 등록 (bootstrap) + +`src/main.ts`: + +``` +1. SkillRegistry.load("src/skills") + - 각 폴더의 SKILL.md frontmatter 파싱 → SkillManifest 배열 +2. factories map = { onboarding, moderation, supervisor } +3. 등록 루프: agent.registerSkill(factory({ ...deps })) + - deps: garnetClient, llmClient, roomContextService, shadowEvalBus, + natsClient, memoryService, escalationHistoryService, ... +4. TransitionMatrix.fromRegistry(skillRegistry) → Agent에 주입 +``` + +### 4.2 SKILL.md frontmatter (capability card) + +각 `src/skills//SKILL.md`: + +```yaml +name: onboarding +description: ... +version: 1 +priority: 70 # canHandle 공동 당첨 시 desc 정렬 +intents: [greeting, ..., age-question] +triggers: [user-message, new-member, ...] +tools: [validate-nickname, ...] +resources: + nickname-format: ./resources/nickname-format.md + ... +trustTier: T3 # T1(가장 신뢰) ~ T4(가장 보수적) +domain: onboarding # Iter4 capability card +canHandle: [...] +cannotHandle: [...] +handoffTargets: [moderation, supervisor] +riskTier: low | medium | high +outputContract: + format: text + maxChars: 600 +``` + +### 4.3 라우팅 결정 — `selectSkillForRequest` + +`src/agent/agent.ts:28-37`: + +``` +1. skills.filter(s => s.canHandle(request)) + - 각 Skill의 canHandle: 보통 request.skill === self.name (정확 매칭) +2. priority desc 정렬 +3. [0] 반환 +``` + +- onboarding: `request.skill === "onboarding"` (priority 70) +- moderation: `request.skill === "moderation"` (priority 50) +- supervisor: `request.skill === "supervisor"` OR fallback (priority 10) + +### 4.4 TransitionMatrix (handoff 그래프) + +`src/skills/transition-matrix.ts`. 각 capability card의 `handoffTargets`를 모아 그래프 구성. `supervisor`는 모든 항목의 fallback 자동 포함. + +``` +isAllowed(from, to) # true/false +targetsOf(from) # 허용 target 집합 +validateGraph() # 빌드타임 일관성 +``` + +Dispatch loop에서 handoff 시 `transitionMatrix.isAllowed()` 검사. 거부 시 강제 supervisor로 escalation. + +### 4.5 검증 — `npm run lint:skills` + +`src/skills/validate-manifests.ts`: + +- intents 배열 ⊆ 코드 `IntentKind` (manifest-binding.ts에서 derive) +- tools 배열 == ToolRegistry tool names +- resources 키들 실제 파일 존재 +- priority 중복 0 +- handoffTargets 항목이 실제 skill name 또는 `supervisor` +- cannotHandle ∩ canHandle = ∅ + +매 PR 시 CI에서 `lint:skills` 통과 필수. + +--- + +## 5. Memory 계층 + +자료의 강조점 "context engineering > prompt engineering" + MemoryAgentBench 4축 (accurate retrieval / test-time learning / long-range understanding / selective forgetting) 모두 구현. + +### 5.1 3-layer 인터페이스 (Iter10) + +`src/memory/memory-types.ts`: + +```ts +WorkingMemory // 매 turn 조립. dispatch loop가 사용 +SessionMemory // 현 대화 세션. Garnet 12h TTL +LongTermMemory // 영속. Garnet (profile) + MongoDB (knowledge) +``` + +### 5.2 SessionMemory (Garnet, 12h~24h) + +| 데이터 | Garnet key | TTL | 소스 | +|---|---|---|---| +| Blackboard (skill-local) | `lolympus-agent:bb:{skill}:{chatroom}:{user}` | 12h | `src/skills/onboarding/state/blackboard.ts` | +| Conversation history | `lolympus-agent:conv:{chatroom}:{user}` | 12h | `src/conversation/conversation-store.ts`, max 20 메시지 | +| Conversation summary | `lolympus-agent:summary:{chatroom}:{user}` | 24h | `src/conversation/summarizer.ts`, 5+ 메시지 trigger | + +### 5.3 LongTermMemory (Garnet + MongoDB, 30d+) + +| 데이터 | 저장소 | TTL | +|---|---|---| +| UserProfile (preferredStyle, pastIntents, flaggedIssues, longTermSummary) | Garnet `lolympus-agent:user:profile:{user}` | 30d (env: `AGENT_USER_MEMORY_TTL_DAYS`) | +| KnowledgeDocuments (RAG) | MongoDB collection `knowledge_documents` | 영속 (eviction policy로 archive) | +| UserEscalationHistory | MongoDB collection `user_escalation_history` | 영속 | + +`UserProfile` 필드 (`src/skills/onboarding/state/user-memory.ts`): + +```ts +userId: string // anonymizeText() 처리 +firstSeenAt, lastSeenAt: string +completedOnboardingAt?: string +totalTurns: number +pastIntents: Array<{kind, count}> // 최근 10개 +preferredStyle: "warm" | "concise" | "neutral" // 자동 추론 +flaggedIssues: string[] // 최대 10 +longTermSummary?: { text, updatedAt, turnSnapshot } // Iter11 +``` + +### 5.4 WorkingMemory 조립 + +매 요청 시 `message-builder.ts`에서 호출: + +```ts +const wm = await memoryService.assembleWorkingMemory({ + skill: "onboarding", + chatroomId, userId, + includeBlackboard: true, + includeProfile: true, + retrieval: shouldRetrieve(derived) + ? { query: ..., options: { skill, category, topK: 2 } } + : undefined, +}); +// wm: { conversationHistory, summary, blackboard, userProfile, retrievedKnowledge } +``` + +`shouldRetrieve(derived)` (Iter10 Phase D): help-how, help-where, nickname-question, discord-question, age-question 5개 intent만 RAG. greeting/acknowledgement는 skip(비용 절감). + +### 5.5 LLM prompt 주입 블록 + +`message-builder.ts` + `prompt-data.ts`: + +``` +[reference] ← retrieval된 knowledge chunk +[user_context] ← test-time learning snippets (자주 묻는 intent 등) +[long_term_memory] ← UserProfile.longTermSummary (Iter11) +[user_escalation_history] ← supervisor capability-prompt에만 (internal only) +``` + +--- + +## 6. Agentic RAG + +자료의 "modular > monolithic" + "retrieval은 pre-filter → rerank → compression" 권고 적용. + +### 6.1 Embedding (`src/memory/embedding-client.ts`) + +추상화 인터페이스 + 2개 구현 (Iter10): + +```ts +interface EmbeddingClient { + embed(text): Promise + embedBatch(texts): Promise + model: string + dimensions: number +} + +createEmbeddingClient() + // env: EMBEDDING_PROVIDER ("openrouter" | "openai") + // OpenRouter 우선 → 미지원 시 OpenAI fallback + // env: EMBEDDING_MODEL (예: "text-embedding-3-small") + // env: EMBEDDING_API_KEY 또는 OPENROUTER_API_KEY +``` + +### 6.2 Ingest — `npm run ingest:knowledge` + +`scripts/ingest-knowledge.ts` → `src/memory/knowledge-ingest.ts`: + +1. `src/skills/{onboarding,moderation}/resources/*.md` 5개 파일 로드 +2. `## ` heading 단위로 split +3. 각 chunk를 `{ skill, category, text, metadata: { sourceFile, sectionTitle } }`로 build +4. `embeddingClient.embed(text)` → vector +5. MongoDB `knowledge_documents` collection upsert (`metadata.sourceFile + sectionTitle` 기준 idempotent) +6. `embeddingModel` 필드 자동 기록 (마이그레이션 식별용) + +총 20~50 chunks 예상. 향후 1000+ 시 MongoDB Vector Search 마이그레이션. + +### 6.3 Retrieval pipeline (Iter10 → Iter11) + +`src/memory/mongo-long-term-memory.ts:search()`: + +``` +1. embeddingClient.embed(query) → query vector +2. MongoDB find with { skill?, category?, metadata.archived: { $ne: true } } + → 후보 N개 (env: AGENT_RAG_CANDIDATE_POOL_SIZE, 기본 50) +3. (옵션) Hybrid (Iter11 Phase A) + - cosine score (in-app, src/memory/cosine.ts) + - BM25 score (in-app, src/memory/bm25.ts) + - 가중 평균 (각 0.5) +4. top-K 선정 (기본 10) +5. (옵션) Reranker (Iter11 Phase A) + - LlmReranker (tier 0, maxTokens 200, jsonMode) + - 실패 시 원래 순서 (graceful) +6. top-N 반환 (기본 5, env: AGENT_RAG_RERANKER_TOP_K) +``` + +각 단계에서 Phoenix span attribute: +- `memory.retrieval.candidate_count` +- `memory.retrieval.top_score` +- `memory.retrieval.duration_ms` +- `memory.reranker.candidates_count` +- `memory.reranker.changed_order` (boolean) + +### 6.4 Selective forgetting (Iter11 Phase C) + +`src/memory/forgetting-policy.ts` + `src/memory/contradiction-detector.ts`: + +3가지 규칙: +- **Relevance**: 30일간 reference 0회 + `referenceCount30d < 3` → archive +- **Freshness**: `updatedAt` 90일 이상 + 사용 빈도 낮음 → archive +- **Contradiction**: 같은 `sourceFile + sectionTitle` 그룹 내 텍스트 30%+ 차이 또는 명시적 키워드 → 기존 archive, 새 doc primary (staging 필수) + +`scripts/memory-eviction-cron.ts` (`npm run memory:evict --dry-run`): +- archived는 `metadata.archived = true` flag만 (delete 아님). 복구 가능 +- search() 자동으로 archived 제외 + +### 6.5 사용 site + +**Moderation `lookup-rule` tool** (Iter10 Phase C): +``` +args: { topic: string, topK?: number } +→ memoryService.longTerm.search(topic, { skill: "moderation", category: "rule" }) +→ tool result에 chunk text 포함 +→ LLM이 retrieval된 reference로 답변 +``` + +**Onboarding `message-builder.ts`** (Iter10 Phase D): +- 5개 intent에서만 retrieval +- `[reference]` block에 chunk 주입 +- LLM이 ground truth 기반 답변 + +--- + +## 7. Tool 실행 + +### 7.1 Tool 인터페이스 (Iter12 Phase B) + +`src/tools/tool.ts`: + +```ts +interface Tool { + name: string + description: string + argsSchema: ZodSchema + resultSchema: ZodSchema + requires?: readonly ToolPermission[] // Iter12: 권한 declaration + execute(args: A, ctx: ToolContext): Promise> +} + +type ToolPermission = + | "garnet" | "nats" | "room-context" + | "user-session" | "tool-registry" + | "memory-search" | "memory-profile" +``` + +### 7.2 Scoped Context Injection (Iter12 Phase B) + +`src/tools/scoped-context.ts`: + +``` +buildScopedContext(tool.requires ?? [], fullContext) → scopedContext +``` + +- 권한 없는 필드 접근 시: + - `AGENT_TOOL_SANDBOX_ENABLED=true`: throw + - `AGENT_TOOL_SANDBOX_ENABLED=false` (log-only): warn 로그만 + +### 7.3 등록된 tool + +| Tool | Skill | requires | +|---|---|---| +| validate-nickname | onboarding | `[]` (pure) | +| check-onboarding-progress | onboarding | `["garnet", "user-session"]` | +| lookup-community-rule | onboarding | `["memory-search"]` | +| get-room-context | onboarding | `["room-context"]` | +| verify-discord-nickname | onboarding | `["garnet", "nats", "user-session"]` | +| lookup-rule | moderation | `["memory-search"]` | + +### 7.4 Tool Audit (Iter12 Phase B) + +`src/tools/audit.ts` + Garnet stream `lolympus-agent:tool-calls`: + +매 tool 호출 후 envelope append: +```ts +{ + type: "tool-call", + toolName, chatroomId, userId, + argsHash: string, // PII 마스킹된 hash (원본 args 미저장) + resultSummary, ok, errorKind?, + durationMs, skill, traceId, calledAt +} +``` + +- maxLen 100_000 자동 trim +- Operations CLI: `npm run tool-audit:list`, `npm run tool-audit:show ` + +### 7.5 외부 통합 (lolympus 도메인) + +- **Discord**: `verify-discord-nickname` tool이 NATS subject `lolympus.discord.lookup-members`로 외부 microservice에 위임 (HTTP 직접 호출 X) +- **Kakao**: session에 nickname 정보만 (외부 API 호출 없음, 클라이언트 제공) + +--- + +## 8. Handoff + Supervisor (Bounded Self-Escalation Graph) + +자료의 "Bounded Self-Escalation Graph" 8가지 규칙 모두 적용 (Iter4-6). + +### 8.1 HandoffRequest schema (Iter4) + +`src/agent/types.ts`: + +```ts +interface HandoffRequest { + targetAgent: string + reason: HandoffReason // 8가지 enum + taskSpec: string + evidenceRefs: string[] // toolCall id 또는 messageId + confidence: number // 0..1 + risk: "low" | "medium" | "high" + expectedOutput: string + userVisibleMessage?: string +} + +type HandoffReason = + | "out-of-scope" | "missing-tool" | "low-confidence" + | "validation-failed" | "same-agent-revisit" + | "max-hop" | "high-risk-action" | "policy-risk" +``` + +### 8.2 [HANDOFF:..] 토큰 파이프라인 (Iter4 Phase D) + +기존 `[NO_RESPONSE]` 패턴을 확장. 5단계: + +1. **Prompt** — `prompt-data.ts` persona에 사용법 명시 (capability card 기반 자동 합성) +2. **Abstention** — `runtime/abstention.ts`의 `AbstentionAction`에 `{kind:"handoff", targetSkill, reason}` variant +3. **Validator** — `runtime/validator.ts:142+`에서 `[HANDOFF:..]` token pass +4. **Text generator** — `runtime/text-generator.ts:79+`에서 `{status:"handoff", request}` 반환 +5. **Skill.handle** — `makeEmptyResponse + metadata.handoffRequest` 반환 + +Parser: `src/skills/control-token-parser.ts` (regex `/\[HANDOFF:([a-z0-9_-]+):([a-z-]+)\]/giu`). + +### 8.3 HandoffPacket (Iter4 Phase E) + +`src/agent/handoff-packet.ts`. PII 차단 minimal context: + +```ts +{ + summary: string // src/conversation/summarizer.ts 결과 + userGoal: string // 240자 제한, PII 제거 + currentState: Record // phase/responseMode/tier만 (userMessage 원문 미포함) + handoffReason: HandoffReason + evidence: HandoffEvidence[] // toolCalls[].id + constraints: string[] + expectedOutputSchema: string + sourceChain: string[] // 거쳐간 skill 경로 +} +``` + +다음 dispatch hop에서 `request.context.handoffPacket`으로 전달. 수신 skill은 `readHandoffPacket(request)` 호출. + +### 8.4 BudgetTracker (Iter4 Phase F + Iter12 Phase A) + +`src/agent/budget.ts`: + +| 한도 | 기본 | env | +|---|---|---| +| MAX_HANDOFFS | 4 | `AGENT_MAX_HANDOFFS` | +| MAX_SAME_AGENT_VISITS | 2 | `AGENT_MAX_SAME_AGENT_VISITS` | +| MAX_TOTAL_TOOL_CALLS | 20 | `AGENT_MAX_TOOL_CALLS` | +| MAX_COST_USD | 0.5 | `AGENT_MAX_COST_USD` | + +`assertBeforeHandoff(target)` verdict: `"ok" | "max-hops" | "same-agent-loop" | "tool-budget" | "cost-budget"`. + +**Iter12 핵심**: `recordLlmCost(costUsd)` — LlmClient 호출 끝에서 자동 hook (`context.budget?.recordLlmCost(cost)`). snapshot에 `totalLlmCostUsd`, `totalToolCostUsd` 분리 노출. + +### 8.5 Supervisor (Iter4 Phase G + Iter6 + Iter12 Phase A) + +`src/skills/supervisor/skill.ts`. **LLM + 룰 기반 hybrid**: + +``` +1. readHandoffPacket(request) +2. routeByRegex(packet) → { target, confidence } +3. confidence ≥ 0.85 (env: SUPERVISOR_FAST_PATH_CONFIDENCE) + → LLM skip. metadata.routedBy = "regex-fast-path" ← Iter12 G 패턴 +4. 그 외 → routeByLlm(packet): + - capability-prompt에 모든 skill capability card 자동 합성 + - + escalation history (Iter11) [user_escalation_history] block + - tier 0, maxTokens 150, jsonMode + - { target, confidence, reason } JSON 응답 +5. LLM 실패 → 정규식 fallback. metadata.routedBy = "rules-fallback" +6. appendSupervisorReview() → review queue + audit +7. AlertService.sendAlert (NATS + Slack, Iter7) +8. 응답 텍스트: 안정성 위해 template 유지 (Iter6 결정) +``` + +### 8.6 Runtime Evaluator (Iter4 Phase H) + +`SkillJudgeConfig.runtimeVerifier?: (input) => "pass" | "retry" | "escalate"`: + +- pass → 진행 +- retry → 1회 재호출 (`trigger="retry"`, `MAX_RUNTIME_RETRIES=1`) +- escalate → supervisor 강제 handoff (`reason="validation-failed"`) + +Onboarding 기본 verifier: `validateOnboardingResponse`. Moderation: PII/외부 링크 차단 regex. + +--- + +## 9. Cost-aware Tier Selection (Iter12 Phase A) + +자료 G 패턴 + OpenInference cost 활용. + +### 9.1 Tier 구조 + +| Tier | 용도 | 기본 모델 | maxTokens | env | +|---|---|---|---|---| +| 0 | 분류, 짧은 응답, supervisor LLM, reranker | gpt-5.3-codex-spark | 384 | `TIER0_MODEL`, `TIER0_MAX_TOKENS` | +| 1 | planner / responder | gpt-5.4 / gpt-5.3 | 512 | `TIER1_PLANNER_MODEL`, `TIER1_RESPONDER_MODEL` | +| 2 | critic | gpt-5.4 | 512 | `TIER2_CRITIC_MODEL` | + +### 9.2 Router 결정 입력 (Iter12 확장) + +`src/skills/onboarding/runtime/router.ts:decidePreGenerationRouting()`: + +```ts +input: { + classified: { intent, confidence }, + derived, + classifyConfidence: number, // Iter12 + intentRiskTier?: "low"|"medium"|"high", // Iter12 (capability card) + userEscalationCount?: number, // Iter12 (escalation history) + recentLatencyP95Ms?: number // Iter12 (선택) +} +``` + +룰: +- confidence ≥ 0.9 + low-risk → tier 0 (cost 최소) +- confidence < 0.5 또는 high-risk + escalation > 3 → tier 1 (quality 우선) +- p95 latency SLA 위반 추세 → tier 0 (degradation 보호) + +### 9.3 LLM cost 추적 (Iter9 + Iter12) + +`src/llm/llm-client.ts`: + +```ts +const costUsd = estimateChatCost(model, prompt_tokens, completion_tokens); +metrics.tokenCostUsd.observe(costUsd, { model, skill: getContext()?.skill }); +getContext()?.budget?.recordLlmCost(costUsd); // Iter12 +// Phoenix span: gen_ai.usage.cost_usd = costUsd +``` + +`estimateChatCost` (`src/llm/cost.ts`): 모델별 input/output rate 테이블. + +### 9.4 환경 변수 (cost-aware) + +| env | 기본 | 효과 | +|---|---|---| +| `AGENT_COST_AWARE_TIER_ENABLED` | true | confidence 기반 tier 결정 | +| `AGENT_BUDGET_LLM_COST_ENABLED` | true | LlmClient → Budget 자동 hook | +| `AGENT_SUPERVISOR_FAST_PATH_ENABLED` | true | regex 명확 시 LLM skip | +| `SUPERVISOR_FAST_PATH_CONFIDENCE` | 0.85 | 임계값 | + +--- + +## 10. Evaluation 파이프라인 + +자료의 "평가 가능한 실행 환경 없이 데모는 의미 없음" 권고. Iter3에서 skill-aware로 전환됨. + +### 10.1 Golden set 구조 (Iter2 + 후속) + +``` +src/eval/golden/ +├── onboarding/ +│ ├── cases.jsonl # 활성 golden, ~11개 +│ ├── candidate.jsonl # 자동 수집 후보 (promote 후보) +│ └── capture.jsonl # 실시간 prod 캡처 buffer +├── moderation/ +│ └── cases.jsonl # 5개 (Iter2) +├── handoff/ +│ └── cases.jsonl # 6개 (Iter5) +└── memory/ + └── cases.jsonl # 6개 (Iter11, 4축 × 1~2개) +``` + +EvalCase 구조 (`src/eval/types.ts`): + +```ts +{ + id, name, notes, + input: { request, conversationHistory, roomContext, session, blackboard, summary }, + expected: { + intent?, mustIncludeAny?, mustNotInclude?, responseMode?, + processExpected?: { expectedTools, expectedApproach, minSteps, toolMode }, + expectedHandoff?: { shouldHandoff, expectedReason, expectedTarget }, // Iter5 + expectedFinalSkill?: string, + expectedSourceChain?: { mode: "exact"|"prefix"|"contains", chain }, + expectedMemory?: MemoryJudgeExpected // Iter11 + } +} +``` + +### 10.2 Runner 흐름 (`src/eval/runner.ts`) + +``` +1. resolveEvalSetPath(setName) + "onboarding/cases" → src/eval/golden/onboarding/cases.jsonl + "golden" → legacy alias for "onboarding/cases" +2. parseEvalCaseLine() → EvalCase[] +3. for each case: + a. SKILL_RUNNER_FACTORIES[skill]({deps}) → Skill instance + b. handoff/cases set이면 runEvalDispatch (mini-dispatcher, Iter5) + 그 외엔 skill.handle() 직접 호출 + c. judges 실행: + - rule-judge (mustIncludeAny, mustNotInclude, intent) + - llm-judge (skill별 평가 축, SkillJudgeConfig) + - process-judge (tool trace 검증) + - handoff-judge (chain, reason, target) — handoff set만 + - memory-judge (retrieval/long-range/test-time/forgetting) — memory set만 + d. summary 집계 +4. baseline (eval-reports/baseline.{skill}.json) 비교 +5. eval-reports/.json 저장 +``` + +### 10.3 Judges (`src/eval/judges/`) + +| Judge | 책임 | Iter | +|---|---|---| +| `rule-judge.ts` | mustIncludeAny / mustNotInclude / intent 매칭 | Iter3 일반화 | +| `llm-judge.ts` | skill별 평가 축. `SkillJudgeConfig.axes` | Iter3 | +| `process-judge.ts` | tool trace, skill별 tool registry | Iter3 | +| `handoff-judge.ts` | dispatch chain 검증 | Iter5 | +| `memory-judge.ts` | retrieval 정확도, long-range 회상 등 | Iter11 | +| `attribution.ts` | slice 기반 root cause | (이전) | + +`SkillJudgeConfig` (`src/eval/judges/skill-config.ts`): + +```ts +{ + onboarding: { + axes: ["friendliness", "accuracy", "concision", "scope"], + toolRegistryFactory: createOnboardingToolRegistry, + validator: validateOnboardingResponse, + promptTemplate, tournamentInstruction, + runtimeVerifier?: (input) => "pass"|"retry"|"escalate" + }, + moderation: { + axes: ["appropriateness", "accuracy", "helpfulness"], + toolRegistryFactory: createModerationToolRegistry, + ... + } +} +``` + +### 10.4 자동 개선 루프 + +``` +[1] Capture (shadow-eval-bus) — prod 트래픽 sample → shadow envelope + ↓ +[2] Run — golden cases 실행, judges 적용, eval-reports/.json + ↓ +[3] Propose (eval:propose CLI) — 저점수 slice → LLM이 prompt variant 생성 + → src/eval/proposals/.json + ↓ +[4] Tournament (eval:tournament) — control vs variant A/B, ELO 갱신 + ↓ +[5] Governance gates (G1~G4) — trust tier 검증 + ↓ +[6] Promote (eval:promote) — golden 후보 → 활성 cases +``` + +### 10.5 변형 적용 (PROMPT_VARIANT_ID) + +`src/skills/onboarding/prompts/builder.ts`: + +- `loadPromptVariantManifest(variantId)` → variant JSON 로드 +- `applyLayerDiff(layers, diff)` — `persona/policy/state/context/examples/tool-policy` 6 layer 중 일부 replace/append/remove +- env `PROMPT_VARIANT_ID=` → 프로덕션에서 활성 + +### 10.6 Calibration / Drift / Governance + +- `src/eval/aggregator.ts` — SliceMetrics (intent/phase/tier 차원) +- `src/eval/drift.ts` — baseline 대비 점수 하락 감지 +- `src/eval/governance/trust.ts` — T1~T4 enum +- `src/eval/governance/gates.ts` — variant 승격 4 gate (G1~G4) +- `src/eval/router-shadow-report.ts` (Iter6) — learned vs static routing divergence + +--- + +## 11. Memory 평가 (Iter11) + +`src/eval/golden/memory/cases.jsonl` 6 cases: + +| # | 이름 | 검증 | +|---|---|---| +| 1 | retrieval-accurate | "광고 도배" 질문 → moderation lookup-rule retrieval 정확 chunk | +| 2 | long-range-understanding | 5턴 전 "닉네임 LolName_KR" → 회상 | +| 3 | test-time-learning | flaggedIssues 3회 누적 → prophylactic guidance | +| 4 | selective-forgetting | 신규 doc 30% 차이 + "정정" 키워드 → 기존 archive | +| 5 | escalation-history | 사용자 3회 광고 시도 → supervisor가 moderation 우선 라우팅 | +| 6 | memory-contradiction | "닉네임 X로 변경" → profile 반영, 다음 응답이 새 닉네임 인지 | + +`MemoryJudgeResult` (`src/eval/judges/memory-judge.ts`): +- `expectedRetrievedDocIds`, `expectedRecalledFact`, `expectedProphylacticHint`, `expectedArchivedDocPattern`, `expectedSupervisorTarget`, `expectedContradictionResolved` + +명령: `npm run eval -- --set=memory/cases`. + +--- + +## 12. Operations — 운영 루프 + +### 12.1 Review Queue 두 종류 + +| Queue | Garnet stream | 목적 | 진입 | +|---|---|---|---| +| Eval Review | `lolympus-agent:eval:review:queue` | 학습용 (저점수/희귀 slice/rule-llm 불일치) | runner가 자동 enqueue | +| Supervisor Handoff | `lolympus-agent:review-queue` | 운영용 (escalation case 사람 검토) | supervisor.skill.ts | + +### 12.2 Operations CLI (Iter5 + Iter12) + +`src/operations/cli.ts`: + +```bash +# Supervisor handoff queue +npm run review:list -- --limit=20 --since=24h +npm run review:show -- [--history] # --history는 Iter6 audit log +npm run review:resolve -- --action=manual-reply --note="..." +npm run review:stats -- --since=7d + +# Tool audit (Iter12) +npm run tool-audit:list -- --skill=onboarding --since=1h --limit=50 +npm run tool-audit:show -- +``` + +### 12.3 Resolution audit log (Iter6) + +이중 저장: +- **Hash** `lolympus-agent:review-resolved:` — latest state (덮어쓰기, 7d TTL) +- **Stream** `lolympus-agent:review-resolutions` — append-only (운영자 변경 이력) + +`getResolutionHistory(garnet, id)` — 사용자별 resolution 변경 이력 조회 (Iter11 escalation history 입력). + +### 12.4 Alert flow (Iter7) + +`src/observability/alert.ts:AlertService.sendAlert(payload)`: + +``` +1. Garnet dedupe key 검사 (5분 throttle) + key: lolympus-agent:alert:dedupe:: +2. NATS publish → AGENT_ALERT_SUBJECT +3. Slack webhook POST (env: SLACK_ALERT_WEBHOOK_URL) + - fire-and-forget, AbortSignal.timeout 3초 + - block kit message (severity emoji + 필드) +4. logger.warn (구조화 로그) +``` + +AlertPayload: +```ts +{ + kind: "archive-stalled"|"endpoint-down"|"eval-drift"|"eval-score-drop" + |"latency"|"llm-error-rate"|"supervisor-handoff", + metric, severity, skill?, threshold, value, window, timestamp, ... +} +``` + +### 12.5 Slack message 포맷 (Iter7) + +`src/observability/slack-notifier.ts:formatSlackMessage()`: +- severity emoji (`:information_source:`/`:warning:`/`:rotating_light:`) +- Section + fields (metric / value vs threshold / window / skill) +- suggestedAction (있다면) + +--- + +## 13. Observability + +### 13.1 Logger (`src/observability/logger.ts`) + +- Pino + AsyncLocalStorage +- `getLogger()` → context의 child logger 자동 (skill, requestId, chatroomId, userId) +- redact: `authorization`, `env.`, `secrets.`, `request.` (KNOWN_SECRET_KEYS 확장됨, Iter12) + +### 13.2 Tracing (Iter9) + +`src/observability/tracing.ts`: +- OTel NodeSDK + auto-instrumentation +- Exporter: env `PHOENIX_OTLP_ENDPOINT` 설정 시 OTLP HTTP, 미설정 시 ConsoleSpanExporter (또는 OFF, env: `OTEL_EXPORTER`) +- Sampling: `OTEL_SAMPLING_RATIO` (prod 0.1, dev 1.0) + +`src/observability/spans.ts` helper: +- `withSkillSpan(skill, attrs, fn)` — `skill..handle` +- `withSpan(name, attrs, fn)` — generic + auto-error +- `withClassifierSpan`, `withToolSpan`, `withLlmSpan` + +### 13.3 LLM Span 속성 (Iter9 OpenInference) + +`src/llm/llm-client.ts`: +``` +gen_ai.request.model +gen_ai.request.temperature +gen_ai.request.max_tokens +gen_ai.system (provider name) +gen_ai.response.id +gen_ai.response.finish_reasons[] +gen_ai.usage.input_tokens +gen_ai.usage.output_tokens +gen_ai.usage.cost_usd ← Iter9, Iter12에서 routing 활용 +llm.token_count.prompt ← Phoenix 친화 alias +llm.token_count.completion +gen_ai.prompt.0.role/content ← optional (PHOENIX_LOG_PROMPTS) +gen_ai.completion.0.content ← optional + PII 마스킹 +``` + +Memory 관련 (Iter10-11): +- `memory.retrieval.candidate_count` +- `memory.retrieval.top_score` +- `memory.retrieval.duration_ms` +- `memory.reranker.changed_order` + +Handoff 관련: +- `agent.handoff_depth` +- `agent.handoff_chain` +- `supervisor.routed_by` + +### 13.4 Metrics (Iter4 Phase I) + +자체 구현 (`src/observability/metrics.ts`). `/metrics` HTTP endpoint (`metrics-server.ts`, port 기본 9464). 10 counter/histogram: + +| Metric | 타입 | Labels | +|---|---|---| +| `agent_handoff_total` | counter | from, to, reason | +| `agent_handoff_loop_total` | counter | from | +| `agent_handoff_count` | histogram (0..4) | — | +| `agent_human_override_total` | counter | kind | +| `agent_missed_escalation_total` | counter | — | +| `agent_routing_accuracy_total` | counter | result | +| `agent_unnecessary_escalation_total` | counter | — | +| `agent_validator_fail_total` | counter | skill, stage | +| `agent_token_cost_usd` | histogram | model, skill | +| `agent_time_to_resolution_ms` | histogram | skill | + +`/healthz` endpoint: 200 "ok" (Iter7). + +### 13.5 Alert threshold (skill별, Iter7) + +`src/config.ts`: +``` +getAlertThresholdP95LatencyMs(skill?: string) +getAlertThresholdEvalScoreDropPct(skill?: string) +getAlertThresholdLlmErrorRate(skill?: string) +getAlertThresholdDriftPct(skill?: string) +``` + +env 패턴: `ALERT_THRESHOLD__` 우선 → `ALERT_THRESHOLD_` fallback. + +예: `ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING=5000` 또는 글로벌 `ALERT_THRESHOLD_P95_LATENCY_MS=3000`. + +--- + +## 14. 보안 + +### 14.1 Secret Manager (`src/common/secret-manager.ts`) + +계층: +1. **Infisical** (외부 secret manager, `INFISICAL_SITE_URL`) +2. **AWS IMDS** (EC2 메타데이터 fallback) +3. **환경변수** (마지막 fallback) + +`KNOWN_SECRET_KEYS` (`src/common/known-secret-keys.ts`, Iter12 확장): +``` +INFISICAL_*, NATS_*, GARNET_*, MONGODB_*, SLACK_ALERT_WEBHOOK_URL, +OPENROUTER_API_KEY, AGENT_LLM_API_KEY, OPENAI_API_KEY, +EMBEDDING_API_KEY, PHOENIX_API_KEY +``` + +`getSecret(key)`: KNOWN_SECRET_KEYS 검증. `getRawSecret(key)`: 임의 키 (운영 도구용). + +### 14.2 NATS Subject ACL (Iter12 Phase C) + +`src/common/nats-skill-acl.ts:DEFAULT_NATS_ACL`: +```ts +[ + { + skill: "onboarding", + allowedPublishSubjects: ["lolympus.discord.lookup-members"], + allowedSubscribeSubjects: [] + }, + { skill: "moderation", allowedPublishSubjects: [], ... }, + { skill: "supervisor", allowedPublishSubjects: ["lolympus.agent.alert"], ... } +] +``` + +`src/common/nats.ts`의 publish wrapper: +- `getNatsAclMode()`: + - `"off"`: 검사 skip + - `"log-only"` (기본): warn 로그, 정상 publish + - `"enforce"`: throw + alert + +운영 흐름: log-only 1개월 → false positive 없으면 enforce 전환 (Iter13+ task). + +### 14.3 PII 차단 + +- **HandoffPacket**: `userMessage` 원문 포함 X. `userGoal`은 240자 + anonymize +- **Memory user-profile**: `anonymizeText()` 처리 +- **Tool audit**: `argsHash`만 저장, 원본 args X +- **Phoenix prompt content**: `PHOENIX_LOG_PROMPTS=false` 기본 +- **escalation history**: supervisor system prompt 내부만, 응답 텍스트에 노출 금지 + +--- + +## 15. 환경 변수 + 배포 + +### 15.1 인프라 + +| env | 용도 | +|---|---| +| `NATS_ENDPOINT`, `NATS_USER`, `NATS_PASS` | NATS 연결 | +| `GARNET_ENDPOINT`, `GARNET_USERID`, `GARNET_PASSWORD` | Garnet 연결 | +| `MONGODB_CONNECTION_STRING`, `MONGODB_PROJECT_ID` | MongoDB (optional) | +| `INFISICAL_CLIENT_ID`, `INFISICAL_CLIENT_SECRET` | Secret manager | + +### 15.2 모델 + +| env | 기본 | 용도 | +|---|---|---| +| `TIER0_MODEL` | gpt-5.3-codex-spark | 분류, supervisor LLM, reranker | +| `TIER1_PLANNER_MODEL`, `TIER1_RESPONDER_MODEL` | gpt-5.4, gpt-5.3 | 메인 응답 | +| `TIER2_CRITIC_MODEL` | gpt-5.4 | critic | +| `TIER0_MAX_TOKENS` | 384 | | +| `EMBEDDING_PROVIDER` | openrouter | OpenRouter→OpenAI fallback | +| `EMBEDDING_MODEL` | text-embedding-3-small | | +| `EMBEDDING_API_KEY` | — | (또는 OPENROUTER/OPENAI 키 재사용) | + +### 15.3 기능 flag (Iter별) + +| env | 기본 | Iter | +|---|---|---| +| `AGENT_ROUTER_MODE` | static | Iter3 | +| `AGENT_RAG_ENABLED` | true | Iter10 | +| `AGENT_RAG_HYBRID_ENABLED` | true | Iter11 | +| `AGENT_RAG_RERANKER_ENABLED` | true | Iter11 | +| `AGENT_RAG_CANDIDATE_POOL_SIZE` | 50 | Iter11 | +| `AGENT_RAG_RERANKER_TOP_K` | 5 | Iter11 | +| `AGENT_LONG_TERM_SUMMARY_ENABLED` | true | Iter11 | +| `AGENT_MEMORY_EVICTION_ENABLED` | false (cron만) | Iter11 | +| `AGENT_COST_AWARE_TIER_ENABLED` | true | Iter12 | +| `AGENT_BUDGET_LLM_COST_ENABLED` | true | Iter12 | +| `AGENT_SUPERVISOR_FAST_PATH_ENABLED` | true | Iter12 | +| `AGENT_TOOL_SANDBOX_ENABLED` | false (1주 검증 후 true) | Iter12 | +| `AGENT_NATS_ACL_MODE` | log-only | Iter12 | +| `AGENT_SUPERVISOR_LLM_ENABLED` | true | Iter6 | +| `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | false (template 우선) | Iter8 | + +### 15.4 한도 (Budget) + +| env | 기본 | +|---|---| +| `AGENT_MAX_HANDOFFS` | 4 | +| `AGENT_MAX_SAME_AGENT_VISITS` | 2 | +| `AGENT_MAX_TOOL_CALLS` | 20 | +| `AGENT_MAX_COST_USD` | 0.5 | + +### 15.5 임계값 (Alert) + +| env (skill suffix optional) | 기본 | +|---|---| +| `ALERT_THRESHOLD_P95_LATENCY_MS` | 3000 | +| `ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT` | 10 | +| `ALERT_THRESHOLD_LLM_ERROR_RATE` | 0.1 | +| `ALERT_THRESHOLD_DRIFT_PCT` | 15 | +| `ALERT_RATE_LIMIT_SEC` | 300 | + +### 15.6 Observability + +| env | 기본 | +|---|---| +| `PHOENIX_OTLP_ENDPOINT` | unset (console fallback) | +| `PHOENIX_LOG_PROMPTS` | false | +| `SLACK_ALERT_WEBHOOK_URL` | unset (Slack 비활성) | +| `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | +| `METRICS_PORT` | 9464 | +| `OTEL_SAMPLING_RATIO` | 0.1 (prod) | + +### 15.7 Bootstrap (`src/main.ts`) + +순서: +``` +1. initLogger() +2. initTracing() (Phoenix 또는 console) +3. startMetricsServer() (port 9464) +4. SecretManager.load (Infisical + IMDS) +5. NatsClient.connect +6. GarnetClient.connect +7. initMongoDB (optional) +8. SkillRegistry.load("src/skills") +9. TransitionMatrix.fromRegistry +10. ConversationStore + ShadowEvalBus +11. memoryService, escalationHistoryService, alertService (Slack notifier) +12. factories map { onboarding, moderation, supervisor } +13. for manifest in registry: agent.registerSkill(factory(deps)) +14. agent.start() (NATS subscribe) +15. shutdown hooks +``` + +--- + +## 16. 도메인 데이터 (Lolympus 카카오톡 오픈채팅) + +### 16.1 Onboarding (`src/skills/onboarding/`) + +**Capability card**: +- priority: 70, trustTier: T3, riskTier: low +- domain: onboarding +- handoffTargets: [moderation, supervisor] + +**Intent** (`intents.ts`, 12개): +- greeting, acknowledgement +- status-question +- help-how, help-where +- nickname-question, photo-question +- discord-question, discord-verify-question +- age-question +- chitchat, off-topic-to-other, unknown + +**Tools** (5개): validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname + +**Resources** (4개 .md): nickname-format, heart-rule, discord-format, age-policy + +**핵심 로직**: +- `OnboardingIntentClassifier` (3-stage: trigger → regex → LLM) +- `deriveState(session, trigger, intent)` → phase + responseMode + tier +- `runtime/`: orchestrator, router, validator, critic, tool-plan-runner, message-builder, text-generator, abstention 등 ~10개 모듈 + +### 16.2 Moderation (`src/skills/moderation/`) + +**Capability card**: +- priority: 50, trustTier: T4, riskTier: medium +- domain: moderation +- handoffTargets: [onboarding, supervisor] + +**Intent** (5개): greeting, rule-question, violation-report, appeal, off-topic + +**Tools** (1개): lookup-rule (Iter10에서 RAG 기반으로 전환) + +**Resources** (1개 .md): community-rules (욕설, 광고, 도배, 신고 절차, 제재, 이의제기, 미성년자, 개인정보 등 섹션) + +### 16.3 Supervisor (`src/skills/supervisor/`) + +**Capability card**: +- priority: 10 (최저, fallback 전용), riskTier: medium +- domain: meta +- canHandle: ["*"] +- handoffTargets: [onboarding, moderation] + +**핵심**: +- `capability-prompt.ts` — 모든 skill의 capability card를 system prompt로 자동 합성 +- `routing-rules.ts` — regex 기반 키워드 매칭 (Iter12 fast-path) +- `review-queue.ts` — appendSupervisorReview + getResolutionHistory + 도메인 escalation history 연동 +- 응답 텍스트: 안정성 위해 template ("운영자에게 전달했어요") + +### 16.4 User Escalation History (Iter11 — lolympus 차별점) + +`user_escalation_history` collection (MongoDB): + +```ts +{ + userId, chatroomId, + escalations: [{ + queuedAt, reason, target, + resolution: { resolvedAt, action, note }, + packetSummary, sourceChain + }], + totalEscalations, lastEscalationAt, + topReasons: [{ reason, count }] +} +``` + +Supervisor가 매 escalation 시 history 조회 → capability prompt의 `[user_escalation_history]` 블록 주입: +> "이 사용자는 광고 시도 3회 (out-of-scope) → 다음 위반 시 즉시 차단 권장" +> "닉네임 검증 5회 실패 → 친절한 step-by-step 우선" + +--- + +## 17. 명령어 cheat sheet + +### 17.1 개발 + +```bash +npm run typecheck # TypeScript 검증 +npm run test # Vitest 전체 +npm run test -- --reporter=verbose # 상세 +npm run lint # ESLint +npm run lint:skills # SKILL.md ↔ 코드 일관성 +npm run build # tsc + tsc-alias +npm run dev # 로컬 부트스트랩 +``` + +### 17.2 Evaluation + +```bash +# Run +npm run eval -- --set=onboarding/cases +npm run eval -- --set=moderation/cases +npm run eval:handoff # = --set=handoff/cases +npm run eval -- --set=memory/cases # Iter11 + +# Baseline / Report +npm run eval:baseline +npm run eval:report + +# 자동 개선 loop +npm run eval:capture +npm run eval:propose -- --slice="tier=0,intent=help-how,phase=initial,skill=onboarding" +npm run eval:tournament -- --skill=onboarding +npm run eval:promote +npm run eval:calibrate -- --check +npm run eval:router-shadow-report -- --since=7d # Iter6 +``` + +### 17.3 Operations (Iter5 + Iter12) + +```bash +# Supervisor review queue +npm run review:list +npm run review:show -- +npm run review:show -- --history # Iter6 resolution audit +npm run review:resolve -- --action=manual-reply --note="..." +npm run review:stats + +# Tool audit +npm run tool-audit:list +npm run tool-audit:show -- +``` + +### 17.4 Memory (Iter10-11) + +```bash +npm run ingest:knowledge # resources/*.md → MongoDB embedding +npm run memory:evict -- --dry-run --since=30d +npm run memory:evict -- --since=30d # 실제 archive +``` + +### 17.5 디버깅 + +```bash +PHOENIX_OTLP_ENDPOINT=http://localhost:6006/v1/traces npm run dev +curl http://localhost:9464/metrics | grep agent_ +curl http://localhost:9464/healthz + +# Specific log level +LOG_LEVEL=debug npm run dev + +# RAG/sandbox 비활성으로 회귀 확인 +AGENT_RAG_ENABLED=false npm run eval -- --set=onboarding/cases +AGENT_TOOL_SANDBOX_ENABLED=false npm test +``` + +--- + +## 18. 참고 자료 + +### 18.1 Iter별 plan 문서 + +`docs/`에 dev 브랜치 보존 (main에는 미포함): + +| 문서 | 내용 | +|---|---| +| `refactor-skill-ssot.md` | Iter1 plan | +| `refactor-skill-ssot-iter2~8.md` | Iter2~8 각 plan | +| `optimization-roadmap.md` | Iter8 closing 시 trigger 정의 | +| `refactor-skill-ssot-iter10~12.md` | Iter10~12 plan | +| `optimization-roadmap-memory-ops.md` | Iter9~12 통합 로드맵 | +| `service-evaluation-operations.md` | 운영자 매뉴얼 | +| `최신기술동향.md`, `멀티에이전트동향.md` | 자료 reference | + +### 18.2 외부 자료 + +- **MemoryAgentBench** — accurate retrieval, test-time learning, long-range understanding, selective forgetting +- **Bounded Self-Escalation Graph** — Microsoft orchestration guide +- **OpenInference semantic conventions** — `gen_ai.*`, `llm.*` span attributes +- **MCP / A2A** — 향후 채택 후보 (Iter13+ trigger 기반) + +### 18.3 다음 단계 trigger (Iter13+) + +`docs/optimization-roadmap.md`의 trigger 발동 시 개별 plan으로 시작: + +| 항목 | Trigger | +|---|---| +| NATS ACL enforce 전환 | log-only 1개월 안정 + false positive 0 | +| Tool sandbox enforce 전환 | log-only 1주 안정 | +| MongoDB Vector Search 마이그레이션 | preview → GA | +| Resolution hash sunset | Iter6 dual-write 2개월 안정 | +| Cross-room context aggregation | cross-room 패턴 발견 | +| Parallel fan-out | handoff 평균 hop ≥ 2.0 | +| Web UI for review queue | review 진입 ≥ N/day 1개월 | +| Workflow auto-discovery (AFlow) | propose/tournament 안정화 | +| MCP/A2A 채택 | 외부 ecosystem 연결 필요 | + +각 항목은 evidence(metric/운영 사례) 기반으로 plan 작성. + +--- + +## 19. 변경 이력 (commit 별) + +main 브랜치의 주요 milestones: + +| sha | 메시지 | Iter | +|---|---|---| +| `bc0f8e0` | Add secret coverage and NATS subject ACL | 12 Phase C | +| `d011a28` | Add tool permission sandbox and audit stream | 12 Phase B | +| `21df14c` | Add cost-aware routing and LLM budget tracking | 12 Phase A | +| `211de18` | Add hybrid memory retrieval and escalation history | 11 | +| `1a86bde` | Add memory layers and RAG retrieval | 10 | +| `1d0be56` | Add Phoenix OTLP tracing and LLM span attributes | 9 | +| (이전) | Close multi-agent plan and add router shadow reporting | 8 | +| (이전) | Add Slack alerts and skill-aware monitoring | 7 | +| (이전) | Unify dispatch and add supervisor routing signals | 6 | +| (이전) | Close supervisor review loop and add handoff eval | 5 | +| (이전) | Make eval pipeline skill-aware and validate SKILL manifests | 3 | + +각 commit의 detail은 `git log -p` 또는 dev 브랜치의 `docs/refactor-skill-ssot-iterN.md` 참고. + +--- + +## 끝 + +이 문서가 outdated되면 last-verified commit을 footer에 명시: +- last-verified: `bc0f8e0` (2026-05-19) + +업데이트 책임: 신규 큰 변경(새 skill 추가, 새 memory layer, 새 metric 등) PR 시 함께 갱신. diff --git a/launch-server.ps1 b/launch-server.ps1 index 472bc05..fee207e 100644 --- a/launch-server.ps1 +++ b/launch-server.ps1 @@ -16,20 +16,36 @@ if ($isMacOS -or $isLinux) { $env:ENV_FILE = "$PSScriptRoot\env-vars\$env:NODE_ENV.env" } -# 환경 변수 파일 확인 -if (-Not (Test-Path $env:ENV_FILE)) { - Write-Error "The file '$env:ENV_FILE' does not exist." - exit 1 -} +$GLOBAL_ENV_FILE = Join-Path $HOME "source/global-env-vars.txt" -# 'dahyeon-local.env' 파일에서 환경 변수 설정 -Get-Content $env:ENV_FILE | ForEach-Object { - if ($_ -match '^\s*([^#]+?)\s*=\s*(.*)$') { - $key, $value = $matches[1].Trim(), $matches[2].Trim().Trim("`"") - [System.Environment]::SetEnvironmentVariable($key, $value) +function Import-EnvFileIfPresent { + param( + [string]$Path, + [switch]$Required + ) + + if (-Not (Test-Path $Path)) { + if ($Required) { + Write-Error "The file '$Path' does not exist." + exit 1 + } + return + } + + Get-Content $Path | ForEach-Object { + if ($_ -match '^\s*([^#]+?)\s*=\s*(.*)$') { + $key, $value = $matches[1].Trim(), $matches[2].Trim().Trim("`"") + if (-not [System.Environment]::GetEnvironmentVariable($key)) { + [System.Environment]::SetEnvironmentVariable($key, $value) + } + } } } +# 환경 변수 파일 확인 +Import-EnvFileIfPresent -Path $GLOBAL_ENV_FILE +Import-EnvFileIfPresent -Path $env:ENV_FILE -Required + Get-ChildItem Env: | Where-Object { $_.Name -eq "NODE_ENV" -or $_.Name -like "IMDS_*" -or diff --git a/src/common/env-loader.ts b/src/common/env-loader.ts index e3ddcde..2181dbc 100644 --- a/src/common/env-loader.ts +++ b/src/common/env-loader.ts @@ -1,4 +1,5 @@ import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import * as dotenv from "dotenv"; @@ -34,6 +35,11 @@ function resolveEnvFile(mode: NodeEnvMode): string | null { return null; } +function resolveGlobalEnvFile(): string | null { + const candidate = path.resolve(os.homedir(), "source", "global-env-vars.txt"); + return fs.existsSync(candidate) ? candidate : null; +} + function loadDotenvFile(filePath: string, label: string): void { const result = dotenv.config({ path: filePath, override: false }); if (result.error) { @@ -48,6 +54,11 @@ export function loadEnvironmentFromNodeEnv(): void { const mode = normalizeMode(process.env["NODE_ENV"]); process.env["NODE_ENV"] = mode; + const globalEnvFile = resolveGlobalEnvFile(); + if (globalEnvFile) { + loadDotenvFile(globalEnvFile, "global-env-vars.txt"); + } + const envFile = resolveEnvFile(mode); if (!envFile) { logger.warn(`[Env] env-vars/${mode}.env not found. Skipping dotenv load.`); diff --git a/src/common/garnet-cli.ts b/src/common/garnet-cli.ts index b141f57..730fbdd 100644 --- a/src/common/garnet-cli.ts +++ b/src/common/garnet-cli.ts @@ -1,9 +1,10 @@ -import { GarnetClient } from "./garnet.js"; +import type { GarnetClient as GarnetClientType } from "./garnet.js"; export async function connectGarnetForCli(): Promise<{ close: () => Promise; - garnetClient: GarnetClient; + garnetClient: GarnetClientType; }> { + const { GarnetClient } = await import("./garnet.js"); const garnetClient = new GarnetClient(); await garnetClient.connect(); return { diff --git a/src/common/nats-subjects.ts b/src/common/nats-subjects.ts index 4e2b82f..13497d7 100644 --- a/src/common/nats-subjects.ts +++ b/src/common/nats-subjects.ts @@ -34,6 +34,12 @@ export type DiscordMemberMatchSource = | "username" | "joined-after"; +export type DiscordResidentApplicationMessageMatchSource = + | "author-id" + | "author-name" + | "content" + | "joined-after"; + export interface DiscordMemberCandidate { memberId: string; joinedAt: string; @@ -43,8 +49,20 @@ export interface DiscordMemberCandidate { username: string; } +export interface DiscordResidentApplicationMessageCandidate { + authorDisplayName?: string | undefined; + authorGuildNickname?: string | undefined; + authorId: string; + channelId: string; + content: string; + createdAt: string; + matchedBy?: DiscordResidentApplicationMessageMatchSource | undefined; + messageId: string; +} + export interface DiscordMemberLookupRequest { chatroomId: string; + includeResidentApplicationMessages?: boolean | undefined; joinedAfter: string; searchName?: string | undefined; userId: string; @@ -55,6 +73,9 @@ export type DiscordMemberLookupResponse = status: "ok"; joinedAfterMatches: DiscordMemberCandidate[]; nicknameMatches: DiscordMemberCandidate[]; + residentApplicationMessageMatches?: + | DiscordResidentApplicationMessageCandidate[] + | undefined; } | { status: "error"; diff --git a/src/config.test.ts b/src/config.test.ts index 7db3c31..6c3a619 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -11,7 +11,12 @@ import { getAgentToolSandboxEnabled, getEmbeddingModel, getEmbeddingProvider, + getEvalCaseDeadlineMs, + getEvalLlmProvider, + getEvalRuntimeLlmEnabled, getLongTermSummaryEnabled, + getLlmModelRoundRobinEnabled, + getRuntimeLlmModelsForProvider, getMemoryEvictionEnabled, getNatsAclMode, getPhoenixLogPrompts, @@ -56,6 +61,12 @@ const ENV_KEYS = [ "SLACK_ALERT_WEBHOOK_URL", "SLACK_WEBHOOK_TIMEOUT_MS", "AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", + "CLIPROXY_TIER1_PLANNER_MODEL", + "EVAL_CASE_DEADLINE_MS", + "EVAL_LLM_PROVIDER", + "EVAL_RUNTIME_LLM_ENABLED", + "LLM_MODEL_ROUND_ROBIN_ENABLED", + "OPENROUTER_TIER1_RESPONDER_MODEL", ] as const; describe("config", () => { @@ -169,4 +180,71 @@ describe("config", () => { expect(getToolAuditMaxLen()).toBe(42); expect(getNatsAclMode()).toBe("enforce"); }); + + it("reads eval case deadline configuration", () => { + expect(getEvalCaseDeadlineMs()).toBe(4_500); + + vi.stubEnv("EVAL_CASE_DEADLINE_MS", "45000"); + + expect(getEvalCaseDeadlineMs()).toBe(45_000); + }); + + it("enables eval runtime LLM by default with env override", () => { + expect(getEvalRuntimeLlmEnabled()).toBe(true); + + vi.stubEnv("EVAL_RUNTIME_LLM_ENABLED", "false"); + + expect(getEvalRuntimeLlmEnabled()).toBe(false); + }); + + it("pins eval LLM provider to cliproxy by default", () => { + expect(getEvalLlmProvider()).toBe("cliproxy"); + + vi.stubEnv("EVAL_LLM_PROVIDER", "all"); + expect(getEvalLlmProvider()).toBe("all"); + + vi.stubEnv("EVAL_LLM_PROVIDER", "openrouter"); + expect(getEvalLlmProvider()).toBe("openrouter"); + + vi.stubEnv("EVAL_LLM_PROVIDER", "invalid"); + expect(getEvalLlmProvider()).toBe("cliproxy"); + }); + + it("reads provider-scoped runtime LLM models", () => { + vi.stubEnv("CLIPROXY_TIER1_PLANNER_MODEL", "cliproxy-planner"); + vi.stubEnv( + "OPENROUTER_TIER1_RESPONDER_MODEL", + "openrouter-fast, openrouter-backup", + ); + + expect( + getRuntimeLlmModelsForProvider( + "cliproxy", + { role: "planner", tier: 1 }, + "local-qwen", + ), + ).toEqual(["cliproxy-planner"]); + expect( + getRuntimeLlmModelsForProvider( + "openrouter", + { role: "responder", tier: 1 }, + "local-qwen", + ), + ).toEqual(["openrouter-fast", "openrouter-backup"]); + expect( + getRuntimeLlmModelsForProvider( + "llama", + { role: "responder", tier: 1 }, + "local-qwen", + ), + ).toEqual(["local-qwen"]); + }); + + it("keeps model round-robin disabled by default", () => { + expect(getLlmModelRoundRobinEnabled()).toBe(false); + + vi.stubEnv("LLM_MODEL_ROUND_ROBIN_ENABLED", "true"); + + expect(getLlmModelRoundRobinEnabled()).toBe(true); + }); }); diff --git a/src/config.ts b/src/config.ts index c17c8ad..ace3c35 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,6 +77,23 @@ function getBoolean(envKey: string, fallback: boolean): boolean { return fallback; } +function getStringListFromEnv(...envKeys: string[]): string[] | undefined { + for (const envKey of envKeys) { + const raw = process.env[envKey]?.trim(); + if (!raw) continue; + + const values = raw + .split(/[\n,]/u) + .map((value) => value.trim()) + .filter((value) => value.length > 0); + if (values.length > 0) { + return values; + } + } + + return undefined; +} + export function getNodeEnv(): "dev" | "test" | "prod" { const value = process.env["NODE_ENV"]?.trim(); return value === "test" || value === "prod" ? value : "dev"; @@ -390,6 +407,23 @@ export function getSupervisorLlmResponseEnabled(): boolean { return getBoolean("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", true); } +export type EvalLlmProvider = "all" | "cliproxy" | "llama" | "openrouter"; + +const EVAL_LLM_PROVIDERS = new Set([ + "all", + "cliproxy", + "llama", + "openrouter", +]); + +export function getEvalLlmProvider(): EvalLlmProvider { + const raw = process.env["EVAL_LLM_PROVIDER"]?.trim().toLowerCase(); + if (raw && EVAL_LLM_PROVIDERS.has(raw as EvalLlmProvider)) { + return raw as EvalLlmProvider; + } + return "cliproxy"; +} + export function getEvalCalibrationMinCorr(): number { return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5); } @@ -515,6 +549,10 @@ export function getLlmCircuitCooldownMs(): number { return getPositiveNumber("LLM_CIRCUIT_COOLDOWN_MS", 60_000); } +export function getLlmModelRoundRobinEnabled(): boolean { + return getBoolean("LLM_MODEL_ROUND_ROBIN_ENABLED", false); +} + export function getClassifierMaxTokens(): number { return getPositiveNumber("CLASSIFIER_MAX_TOKENS", 16); } @@ -551,6 +589,14 @@ export function getEvalPrmMinLabels(): number { return getPositiveNumber("EVAL_PRM_MIN_LABELS", 500); } +export function getEvalCaseDeadlineMs(): number { + return getPositiveNumber("EVAL_CASE_DEADLINE_MS", 4_500); +} + +export function getEvalRuntimeLlmEnabled(): boolean { + return getBoolean("EVAL_RUNTIME_LLM_ENABLED", true); +} + export function getEvalAttributionTickMs(): number { return getPositiveNumber("EVAL_ATTRIBUTION_TICK_MS", 6 * 60 * 60 * 1000); } @@ -606,15 +652,91 @@ export function getTier2CriticModel(): string { } export function getOpenRouterModels(): string[] { - const raw = process.env["OPENROUTER_MODEL"]?.trim(); - if (!raw) { - return ["openai/gpt-5.3-codex-spark"]; + return ( + getStringListFromEnv("OPENROUTER_MODEL", "OPENROUTER_MODELS") ?? [ + "openai/gpt-5.3-codex-spark", + ] + ); +} + +type RuntimeLlmProvider = "cliproxy" | "llama" | "openrouter"; +type RuntimeLlmRole = "critic" | "planner" | "responder" | undefined; +type RuntimeLlmTier = 0 | 1 | 2 | undefined; + +function getRuntimeRouteSuffix(args: { + role?: RuntimeLlmRole; + tier?: RuntimeLlmTier; +}): string { + if (args.role === "planner") { + return "TIER1_PLANNER"; } - const models = raw - .split(",") - .map((value) => value.trim()) - .filter((value) => value.length > 0); - return models.length > 0 ? models : ["openai/gpt-5.3-codex-spark"]; + if (args.role === "critic") { + return "TIER2_CRITIC"; + } + if (args.tier === 0) { + return "TIER0"; + } + return "TIER1_RESPONDER"; +} + +function getProviderPrefix(provider: RuntimeLlmProvider): string { + return provider.toUpperCase(); +} + +function getProviderDefaultModels( + provider: RuntimeLlmProvider, + llamaFallbackModel: string, +): string[] { + if (provider === "cliproxy") { + return ( + getStringListFromEnv( + "CLIPROXY_MODEL", + "CLIPROXY_MODELS", + "CLI_PROXY_LLM_MODEL", + ) ?? ["gpt-5.3-codex-spark"] + ); + } + + if (provider === "openrouter") { + return getOpenRouterModels(); + } + + return ( + getStringListFromEnv("LLAMA_MODEL", "LLAMA_MODELS", "AGENT_LLM_MODEL") ?? [ + llamaFallbackModel, + ] + ); +} + +export function getRuntimeLlmModelsForProvider( + provider: RuntimeLlmProvider, + args: { + role?: RuntimeLlmRole; + tier?: RuntimeLlmTier; + }, + llamaFallbackModel: string, +): string[] { + const suffix = getRuntimeRouteSuffix(args); + const providerPrefix = getProviderPrefix(provider); + const providerScopedModels = getStringListFromEnv( + `${providerPrefix}_${suffix}_MODEL`, + `${providerPrefix}_${suffix}_MODELS`, + ); + if (providerScopedModels) { + return providerScopedModels; + } + + if (provider === "cliproxy") { + const legacyModels = getStringListFromEnv( + `${suffix}_MODEL`, + `${suffix}_MODELS`, + ); + if (legacyModels) { + return legacyModels; + } + } + + return getProviderDefaultModels(provider, llamaFallbackModel); } export function getLlmMaxTokens(): number { diff --git a/src/eval/golden/onboarding/candidate.jsonl b/src/eval/golden/onboarding/candidate.jsonl index 882c596..4fd4df6 100644 --- a/src/eval/golden/onboarding/candidate.jsonl +++ b/src/eval/golden/onboarding/candidate.jsonl @@ -138,3 +138,63 @@ {"id":"case_acd612e0","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778740258770-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_7169cd5a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_81f2ad8f","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} {"id":"case_6cd7106c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778718965990-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse={\"reply\":\"안녕하세요!\"}","input":{"request":{"requestId":"req_1477394f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_24cff593","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} {"id":"case_de51735e","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778674030965-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_32cc22d6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cd95eff0","nickname":"","trigger":"user-message","userMessage":"안녕하세오"}},"expected":{}} +{"id":"case_4f45294b","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778756106669-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 여기 채팅방에 올리면 돼요. \n채팅방 오른쪽 상단 메뉴의 '커버 보기'로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷으로 보내면 됩니다.","input":{"request":{"requestId":"req_e6e21e71","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0dd56d1c","nickname":"","trigger":"user-message","userMessage":"하트인증\n여따 올리면 되요?"}},"expected":{}} +{"id":"case_e3ff18c6","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778756063255-0; llmOverall=2.75; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 확인했어요. 천천히 진행해 주세요.","input":{"request":{"requestId":"req_7b05c872","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0dd56d1c","nickname":"","trigger":"user-message","userMessage":"네"}},"expected":{}} +{"id":"case_1b906220","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778763820454-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 순서는 이름/출생연도/성별/티어로 맞게 돼 있고, 안 되는 부분은 마지막 티어 코드뿐이에요. \n현재의 '플'은 허용 티어 코드가 아니므로 GM, I, B, S, G, P, E, D, M, C, U 중 한 글자(예: D)로 바꿔주세요.","input":{"request":{"requestId":"req_a8ba4ffe","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_e3e5b006","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_4ff31f85","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778756157817-0; llmOverall=n/a; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 확인했어요. 천천히 진행해 주세요.","input":{"request":{"requestId":"req_c78edc0e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0dd56d1c","nickname":"","trigger":"user-message","userMessage":"오"}},"expected":{}} +{"id":"case_0fd45691","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778874307141-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=닉네임이 안 되는 이유는 형식이 맞지 않아서예요. \n현재 \"위례\"는 이름만 있고, 연도/성별/티어가 없어 형식이 깨졌습니다. \n카카오톡 닉네임은 [이름]/[출생연도 두 자리]/[남 또는 여]/[티어]로 바꿔주세요.","input":{"request":{"requestId":"req_b6f69c24","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_408c317c","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778874400236-0; llmOverall=2.25; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=채팅방 오른쪽 상단 메뉴 버튼을 눌러 “커버 보기”로 들어가면 하트 인증 화면이 보여요.","input":{"request":{"requestId":"req_1db51665","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"","trigger":"user-message","userMessage":"하트인증은 어디다 하나요?"}},"expected":{}} +{"id":"case_cf22a9d8","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778874533535-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_60f5e74a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"","trigger":"user-message","userMessage":"넵 감사합니다"}},"expected":{}} +{"id":"case_fd404eb4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778874508177-0; llmOverall=5; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=네, 가능합니다. 카카오톡 1단계가 완료된 상태라면 디스코드는 저녁에도 접속해 진행하시면 돼요.","input":{"request":{"requestId":"req_d814bae6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"","trigger":"user-message","userMessage":"디스코드는 저녁에 해도될까요?"}},"expected":{}} +{"id":"case_d42b8624","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778874344992-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_47d100d2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"","trigger":"user-message","userMessage":"안녕하세여"}},"expected":{}} +{"id":"case_bf09ad3a","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778954734843-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=현재 남은 절차는 카카오톡 닉네임 변경과 방 하트 인증입니다.","input":{"request":{"requestId":"req_010646c0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a2e8fc0b","nickname":"","trigger":"user-message","userMessage":"혹시 이시간엔 내전 안돌아가죠?"}},"expected":{}} +{"id":"case_275402ce","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778950244576-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기예요.","input":{"request":{"requestId":"req_bc7c8aae","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"","trigger":"user-message","userMessage":"5대5 하려고 하는데 가능 한지 해서용"}},"expected":{}} +{"id":"case_0e56499c","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778950227521-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기예요.","input":{"request":{"requestId":"req_593ffaff","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"","trigger":"user-message","userMessage":"아 지금 들어와 있는 관리자 분이용"}},"expected":{}} +{"id":"case_c8398f09","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778949416363-0; llmOverall=1; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=웅/97/남/M님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_b0f0648f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"","trigger":"user-message","userMessage":"혹시 지 외전 문의 드리려 하는데"}},"expected":{}} +{"id":"case_0a460498","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778950169079-0; llmOverall=2; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=관리분(관리자)은 있어요.","input":{"request":{"requestId":"req_6bffed4f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"","trigger":"user-message","userMessage":"관리분 있으신가요?"}},"expected":{}} +{"id":"case_002ac417","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778931066262-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 \"강서/93/남\"은 티어가 빠져서 형식이 안 맞아요. \n강서/93/남은 티어가 빠져서 형식이 안 맞아요. 마지막에 티어 한 글자만 추가하면 됩니다. \n예: 강서/93/남/D","input":{"request":{"requestId":"req_ed284edb","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0d05dfd2","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_3e163e2e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778949203127-0; llmOverall=2.75; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 형식이 안 맞아요. \n카톡 닉네임은 구분자로 '/'를 써야 하고 4개 항목이 필요해요. 현재는 웅 97 남 M라 안 되고, 현재 닉네임은 웅/97/남/M 이렇게 고치면 됩니다.","input":{"request":{"requestId":"req_ab630198","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_1db9c911","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778958194603-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_f3d19239","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3e4be23b","nickname":"","trigger":"user-message","userMessage":"안녕하세여"}},"expected":{}} +{"id":"case_ee7ae44a","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778955694984-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_fea5fd76","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3592a95a","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_d6bd160f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778950199447-0; llmOverall=2.25; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=운영진께, 특히 관리자님께 문의하시면 돼요.","input":{"request":{"requestId":"req_5fbe8788","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"","trigger":"user-message","userMessage":"누구한테 여쭤봐야 할까요"}},"expected":{}} +{"id":"case_36b3ac9c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778949423115-0; llmOverall=n/a; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=웅/97/남/M님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_35e659fa","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"","trigger":"user-message","userMessage":"어디에 문의 해야 할까용"}},"expected":{}} +{"id":"case_f7a5d661","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779019400869-0; llmOverall=1; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_4861ee12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"","trigger":"user-message","userMessage":"듀랭하실분들 있을까요"}},"expected":{}} +{"id":"case_74eac586","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779019394267-0; llmOverall=1.25; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=톡 게시글의 공지를 참고하셔서 먼저 카카오톡 닉네임 변경과 하트 인증을 진행해 주세요!","input":{"request":{"requestId":"req_743e0456","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"","trigger":"user-message","userMessage":"혹시"}},"expected":{}} +{"id":"case_9e9195bb","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779019388776-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=@벽 넘어가는 탈론 00/남/E3님 닉네임에서 수정할 부분이 있어요!\n나이를 두 자리 숫자로 입력해주세요 (예: 25)\n성별(남/여)과 티어 코드를 '/'로 구분해서 입력해주세요","input":{"request":{"requestId":"req_568d875b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_8acd1f91","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779011411116-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임에서 안 된 부분은 마지막 티어예요. \nG2는 허용 티어가 아니라서, 뒤의 G2를 GM, I, B, S, G, P, E, D, M, C, U 중 하나(예: G)로만 바꾸면 됩니다.","input":{"request":{"requestId":"req_9860e773","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40745cd2","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_d6824fdb","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779023514843-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_4d45f87c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bc77847e","nickname":"","trigger":"user-message","userMessage":"안녕하세요 공지읽고오겠습니다"}},"expected":{}} +{"id":"case_83136947","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779023414663-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요, 반갑습니다.","input":{"request":{"requestId":"req_d55e086e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9ca5eb1","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_1686cd08","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779019324629-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 /로 구분된 4개 항목(이름/출생연도 두 자리/성별/티어)이 아니어서 안 돼요. 지금은 항목이 2개로 잘못되어 있어요. \n고쳐야 할 부분은 연도/성별/티어를 다시 /로 정확히 나눠서 넣는 거예요. \n성별은 남 또는 여 한 글자, 티어는 GM/I/B/S/G/P/E/D/M/C/U 중 한 글자여야 해요.","input":{"request":{"requestId":"req_7cb58844","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_0f722e08","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779015436203-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_a6a1d38f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bc35f985","nickname":"","trigger":"user-message","userMessage":"안녕하세요~"}},"expected":{}} +{"id":"case_009d9558","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779011507015-0; llmOverall=2.25; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 \n카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널 메시지 남기기 입니다.","input":{"request":{"requestId":"req_18174c18","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40745cd2","nickname":"","trigger":"user-message","userMessage":"이러면되나요"}},"expected":{}} +{"id":"case_b51a6dfb","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779011451230-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_5d3182b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40745cd2","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_8f87e018","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779562992008-0; llmOverall=1; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=고구마는보라색/98/남/D님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_d9ff0f8b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ab97b973","nickname":"","trigger":"user-message","userMessage":"봇인가요? 어케 이렇게 빠르지"}},"expected":{}} +{"id":"case_51fec5d7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779563007004-0; llmOverall=1.25; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=고구마는보라색/98/남/D님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_058c9557","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ab97b973","nickname":"","trigger":"user-message","userMessage":"아하"}},"expected":{}} +{"id":"case_47db12bc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779544865402-0; llmOverall=1.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=방 하트 인증은 채팅방 오른쪽 상단 메뉴 버튼에서 \"커버 보기\"를 눌러 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.","input":{"request":{"requestId":"req_887be307","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_e35470e6","nickname":"","trigger":"user-message","userMessage":"어떻게 인증하나요 ?"}},"expected":{}} +{"id":"case_8aa3d0bc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779574878821-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=아직 승인해드리기 어려워요. 남은 항목이 카카오톡 본인인증방 입장과 디스코드 주민신청 채널 메시지라서, 이 두 가지만 먼저 완료해주시면 될 것 같아요.","input":{"request":{"requestId":"req_117cdf74","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"","trigger":"user-message","userMessage":"디코 승인 부탁드려도 될까요?"}},"expected":{}} +{"id":"case_0191603e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779574708797-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디스코드 관련 안내입니다. 디스코드 닉네임은 닉네임#태그/출생 연도 두 자리/티어 형식으로 수정해 주세요. 예시: 할리#프렌치불독/88/D \n디스코드 서버에서 이 닉네임은 본인 식별용이라 꼭 맞춰야 하고, 주민신청 채널에는 서버 소개 메시지도 남겨야 해요.","input":{"request":{"requestId":"req_17d4f38d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"","trigger":"user-message","userMessage":"/디코"}},"expected":{}} +{"id":"case_214e18f8","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779562888510-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=@고구마는보라색/98/D/남님 닉네임에서 수정할 부분이 있어요!\n성별을 '남' 또는 '여'로 입력해주세요\n티어 코드(GM, I, B, S, G, P, E, D, M, C, U 중 하나)를 입력해주세요","input":{"request":{"requestId":"req_dd3e072b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ab97b973","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_b90d32a5","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779535490115-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=네, 다시 보내주시면 돼요. \n방에서 오른쪽 상단 메뉴 버튼 열고 커버 보기로 들어간 뒤 하트를 눌러 빨간색으로 바꿔주세요. \n그 상태 화면을 채팅방에서 바로 캡처해서 편집(자르기/필터/화면 편집) 없이 원본 그대로 보내주시면 됩니다. \n참고로 회색 하트면 아직 인증으로 안 된 상태예요.","input":{"request":{"requestId":"req_8ac4ab90","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7d8e9f5b","nickname":"","trigger":"photo-failed"}},"expected":{}} +{"id":"case_dc095d27","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779535154924-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=네, 다시 시도할 수 있어요. \n방 하트를 빨간색으로 바꾼 뒤, 채팅방에서 바로 캡처한 원본 화면(편집/자르기/필터 없음)만 새로 보내주시면 됩니다.","input":{"request":{"requestId":"req_bb9360ad","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7d8e9f5b","nickname":"","trigger":"photo-failed"}},"expected":{}} +{"id":"case_c33b0453","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779535131374-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 하트는 눌렀다면 하트가 빨간색인 화면을 편집 없이 원본 스크린샷으로 올려주시면 돼요.","input":{"request":{"requestId":"req_5c1b0e13","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7d8e9f5b","nickname":"","trigger":"user-message","userMessage":"하트는 눌렀는대.."}},"expected":{}} +{"id":"case_955dc7df","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1779605646714-0; llmOverall=2.5; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=오케이, 하트 인증은 화면이 제대로 올라오면 바로 확인돼요. \n지금 올리신 걸로 보냈다면 잠깐 확인 기다려볼게요. \n만약 아직 안 보인다면, 하트를 빨간색으로 바꾼 그대로의 원본 캡처만 한 번 더 보내주세요.","input":{"request":{"requestId":"req_fb38ace6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0ea265d7","nickname":"","trigger":"user-message","userMessage":"위에 인증햇어여"}},"expected":{}} +{"id":"case_e75e191e","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779604227776-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=다시 시도하려면 이렇게 하면 돼요.\n\n채팅방 오른쪽 상단 메뉴 버튼에서 커버 보기로 들어가서 하트를 빨간색으로 바꾼 뒤, \n그 장면이 보이게 편집 없이 원본 화면 캡처로 다시 보내주면 돼요.","input":{"request":{"requestId":"req_4368c115","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0ea265d7","nickname":"","trigger":"photo-failed"}},"expected":{}} +{"id":"case_f485439f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779593282207-0; llmOverall=n/a; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=Summer/95/남/S님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_a33cd695","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_18313eb7","nickname":"","trigger":"user-message","userMessage":"가입했습니다"}},"expected":{}} +{"id":"case_75bc3458","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779593274596-0; llmOverall=n/a; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=Summer/95/남/S님 이제 추가 절차만 남았어요. 카카오톡 본인인증방 입장과 디스코드 주민신청 채널에 메시지 남기기을 확인해 주세요.","input":{"request":{"requestId":"req_5bb3e408","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_18313eb7","nickname":"","trigger":"user-message","userMessage":"디스코드"}},"expected":{}} +{"id":"case_c1016e76","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779592708891-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 하세요.","input":{"request":{"requestId":"req_179a599b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_18313eb7","nickname":"","trigger":"user-message","userMessage":"엇 잠시만용"}},"expected":{}} +{"id":"case_12f3372d","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779574484786-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 바로 여기 채팅방에 올리면 됩니다. \n채팅방 오른쪽 상단 메뉴의 \"커버 보기\"에서 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷으로 보내주세요.","input":{"request":{"requestId":"req_b7b755bf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"","trigger":"user-message","userMessage":"하트인증은 여기다 올리면 될까요?"}},"expected":{}} +{"id":"case_9f0cb2f4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779574363525-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_b822a53e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_885a35a9","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629981088-0; llmOverall=1; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=톡 게시글의 공지를 참고하셔서 먼저 카카오톡 닉네임 변경과 하트 인증을 진행해 주세요!","input":{"request":{"requestId":"req_42415f0f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"","trigger":"user-message","userMessage":"진짜 잘하시네요"}},"expected":{}} +{"id":"case_4603c0cc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629976794-0; llmOverall=1; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_5400a31f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"","trigger":"user-message","userMessage":"챌린저 ㄷㄷ"}},"expected":{}} +{"id":"case_2b38adb8","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629930176-0; llmOverall=1; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=채팅방 프로필에서 하트를 눌러 빨간색으로 만든 다음, 그 화면이 보이게 원본 스크린샷을 보내주시면 인증돼요.","input":{"request":{"requestId":"req_bf140842","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"","trigger":"user-message","userMessage":"어떻게 하나여"}},"expected":{}} +{"id":"case_f04b5287","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779633055712-0; llmOverall=1.25; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요, 오늘도 편하게 진행해요.","input":{"request":{"requestId":"req_bdd1114b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"","trigger":"user-message","userMessage":"안녕하세요!"}},"expected":{}} +{"id":"case_3e077513","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779630086906-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 그렇게 해주시면 돼요.","input":{"request":{"requestId":"req_bbc426e2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"","trigger":"user-message","userMessage":"아항"}},"expected":{}} +{"id":"case_274fec57","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629952313-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_c69053e8","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"","trigger":"user-message","userMessage":"보내면 되나여?"}},"expected":{}} +{"id":"case_29f02c6e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629936899-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=톡 게시글의 공지를 참고하셔서 먼저 카카오톡 닉네임 변경과 하트 인증을 진행해 주세요!","input":{"request":{"requestId":"req_be0e7f4a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"","trigger":"user-message","userMessage":"여기 채팅방에"}},"expected":{}} +{"id":"case_59c880e4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779633743117-0; llmOverall=4.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=카카오톡에서 디스코드 초대 안내를 받았던 메시지를 찾아 초대 수락을 눌러 들어가면 돼요.","input":{"request":{"requestId":"req_705c9e3f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"","trigger":"user-message","userMessage":"저 디코는 어떻게 들어가요? 주민신청까지 해뒀습니다"}},"expected":{}} +{"id":"case_4fe5864d","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779632430014-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_781682ec","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"","trigger":"user-message","userMessage":"넵"}},"expected":{}} +{"id":"case_9a3640ce","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779632381253-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=안 되는 이유는 티어가 허용 코드(GM, I, B, S, G, P, E, D, M, C, U) 중 하나의 대문자로 되어야 하는데 p가 소문자여서예요. \n카카오톡 닉네임의 마지막 부분만 P로 바꿔서 세나/95/여/P로 수정해 주세요.","input":{"request":{"requestId":"req_7368778f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_5ea6c4d2","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779629969676-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=환영해요! 반갑습니다.","input":{"request":{"requestId":"req_9352c82d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"","trigger":"user-message","userMessage":"안녕하세요!!"}},"expected":{}} +{"id":"case_207dec80","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779629904020-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=방 하트 인증은 채팅방 오른쪽 상단 메뉴 버튼의 '커버 보기'에서 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 화면이 보이게 스크린샷을 보내주시면 돼요.","input":{"request":{"requestId":"req_37928f7d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"","trigger":"user-message","userMessage":"방하트 인증은"}},"expected":{}} +{"id":"case_b42ca588","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779622035543-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=어서오세요!","input":{"request":{"requestId":"req_e9dcff68","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3adc9936","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_d96d1b98","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779619834487-0; llmOverall=n/a; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 본인인증이 완료됐다면 본인인증방은 나가셔도 돼요. \n현재 남은 절차는 디스코드 주민신청 채널 메시지 남기기 하나예요.","input":{"request":{"requestId":"req_e6bc023c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_d9c80f01","nickname":"","trigger":"user-message","userMessage":"본인인증방은이제나가도돼요?"}},"expected":{}} diff --git a/src/eval/judges/llm-judge.test.ts b/src/eval/judges/llm-judge.test.ts index 193ae93..c16b36d 100644 --- a/src/eval/judges/llm-judge.test.ts +++ b/src/eval/judges/llm-judge.test.ts @@ -93,6 +93,29 @@ describe("judgeOnboardingWithLlm", () => { } }); + it("keeps scores when rationale is longer than the prompt asks for", async () => { + const result = await judgeOnboardingWithLlm( + new FakeLlmClient( + JSON.stringify({ + friendliness: 3, + accuracy: 4, + concision: 5, + scope: 4, + rationale: "가".repeat(800), + }), + ), + evalCase, + "이제 방 하트 인증만 남았어요.", + derived, + ); + + expect(result.skipped).toBe(false); + if (!result.skipped) { + expect(result.scores.overall).toBe(4); + expect(result.rationale).toHaveLength(500); + } + }); + it("skips when the judge returns invalid JSON", async () => { const result = await judgeOnboardingWithLlm( new FakeLlmClient('{"friendliness":"bad"}'), diff --git a/src/eval/judges/llm-judge.ts b/src/eval/judges/llm-judge.ts index 012acae..30b76d4 100644 --- a/src/eval/judges/llm-judge.ts +++ b/src/eval/judges/llm-judge.ts @@ -9,6 +9,8 @@ import type { } from "../types.js"; import { getSkillJudgeConfig } from "./skill-config.js"; +const MAX_RATIONALE_CHARS = 500; + function round(value: number): number { return Number(value.toFixed(3)); } @@ -18,10 +20,14 @@ function createJudgeSchema(axes: readonly string[]): z.ZodObject ...Object.fromEntries( axes.map((axis) => [axis, z.number().int().min(1).max(5)]), ), - rationale: z.string().trim().min(1).max(200), + rationale: z.string().trim().min(1), }); } +function normalizeRationale(value: unknown): string { + return String(value).slice(0, MAX_RATIONALE_CHARS); +} + function buildJudgePrompt( skillName: string, evalCase: EvalCaseDefinition, @@ -35,6 +41,7 @@ function buildJudgePrompt( `다음 ${skillName} 에이전트 응답을 평가해 주세요.`, `평가 축은 ${axes} 입니다.`, "각 점수는 1~5의 정수만 사용하세요.", + "rationale은 200자 이하로 짧게 작성하세요.", config.promptTemplate ?? "accuracy는 기대 조건과 실제 답변의 사실성을, concision은 과도한 부가 설명 여부를 포함해 평가하세요.", "", @@ -129,7 +136,7 @@ export async function judgeSkillWithLlm( endpoint: response.endpoint, provider: response.provider, scores, - rationale: String(parsed["rationale"]), + rationale: normalizeRationale(parsed["rationale"]), raw: response.content, }; } catch { diff --git a/src/eval/proposals/579dcc7d.json b/src/eval/proposals/579dcc7d.json new file mode 100644 index 0000000..ba1ba63 --- /dev/null +++ b/src/eval/proposals/579dcc7d.json @@ -0,0 +1,16 @@ +{ + "description": "deadline-near 상태에서도 사용자가 묻지 않은 남은 등록 절차를 자동으로 붙이지 않도록 우선순위 강화", + "diff": [ + { + "layer": "policy", + "op": "append", + "reason": "첫 번째 실패에서 현재 상태의 remaining 정보가 질문보다 우선되어 무관한 절차 안내가 출력됐다. deadline-near에서도 질문 우선 원칙을 강화한다.", + "text": "[deadline-near 응답 범위 제한]\n- phase가 deadline-near여도 마지막 메시지가 가능 여부/확인 주체/확인 여부를 묻는 질문이면, 독촉성 남은 절차 안내를 붙이지 마.\n- remaining 상태는 사용자가 \"뭐가 남았나요\", \"다 했나요\", \"완료인가요\"처럼 진행 상태를 물을 때만 사용해.\n- 답변 전 마지막으로 확인: 내 응답 첫 문장이 사용자의 질문에 대한 '네/아니요/아직/관리자가 확인' 중 직접 답인지 점검해." + } + ], + "expectedImprovement": "무관한 카카오톡/디스코드 절차 나열이 줄고, 사용자의 마지막 질문 중심으로 1문장 내외의 답변이 생성된다.", + "rationale": "현재 스냅샷에는 remaining 정보가 강하게 주입되어 있어, 사용자가 재등록 가능 여부만 묻는 상황에서도 남은 절차를 안내할 위험이 높다. 진행 상태 질문이 아닐 때 remaining을 쓰지 않게 하면 범위 이탈을 줄일 수 있다.", + "skill": "onboarding", + "variantId": "579dcc7d", + "trustTier": "T2" +} diff --git a/src/eval/proposals/991bfb18.json b/src/eval/proposals/991bfb18.json new file mode 100644 index 0000000..374f2a9 --- /dev/null +++ b/src/eval/proposals/991bfb18.json @@ -0,0 +1,16 @@ +{ + "description": "디스코드 확인 주체/재등록 가능 여부 질문에는 상태 나열이나 불확실 회피 대신 짧게 직접 답하도록 하는 예외 규칙 추가", + "diff": [ + { + "layer": "policy", + "op": "append", + "reason": "실패 케이스가 모두 사용자의 단일 질문에 답하지 못하고, 확인 불가 회피 또는 무관한 남은 절차 나열로 이탈했다. 해당 slice에서 반복되는 직접 질문 패턴을 좁게 처리한다.", + "text": "[discord-verify-question 직접응답 보강]\n- 사용자가 \"디코는 누가 확인하나요\", \"관리자가 확인하나요\", \"관리자 형님들이 확인해야 하나요\"처럼 디스코드 확인 주체를 물으면, 확인되지 않았다고 회피하지 말고 \"네, 디스코드 관련 확인은 관리자/운영진이 확인해요.\"라고 짧게 답해.\n- 사용자가 \"계정 확인 후 다시 등록해도 되나요\", \"제 명의 계정 확인 후 다시 등록해도 될까요\"처럼 본인 계정/명의 확인 뒤 재등록 가능 여부를 물으면, 다른 절차를 나열하지 말고 \"네, 본인 명의 계정 확인 후 다시 등록해 주시면 돼요.\"라고 답해.\n- 위 두 유형에서는 카카오톡 완료 여부, 남은 단계, 디스코드 닉네임 형식, 다음 절차를 사용자가 묻지 않는 한 덧붙이지 마." + } + ], + "expectedImprovement": "디스코드 확인 주체 질문에는 '관리자/운영진이 확인한다'고 정확히 답하고, 계정 확인 후 재등록 질문에는 '가능하다'고 직접 답해 accuracy와 scope가 개선된다.", + "rationale": "현재 프롬프트의 '확인되지 않았으면 확인되지 않았다고만 답'과 '세션 상태' 정보가 과하게 작동해, 실제로는 일반 운영 규칙으로 답할 수 있는 질문에도 회피하거나 상태 목록을 출력한다. 확인 주체와 재등록 가능 여부는 온보딩 절차의 일반 질문이므로 짧은 정답 템플릿을 추가하는 것이 가장 작은 수정이다.", + "skill": "onboarding", + "variantId": "991bfb18", + "trustTier": "T2" +} diff --git a/src/eval/runner.ts b/src/eval/runner.ts index 76eb8f8..254d1aa 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -31,7 +31,13 @@ import type { DerivedState, } from "@skills/onboarding/state/onboarding-state"; import { noopShadowEvalPublisher } from "./shadow-bus.js"; -import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js"; +import { + getEvalCaseDeadlineMs, + getEvalLlmProvider, + getEvalRuntimeLlmEnabled, + getOnboardingSessionKeyPrefix, + getOpenRouterApiUrl, +} from "../config.js"; import { runEvalDispatch } from "./handoff-dispatch.js"; import { judgeHandoff } from "./judges/handoff-judge.js"; import { judgeSkillWithLlm } from "./judges/llm-judge.js"; @@ -285,6 +291,9 @@ function toProviderEndpoints( } export function createEvalLlmClient(): LlmClient { + const evalProvider = getEvalLlmProvider(); + const includeProvider = (provider: ProviderName): boolean => + evalProvider === "all" || evalProvider === provider; const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]); const llamaEndpoints = parseEndpointList( process.env["AGENT_LLM_API_URLS"], @@ -301,7 +310,14 @@ export function createEvalLlmClient(): LlmClient { ); const providerApiKeys = Object.fromEntries( [ - ["cliproxy", firstNonEmpty(process.env["CLI_PROXY_API_KEY"])], + [ + "cliproxy", + firstNonEmpty( + process.env["CLI_PROXY_API_KEY"], + process.env["LLAMA_SERVER_API_KEY"], + process.env["AGENT_LLM_API_KEY"], + ), + ], ["llama", llmApiKey], ["openrouter", openRouterApiKey], ].filter((entry): entry is [string, string] => typeof entry[1] === "string"), @@ -309,9 +325,15 @@ export function createEvalLlmClient(): LlmClient { return new LlmClient({ endpoints: [ - ...toProviderEndpoints(cliproxyEndpoints, "cliproxy"), - ...toProviderEndpoints(openRouterEndpoints, "openrouter"), - ...toProviderEndpoints(llamaEndpoints, "llama"), + ...(includeProvider("cliproxy") + ? toProviderEndpoints(cliproxyEndpoints, "cliproxy") + : []), + ...(includeProvider("openrouter") + ? toProviderEndpoints(openRouterEndpoints, "openrouter") + : []), + ...(includeProvider("llama") + ? toProviderEndpoints(llamaEndpoints, "llama") + : []), ], apiKey: llmApiKey, model: firstNonEmpty(process.env["AGENT_LLM_MODEL"]), @@ -846,7 +868,8 @@ export async function loadEvalCases( async function evaluateCase( evalCase: EvalCaseDefinition, - llmClient: LlmClient, + runtimeLlmClient: LlmClient, + judgeLlmClient: LlmClient, enableLlmJudge: boolean, skillName: string, ): Promise { @@ -860,7 +883,7 @@ async function evaluateCase( : undefined; const deps = { garnet: inMemoryGarnet, - llm: llmClient, + llm: runtimeLlmClient, room: roomContextService, }; if (skillName === "handoff") { @@ -868,7 +891,7 @@ async function evaluateCase( evalCase, input, Object.values(SKILL_RUNNER_FACTORIES).map((factory) => factory(deps)), - llmClient, + judgeLlmClient, enableLlmJudge, ); } @@ -883,7 +906,7 @@ async function evaluateCase( return runWithContext( { chatroomId: input.request.chatroomId, - deadlineAt: new Date(Date.now() + 4_500).toISOString(), + deadlineAt: new Date(Date.now() + getEvalCaseDeadlineMs()).toISOString(), requestId: input.request.requestId, skill: input.request.skill, startedAt: new Date().toISOString(), @@ -896,7 +919,8 @@ async function evaluateCase( evalCase, input, skill, - llmClient, + runtimeLlmClient, + judgeLlmClient, enableLlmJudge, effectiveSkill, ) @@ -904,7 +928,7 @@ async function evaluateCase( evalCase, input, skill, - llmClient, + judgeLlmClient, enableLlmJudge, effectiveSkill, ), @@ -915,11 +939,12 @@ async function evaluateOnboardingCase( evalCase: EvalCaseDefinition, input: EvalCaseInput, skill: Skill, - llmClient: LlmClient, + runtimeLlmClient: LlmClient, + judgeLlmClient: LlmClient, enableLlmJudge: boolean, skillName: string, ): Promise { - const classifier = new OnboardingIntentClassifier(llmClient); + const classifier = new OnboardingIntentClassifier(runtimeLlmClient); const processTrace = new ProcessTraceCollector(); const judgeConfig = getSkillJudgeConfig(skillName); @@ -950,7 +975,7 @@ async function evaluateOnboardingCase( }); const llmJudge = enableLlmJudge ? await judgeSkillWithLlm( - llmClient, + judgeLlmClient, skillName, evalCase, response.text, @@ -1262,12 +1287,23 @@ export async function runEval( limit: options?.limit, }); - const llmClient = createEvalLlmClient(); + const judgeLlmClient = createEvalLlmClient(); + const runtimeLlmClient = getEvalRuntimeLlmEnabled() + ? judgeLlmClient + : new LlmClient({ endpoints: [] }); const enableLlmJudge = options?.enableLlmJudge ?? true; const caseResults: EvalCaseResult[] = []; for (const evalCase of evalCases) { - caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge, skill)); + caseResults.push( + await evaluateCase( + evalCase, + runtimeLlmClient, + judgeLlmClient, + enableLlmJudge, + skill, + ), + ); } const summary = createSummary(caseResults); diff --git a/src/llm/llm-client.ts b/src/llm/llm-client.ts index 15d02cc..af7cb52 100644 --- a/src/llm/llm-client.ts +++ b/src/llm/llm-client.ts @@ -194,9 +194,10 @@ export class LlmClient { continue; } + const configuredRequestTimeoutMs = getLlmRequestTimeoutMs(); const requestTimeoutMs = Math.max( 1, - Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY), + Math.min(configuredRequestTimeoutMs, options?.deadlineMs ?? Number.POSITIVE_INFINITY), ); const models = selectModelsForProvider(provider.name, options, this.llamaModel); let endpointSucceeded = false; @@ -241,6 +242,9 @@ export class LlmClient { model: request.model, status: response.status, body, + configuredTimeoutMs: configuredRequestTimeoutMs, + deadlineMs: options?.deadlineMs, + timeoutMs: requestTimeoutMs, }, "[LLM] Request failed", ); @@ -304,6 +308,16 @@ export class LlmClient { if (llmResult) { await this.circuit.recordSuccess(endpoint.completionUrl); endpointSucceeded = true; + logger.debug( + { + endpoint: endpoint.completionUrl, + model: request.model, + provider: provider.name, + role: options?.role, + tier: options?.tier, + }, + "[LLM] Response received", + ); return llmResult; } } catch (error) { @@ -313,6 +327,9 @@ export class LlmClient { provider: provider.name, model, error: error instanceof Error ? error.message : String(error), + configuredTimeoutMs: configuredRequestTimeoutMs, + deadlineMs: options?.deadlineMs, + timeoutMs: requestTimeoutMs, }, "[LLM] Request error", ); diff --git a/src/llm/routing.test.ts b/src/llm/routing.test.ts index 9a65340..5e4b97a 100644 --- a/src/llm/routing.test.ts +++ b/src/llm/routing.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { LlmEndpointConfig } from "./types.js"; import { + resetLlmRoutingRoundRobinForTests, selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder, @@ -8,6 +9,12 @@ import { describe("llm routing", () => { afterEach(() => { + delete process.env["CLIPROXY_TIER0_MODEL"]; + delete process.env["CLIPROXY_TIER1_PLANNER_MODEL"]; + delete process.env["CLIPROXY_TIER2_CRITIC_MODEL"]; + delete process.env["LLAMA_TIER2_CRITIC_MODEL"]; + delete process.env["LLM_MODEL_ROUND_ROBIN_ENABLED"]; + delete process.env["OPENROUTER_TIER1_RESPONDER_MODEL"]; delete process.env["OPENROUTER_MODEL"]; delete process.env["TIER0_MODEL"]; delete process.env["TIER1_PLANNER_MODEL"]; @@ -16,6 +23,7 @@ describe("llm routing", () => { delete process.env["TIER0_MAX_TOKENS"]; delete process.env["TIER1_MAX_TOKENS"]; delete process.env["TIER2_MAX_TOKENS"]; + resetLlmRoutingRoundRobinForTests(); vi.restoreAllMocks(); }); @@ -93,9 +101,12 @@ describe("llm routing", () => { ).toEqual(["cliproxy", "openrouter"]); }); - it("returns role-aware model for cliproxy and llama fallback as single-item list", () => { - process.env["TIER1_PLANNER_MODEL"] = "cliproxy-planner"; - process.env["TIER2_CRITIC_MODEL"] = "cliproxy-critic"; + it("returns provider-aware models for runtime roles and tiers", () => { + process.env["CLIPROXY_TIER1_PLANNER_MODEL"] = "cliproxy-planner"; + process.env["CLIPROXY_TIER2_CRITIC_MODEL"] = "cliproxy-critic"; + process.env["OPENROUTER_TIER1_RESPONDER_MODEL"] = + "openrouter-fast, openrouter-backup"; + process.env["LLAMA_TIER2_CRITIC_MODEL"] = "llama-critic"; expect( selectModelsForProvider("cliproxy", { role: "planner", tier: 1 }, "local-qwen"), @@ -103,9 +114,12 @@ describe("llm routing", () => { expect( selectModelsForProvider("cliproxy", { role: "critic", tier: 2 }, "local-qwen"), ).toEqual(["cliproxy-critic"]); - expect(selectModelsForProvider("llama", { tier: 2 }, "local-qwen")).toEqual([ - "local-qwen", - ]); + expect( + selectModelsForProvider("openrouter", { role: "responder", tier: 1 }, "local-qwen"), + ).toEqual(["openrouter-fast", "openrouter-backup"]); + expect( + selectModelsForProvider("llama", { role: "critic", tier: 2 }, "local-qwen"), + ).toEqual(["llama-critic"]); }); it("splits OPENROUTER_MODEL by comma and preserves priority order", () => { @@ -124,6 +138,49 @@ describe("llm routing", () => { ]); }); + it("round-robins models within the same provider role and tier when enabled", () => { + process.env["LLM_MODEL_ROUND_ROBIN_ENABLED"] = "true"; + process.env["CLIPROXY_TIER0_MODEL"] = + "gemini-3.1-flash-lite, gpt-5.3-codex-spark"; + + expect( + selectModelsForProvider("cliproxy", { tier: 0 }, "local-qwen"), + ).toEqual(["gemini-3.1-flash-lite", "gpt-5.3-codex-spark"]); + expect( + selectModelsForProvider("cliproxy", { tier: 0 }, "local-qwen"), + ).toEqual(["gpt-5.3-codex-spark", "gemini-3.1-flash-lite"]); + expect( + selectModelsForProvider("cliproxy", { tier: 0 }, "local-qwen"), + ).toEqual(["gemini-3.1-flash-lite", "gpt-5.3-codex-spark"]); + }); + + it("keeps provider order fixed even when model round-robin is enabled", () => { + process.env["LLM_MODEL_ROUND_ROBIN_ENABLED"] = "true"; + const endpoints: LlmEndpointConfig[] = [ + { + completionUrl: "http://llama.test/v1/chat/completions", + healthUrl: "http://llama.test/health", + provider: "llama", + }, + { + completionUrl: "http://openrouter.test/v1/chat/completions", + healthUrl: "http://openrouter.test", + provider: "openrouter", + }, + { + completionUrl: "http://cliproxy.test/v1/chat/completions", + healthUrl: "http://cliproxy.test", + provider: "cliproxy", + }, + ]; + + expect( + selectProviderOrder(endpoints, { role: "responder", tier: 1 }).map( + (endpoint) => endpoint.provider, + ), + ).toEqual(["cliproxy", "openrouter", "llama"]); + }); + it("chooses tier defaults for max tokens when explicit maxTokens is absent", () => { process.env["TIER0_MAX_TOKENS"] = "111"; process.env["TIER1_MAX_TOKENS"] = "222"; diff --git a/src/llm/routing.ts b/src/llm/routing.ts index 88413e4..c29ba0d 100644 --- a/src/llm/routing.ts +++ b/src/llm/routing.ts @@ -1,11 +1,8 @@ import { - getOpenRouterModels, + getLlmModelRoundRobinEnabled, + getRuntimeLlmModelsForProvider, getTier0MaxTokens, - getTier0Model, getTier1MaxTokens, - getTier1PlannerModel, - getTier1ResponderModel, - getTier2CriticModel, getTier2MaxTokens, getLlmMaxTokens, } from "../config.js"; @@ -13,6 +10,7 @@ import type { ChatCompletionOptions, LlmEndpointConfig, ProviderName } from "./t const PROVIDER_PRIORITY: readonly ProviderName[] = ["cliproxy", "openrouter", "llama"]; const TEMPLATE_ONLY_PROVIDER_PRIORITY: readonly ProviderName[] = ["llama"]; +const modelRoundRobinOffsets = new Map(); function getProviderPriority( options: ChatCompletionOptions | undefined, @@ -33,27 +31,15 @@ export function selectModelsForProvider( return [options.modelOverride.trim()]; } - if (provider === "llama") { - return [llamaFallbackModel]; - } - - if (provider === "openrouter") { - return getOpenRouterModels(); - } - - if (options?.role === "planner") { - return [getTier1PlannerModel()]; - } - - if (options?.role === "critic") { - return [getTier2CriticModel()]; - } - - if (options?.tier === 1 || options?.tier === 2) { - return [getTier1ResponderModel()]; - } - - return [getTier0Model()]; + const models = getRuntimeLlmModelsForProvider( + provider, + { + role: options?.role, + tier: options?.tier, + }, + llamaFallbackModel, + ); + return rotateModelsForProvider(provider, options, models); } export function selectMaxTokensForTier( @@ -106,3 +92,28 @@ export function selectProviderOrder( }) .map((item) => item.endpoint); } + +function rotateModelsForProvider( + provider: ProviderName, + options: ChatCompletionOptions | undefined, + models: string[], +): string[] { + if (!getLlmModelRoundRobinEnabled() || models.length <= 1) { + return models; + } + + const key = [ + provider, + options?.role ?? "none", + options?.tier ?? "none", + models.join("|"), + ].join(":"); + const offset = modelRoundRobinOffsets.get(key) ?? 0; + modelRoundRobinOffsets.set(key, offset + 1); + const rotation = offset % models.length; + return [...models.slice(rotation), ...models.slice(0, rotation)]; +} + +export function resetLlmRoutingRoundRobinForTests(): void { + modelRoundRobinOffsets.clear(); +} diff --git a/src/skills/onboarding/intents.ts b/src/skills/onboarding/intents.ts index 0002cfb..edf7ea5 100644 --- a/src/skills/onboarding/intents.ts +++ b/src/skills/onboarding/intents.ts @@ -38,6 +38,48 @@ export const ONBOARDING_INTENT_TABLE = { match: ({ trimmed }) => isRemainingStatusQuestion(trimmed), triggers: ["progress-update", "timeout-warning"], }, + "discord-verify-question": { + confidence: 0.96, + keywords: ["디스코드", "확인", "닉네임", "주민신청"], + llm: true, + match: ({ lower, trimmed }) => { + const mentionsDiscord = /(디스코드|디코|discord)/u.test(lower); + const asksDiscordVerification = + /(확인|조회|찾아|검색|검사|바뀌|변경|닉변|맞(?:아|나요)?|됐(?:나|나요)?|완료됐(?:나|나요)?|승인|반영)/u.test( + trimmed, + ); + const reportsDiscordNicknameCompletion = + /(닉네임|닉변|프로필\s*닉네임|서버\s*닉네임)/u.test(trimmed) && + /(완료했(?:어|어요|습니다)?|끝냈(?:어|어요|습니다)?|설정했(?:어|어요|습니다)?|바꿨(?:어|어요|습니다)?|변경했(?:어|어요|습니다)?|다\s*했(?:어|어요|습니다)?|해놨(?:어|어요|습니다)?|해뒀(?:어|어요|습니다)?|적용됐(?:어|어요|습니다)?)/u.test( + trimmed, + ); + const reportsResidentApplicationCompletion = + /(주민\s*신청|주민신청|주민\s*등록|주민등록|프로필\s*메시지|메시지)/u.test( + trimmed, + ) && + /(완료했(?:어|어요|습니다)?|끝냈(?:어|어요|습니다)?|남겼(?:어|어요|습니다)?|썼(?:어|어요|습니다)?|작성했(?:어|어요|습니다)?|다\s*했(?:어|어요|습니다)?|해놨(?:어|어요|습니다)?|해뒀(?:어|어요|습니다)?|했(?:어|어요|습니다)?)/u.test( + trimmed, + ); + return ( + mentionsDiscord && + (asksDiscordVerification || + reportsDiscordNicknameCompletion || + reportsResidentApplicationCompletion) + ); + }, + }, + "discord-question": { + confidence: 0.95, + keywords: ["디스코드", "주민신청", "프로필", "권한", "설정"], + llm: true, + match: ({ lower }) => /(디스코드|디코|discord)/u.test(lower), + regex: [ + { + confidence: 0.94, + pattern: /(주민신청\s*채널|주민신청채널|서버\s*프로필)/u, + }, + ], + }, "help-where": { confidence: 0.9, keywords: [ @@ -75,39 +117,6 @@ export const ONBOARDING_INTENT_TABLE = { /(가입|등록|절차|도움|도와|뭐해야|뭐 해야|해야 돼|해야해|무엇을 해야)/u, ], }, - "discord-verify-question": { - confidence: 0.96, - keywords: ["디스코드", "확인", "닉네임"], - llm: true, - match: ({ lower, trimmed }) => { - const mentionsDiscord = /(디스코드|디코|discord)/u.test(lower); - const asksDiscordVerification = - /(확인|조회|찾아|검색|검사|바뀌|변경|닉변|맞(?:아|나요)?|됐(?:나|나요)?|완료됐(?:나|나요)?)/u.test( - trimmed, - ); - const reportsDiscordNicknameCompletion = - /(닉네임|닉변|프로필\s*닉네임|서버\s*닉네임)/u.test(trimmed) && - /(완료했(?:어|어요|습니다)?|끝냈(?:어|어요|습니다)?|설정했(?:어|어요|습니다)?|바꿨(?:어|어요|습니다)?|변경했(?:어|어요|습니다)?|다\s*했(?:어|어요|습니다)?|해놨(?:어|어요|습니다)?|해뒀(?:어|어요|습니다)?|적용됐(?:어|어요|습니다)?)/u.test( - trimmed, - ); - return ( - mentionsDiscord && - (asksDiscordVerification || reportsDiscordNicknameCompletion) - ); - }, - }, - "discord-question": { - confidence: 0.95, - keywords: ["디스코드", "주민신청", "프로필", "권한", "설정"], - llm: true, - match: ({ lower }) => /(디스코드|디코|discord)/u.test(lower), - regex: [ - { - confidence: 0.94, - pattern: /(주민신청\s*채널|주민신청채널|서버\s*프로필)/u, - }, - ], - }, "age-question": { confidence: 0.93, keywords: ["27", "가입", "가능"], diff --git a/src/skills/onboarding/prompt-data.ts b/src/skills/onboarding/prompt-data.ts index 4c779c5..a8f846b 100644 --- a/src/skills/onboarding/prompt-data.ts +++ b/src/skills/onboarding/prompt-data.ts @@ -145,12 +145,12 @@ export const ONBOARDING_PROMPT_DATA = { { user: "디스코드 프로필 닉네임 설정 완료했어요", assistant: - "디스코드 닉네임을 확인해봤는데 아직 올바른 형식으로 확인되지는 않았어요. 디스코드 서버 닉네임은 이름#태그/출생 연도/티어 형식이에요. 예: 할리#프렌치불독/88/D", + "디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '날으는꼴뚜기/D' 으로 확인됩니다. 할리#프렌치불독/88/D 와 같이 닉네임#태그/출생연도/티어로 수정해 주세요.", }, { user: "디스코드 닉네임 바꿨어요", assistant: - "디스코드 닉네임이 아직 이름#태그/출생 연도/티어 형식으로 확인되지 않아요. 예: 할리#프렌치불독/88/D처럼 태그를 포함해서 설정해 주세요.", + "디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '날으는꼴뚜기/D' 으로 확인됩니다. 할리#프렌치불독/88/D 와 같이 닉네임#태그/출생연도/티어로 수정해 주세요.", }, ], greeting: [ @@ -236,10 +236,10 @@ export const ONBOARDING_PROMPT_DATA = { ].join("\n"), "discord-question": [ "- 디스코드 일반 안내나 규칙 질문이면 lookup-community-rule 도구를 사용해 정확한 규칙만 답해.", - "- 사용자가 디스코드 닉네임이나 프로필 설정을 끝냈다고 말하거나 반영 여부를 묻는다면 verify-discord-nickname 도구를 먼저 사용해 직접 확인해.", + "- 사용자가 디스코드 닉네임, 프로필 설정, 주민신청 메시지를 끝냈다고 말하거나 반영 여부를 묻는다면 verify-discord-nickname 도구를 먼저 사용해 직접 확인해.", ].join("\n"), "discord-verify-question": - "- 디스코드 닉네임 확인 요청은 verify-discord-nickname 도구를 먼저 사용하고, 조회 결과만 짧게 답해.", + "- 디스코드 닉네임/주민신청 확인 요청은 verify-discord-nickname 도구와 check-onboarding-progress 도구를 먼저 사용하고, 조회 결과만 짧게 답해. 확인 없이 초대 안내만 반복하지 마. 닉네임이 틀렸거나 미설정이면 recommendedNickname/correctionGuide를 우선 사용해 정확히 어떻게 고칠지 안내해.", "help-where": "- 위치 안내가 모호할 때만 get-room-context 도구를 고려하고, 기본적으로는 절차 위치만 답해.", "status-question": diff --git a/src/skills/onboarding/runtime/critic.ts b/src/skills/onboarding/runtime/critic.ts index b880381..18bd421 100644 --- a/src/skills/onboarding/runtime/critic.ts +++ b/src/skills/onboarding/runtime/critic.ts @@ -4,7 +4,6 @@ import { z } from "zod"; import { getCriticMaxTokens, getLlmTemperature, - getTier2CriticModel, } from "../../../config.js"; import type { DerivedState } from "../state/onboarding-state.js"; import { getActiveProcessTrace } from "./trace.js"; @@ -147,7 +146,6 @@ export class OnboardingCritic { { deadlineMs: options?.deadlineMs, maxTokens: getCriticMaxTokens(), - modelOverride: getTier2CriticModel(), role: "critic", temperature: Math.max(0, getLlmTemperature() - 0.15), tier: 2, @@ -205,7 +203,6 @@ export class OnboardingCritic { deadlineMs: options.deadlineMs, jsonMode: true, maxTokens: getCriticMaxTokens(), - modelOverride: getTier2CriticModel(), role: "critic", temperature: Math.max(0, getLlmTemperature() - 0.15), tier: 2, diff --git a/src/skills/onboarding/runtime/orchestrator.test.ts b/src/skills/onboarding/runtime/orchestrator.test.ts index b8fff50..a740094 100644 --- a/src/skills/onboarding/runtime/orchestrator.test.ts +++ b/src/skills/onboarding/runtime/orchestrator.test.ts @@ -207,6 +207,317 @@ describe("OnboardingOrchestrator", () => { expect(calls[1]?.options?.tools).toBeUndefined(); }); + it("forces discord verification tools before answering completion reports", async () => { + const calls: Array<{ + messages: ChatMessage[]; + options: ChatCompletionOptions | undefined; + }> = []; + const executedTools: string[] = []; + const llmClientStub = { + chatCompletion: ( + messages: ChatMessage[], + options?: ChatCompletionOptions, + ) => { + calls.push({ messages, options }); + + if (calls.length === 1) { + return Promise.resolve({ + content: JSON.stringify({ + approach: "answer-direct", + targetPhase: "deadline-near", + mustInclude: [], + mustAvoid: [], + suggestedTools: [], + tone: "friendly", + targetLengthChars: 120, + rationale: "잘못된 직접 답변 계획", + }), + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "{}" } }] }, + message: { role: "assistant" as const, content: "{}" }, + }); + } + + return Promise.resolve({ + content: "디스코드 닉네임은 확인됐고, 주민신청 메시지도 확인됐어요.", + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "확인됐어요." } }] }, + message: { role: "assistant" as const, content: "확인됐어요." }, + }); + }, + }; + const verifyTool: Tool, { status: string }> = { + name: "verify-discord-nickname", + description: "디스코드 닉네임 확인", + argsSchema: z.object({}), + resultSchema: z.object({ + status: z.string(), + }), + execute: () => { + executedTools.push("verify-discord-nickname"); + return Promise.resolve({ status: "matched-renamed" }); + }, + }; + const progressTool: Tool< + { chatroomId: string; userId: string }, + { remainingSteps: string[] } + > = { + name: "check-onboarding-progress", + description: "진행 상태 확인", + argsSchema: z.object({ + chatroomId: z.string(), + userId: z.string(), + }), + resultSchema: z.object({ + remainingSteps: z.array(z.string()), + }), + execute: (args) => { + executedTools.push(`${args.chatroomId}:${args.userId}`); + return Promise.resolve({ remainingSteps: [] }); + }, + }; + + const orchestrator = new OnboardingOrchestrator(llmClientStub as never); + const result = await orchestrator.run( + createDerivedState({ + allowShortResponse: false, + intent: { kind: "discord-verify-question" }, + phase: { kind: "deadline-near", remaining: ["discord-profile-message"] }, + responseMode: "llm-with-tools", + tier: 1, + }), + [ + { role: "system", content: "기본 시스템 프롬프트" }, + { + role: "user", + content: "저 디코는 어떻게 들어가요? 주민신청까지 해뒀습니다", + }, + ], + { + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "프리리/95/여/D", + trigger: "user-message", + }, + toolContext: {}, + tools: [verifyTool, progressTool], + }, + ); + + expect(result).toContain("주민신청"); + expect(executedTools).toEqual([ + "verify-discord-nickname", + "room-1:user-1", + ]); + expect(calls[1]?.messages[0]?.content).toContain("[TOOL_PLAN_RESULTS]"); + expect(calls[1]?.messages[0]?.content).toContain("주민신청 메시지 확인 상태"); + expect(calls[1]?.messages[0]?.content).toContain( + "디스코드 닉네임 상태를 독립 항목처럼 길게 설명", + ); + expect(calls[1]?.options?.tools).toBeUndefined(); + }); + + it("does not expose verification tool again after partial discord tool-plan success", async () => { + const calls: Array<{ + messages: ChatMessage[]; + options: ChatCompletionOptions | undefined; + }> = []; + let verifyCallCount = 0; + const llmClientStub = { + chatCompletion: ( + messages: ChatMessage[], + options?: ChatCompletionOptions, + ) => { + calls.push({ messages, options }); + + if (calls.length === 1) { + return Promise.resolve({ + content: JSON.stringify({ + approach: "validate-then-explain", + targetPhase: "deadline-near", + mustInclude: [], + mustAvoid: [], + suggestedTools: ["verify-discord-nickname"], + tone: "friendly", + targetLengthChars: 120, + rationale: "닉네임 확인", + }), + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "{}" } }] }, + message: { role: "assistant" as const, content: "{}" }, + }); + } + + return Promise.resolve({ + content: "디스코드 닉네임은 확인됐어요.", + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "확인됐어요." } }] }, + message: { role: "assistant" as const, content: "확인됐어요." }, + }); + }, + }; + const verifyTool: Tool, { status: string }> = { + name: "verify-discord-nickname", + description: "디스코드 닉네임 확인", + argsSchema: z.object({}), + resultSchema: z.object({ + status: z.string(), + }), + execute: () => { + verifyCallCount += 1; + return Promise.resolve({ status: "matched-renamed" }); + }, + }; + + const orchestrator = new OnboardingOrchestrator(llmClientStub as never); + const result = await orchestrator.run( + createDerivedState({ + allowShortResponse: false, + intent: { kind: "discord-verify-question" }, + phase: { kind: "deadline-near", remaining: ["discord-profile-nickname"] }, + responseMode: "llm-with-tools", + tier: 1, + }), + [ + { role: "system", content: "기본 시스템 프롬프트" }, + { + role: "user", + content: "디스코드 닉네임 변경했어요", + }, + ], + { + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "프리리/95/여/D", + trigger: "user-message", + }, + toolContext: {}, + tools: [verifyTool], + }, + ); + + expect(result).toContain("닉네임"); + expect(verifyCallCount).toBe(1); + expect(calls[1]?.messages[0]?.content).toContain("[TOOL_PLAN_RESULTS]"); + expect(calls[1]?.messages[0]?.content).toContain("matched-renamed"); + expect(calls[1]?.messages[0]?.content).toContain( + "Tool not registered for step: check-onboarding-progress", + ); + expect(calls[1]?.options?.tools).toBeUndefined(); + }); + + it("focuses discord nickname answers when only nickname verification was asked", async () => { + const calls: Array<{ + messages: ChatMessage[]; + options: ChatCompletionOptions | undefined; + }> = []; + const llmClientStub = { + chatCompletion: ( + messages: ChatMessage[], + options?: ChatCompletionOptions, + ) => { + calls.push({ messages, options }); + + if (calls.length === 1) { + return Promise.resolve({ + content: JSON.stringify({ + approach: "validate-then-explain", + targetPhase: "deadline-near", + mustInclude: [], + mustAvoid: [], + suggestedTools: [], + tone: "friendly", + targetLengthChars: 120, + rationale: "닉네임 확인", + }), + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "{}" } }] }, + message: { role: "assistant" as const, content: "{}" }, + }); + } + + return Promise.resolve({ + content: "디스코드 닉네임은 확인됐어요.", + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "확인됐어요." } }] }, + message: { role: "assistant" as const, content: "확인됐어요." }, + }); + }, + }; + const verifyTool: Tool, { status: string }> = { + name: "verify-discord-nickname", + description: "디스코드 닉네임 확인", + argsSchema: z.object({}), + resultSchema: z.object({ + status: z.string(), + }), + execute: () => Promise.resolve({ status: "matched-renamed" }), + }; + const progressTool: Tool< + { chatroomId: string; userId: string }, + { remainingSteps: string[] } + > = { + name: "check-onboarding-progress", + description: "진행 상태 확인", + argsSchema: z.object({ + chatroomId: z.string(), + userId: z.string(), + }), + resultSchema: z.object({ + remainingSteps: z.array(z.string()), + }), + execute: () => Promise.resolve({ remainingSteps: [] }), + }; + + const orchestrator = new OnboardingOrchestrator(llmClientStub as never); + const result = await orchestrator.run( + createDerivedState({ + allowShortResponse: false, + intent: { kind: "discord-verify-question" }, + phase: { kind: "deadline-near", remaining: ["discord-profile-nickname"] }, + responseMode: "llm-with-tools", + tier: 1, + }), + [ + { role: "system", content: "기본 시스템 프롬프트" }, + { + role: "user", + content: "디스코드 닉네임 변경했어요", + }, + ], + { + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "프리리/95/여/D", + trigger: "user-message", + }, + toolContext: {}, + tools: [verifyTool, progressTool], + }, + ); + + expect(result).toContain("닉네임"); + expect(calls[1]?.messages[0]?.content).toContain("디스코드 닉네임 조회 결과"); + expect(calls[1]?.messages[0]?.content).toContain( + "주민신청 메시지 상태 언급", + ); + }); + it("injects requested SKILL.md resources before responder when enabled", async () => { process.env["AGENT_SKILL_MD_MODE"] = "true"; const calls: Array<{ diff --git a/src/skills/onboarding/runtime/orchestrator.ts b/src/skills/onboarding/runtime/orchestrator.ts index 7c252ba..4ed90b3 100644 --- a/src/skills/onboarding/runtime/orchestrator.ts +++ b/src/skills/onboarding/runtime/orchestrator.ts @@ -10,11 +10,8 @@ import { getPlannerMaxTokens, getTier0EstimatedCallMs, getTier0MaxTokens, - getTier0Model, getTier1EstimatedCallMs, getTier1MaxTokens, - getTier1PlannerModel, - getTier1ResponderModel, getTier2EstimatedCallMs, getTier2MaxTokens, } from "../../../config.js"; @@ -75,6 +72,48 @@ function summarizeMessages(messages: ChatMessage[]): string { .join("\n\n"); } +type DiscordVerificationFocus = + | "both" + | "nickname" + | "resident-application-message"; + +function getLatestUserMessage(messages: ChatMessage[]): string { + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (message?.role === "user") { + return message.content; + } + } + return ""; +} + +function getDiscordVerificationFocus( + derived: DerivedState, + messages: ChatMessage[], +): DiscordVerificationFocus { + if (derived.intent.kind !== "discord-verify-question") { + return "both"; + } + + const latestUserMessage = getLatestUserMessage(messages).toLowerCase(); + const asksResidentApplication = + /(주민\s*신청|주민신청|주민\s*등록|주민등록|프로필\s*메시지|채널\s*메시지|메시지\s*(작성|남김|보냄|썼|완료|했습니다|했어요))/u.test( + latestUserMessage, + ); + const asksNickname = + /(닉네임|닉변|서버\s*프로필|서버프로필|프로필\s*닉네임|서버\s*표시명|서버표시명)/u.test( + latestUserMessage, + ); + + if (asksResidentApplication && !asksNickname) { + return "resident-application-message"; + } + if (asksNickname && !asksResidentApplication) { + return "nickname"; + } + return "both"; +} + function tryParsePlannerOutput(raw: string): PlannerOutput | null { try { const parsed = JSON.parse(raw) as unknown; @@ -95,7 +134,24 @@ function tryParsePlannerOutput(raw: string): PlannerOutput | null { } } -function getDefaultPlan(derived: DerivedState): PlannerOutput { +function getDefaultPlan( + derived: DerivedState, + focus: DiscordVerificationFocus = "both", +): PlannerOutput { + if (derived.intent.kind === "discord-verify-question") { + return enforceRequiredToolPlan(derived, { + approach: "validate-then-explain", + targetPhase: derived.phase.kind, + mustInclude: getDiscordVerificationMustInclude(focus), + mustAvoid: getDiscordVerificationMustAvoid(focus), + requestResources: [], + suggestedTools: [], + tone: "friendly", + targetLengthChars: 150, + rationale: "discord-verification-fallback-plan", + }, focus); + } + return { approach: derived.responseMode === "llm-with-tools" @@ -115,6 +171,104 @@ function getDefaultPlan(derived: DerivedState): PlannerOutput { }; } +function hasToolPlanStep(plan: PlannerOutput, toolName: string): boolean { + return plan.toolPlan?.steps.some((step) => step.tool === toolName) ?? false; +} + +function getDiscordVerificationMustInclude( + focus: DiscordVerificationFocus, +): string[] { + if (focus === "nickname") { + return ["디스코드 닉네임 조회 결과"]; + } + if (focus === "resident-application-message") { + return [ + "주민신청 메시지 확인 상태", + "메시지 형식이 틀리면 서버프로필 닉네임과 동일하게 다시 작성하라는 안내", + ]; + } + return [ + "디스코드 닉네임 조회 결과", + "주민신청 메시지 확인 상태", + ]; +} + +function getDiscordVerificationMustAvoid( + focus: DiscordVerificationFocus, +): string[] { + const base = ["확인 없이 완료 단정", "초대 안내만 반복"]; + if (focus === "nickname") { + return [...base, "주민신청 메시지 상태 언급"]; + } + if (focus === "resident-application-message") { + return [ + ...base, + "디스코드 닉네임 상태를 독립 항목처럼 길게 설명", + "묻지 않은 남은 절차 나열", + ]; + } + return base; +} + +function enforceRequiredToolPlan( + derived: DerivedState, + plan: PlannerOutput, + focus: DiscordVerificationFocus = "both", +): PlannerOutput { + if (derived.intent.kind !== "discord-verify-question") { + return plan; + } + + const suggestedTools = [ + ...new Set([ + ...plan.suggestedTools, + "verify-discord-nickname" as const, + "check-onboarding-progress" as const, + ]), + ]; + const steps = [...(plan.toolPlan?.steps ?? [])]; + if (!hasToolPlanStep(plan, "verify-discord-nickname")) { + steps.unshift({ + id: "verify-discord", + tool: "verify-discord-nickname", + on: { error: "fallback-llm" }, + }); + } + if (!hasToolPlanStep(plan, "check-onboarding-progress")) { + steps.push({ + id: "check-progress", + tool: "check-onboarding-progress", + argsFromContext: { + chatroomId: "$request.chatroomId", + userId: "$request.userId", + }, + on: { error: "fallback-llm" }, + }); + } + + return { + ...plan, + approach: "validate-then-explain", + mustAvoid: [ + ...new Set([ + ...plan.mustAvoid, + ...getDiscordVerificationMustAvoid(focus), + ]), + ].slice(0, 6), + mustInclude: [ + ...new Set([ + ...plan.mustInclude, + ...getDiscordVerificationMustInclude(focus), + ]), + ].slice(0, 6), + suggestedTools: suggestedTools.slice(0, 3), + toolPlan: { + composeWith: "all-then-llm", + steps: steps.slice(0, 4), + }, + }; +} + function getRemainingDeadlineMs(): number | undefined { const deadlineAt = getContext()?.deadlineAt; if (!deadlineAt) { @@ -224,6 +378,49 @@ function buildToolPlanAddendum(result: ToolPlanResult): string { ].join("\n"); } +function hasToolPlanResultForTool( + plan: PlannerOutput, + result: ToolPlanResult, + toolName: string, +): boolean { + return ( + plan.toolPlan?.steps.some( + (step) => + step.tool === toolName && + Object.hasOwn(result.results, step.id), + ) ?? false + ); +} + +function shouldInjectToolPlanResult( + derived: DerivedState, + result: ToolPlanResult, +): boolean { + if (!result.fallbackToLlm) { + return true; + } + + return ( + derived.intent.kind === "discord-verify-question" && + Object.keys(result.results).length > 0 + ); +} + +function shouldSuppressResponderTools( + derived: DerivedState, + plan: PlannerOutput, + result: ToolPlanResult, +): boolean { + if (!result.fallbackToLlm) { + return true; + } + + return ( + derived.intent.kind === "discord-verify-question" && + hasToolPlanResultForTool(plan, result, "verify-discord-nickname") + ); +} + function pickToolsForPlan( plan: PlannerOutput, tools: readonly Tool[], @@ -254,7 +451,6 @@ export class OnboardingOrchestrator { options: OrchestratorRunOptions, responderTools: readonly Tool[], maxTokens: number, - modelOverride: string, ): Promise { const logger = getRequestLogger(); const deadlineMs = getRemainingDeadlineMs(); @@ -275,7 +471,6 @@ export class OnboardingOrchestrator { llmClient: this.llmClient, maxTokens, messages, - modelOverride, role: "responder", routingDecision: derived.routingDecision, tier: derived.tier, @@ -288,7 +483,6 @@ export class OnboardingOrchestrator { const response = await this.llmClient.chatCompletion(messages, { deadlineMs, maxTokens, - modelOverride, role: "responder", routingDecision: derived.routingDecision, temperature: options.temperature ?? getLlmTemperature(), @@ -304,24 +498,29 @@ export class OnboardingOrchestrator { ): Promise { const logger = getRequestLogger(); const traceStep = getActiveProcessTrace()?.startStep("plan"); + const verificationFocus = getDiscordVerificationFocus(derived, messages); const finishTrace = (plan: PlannerOutput): PlannerOutput => { + const normalizedPlan = enforceRequiredToolPlan( + derived, + plan, + verificationFocus, + ); traceStep?.finish({ - approach: plan.approach, - mustAvoid: plan.mustAvoid, - mustInclude: plan.mustInclude, - requestResources: plan.requestResources, - suggestedTools: plan.suggestedTools, - targetLengthChars: plan.targetLengthChars, - toolPlanStepCount: plan.toolPlan?.steps.length, + approach: normalizedPlan.approach, + mustAvoid: normalizedPlan.mustAvoid, + mustInclude: normalizedPlan.mustInclude, + requestResources: normalizedPlan.requestResources, + suggestedTools: normalizedPlan.suggestedTools, + targetLengthChars: normalizedPlan.targetLengthChars, + toolPlanStepCount: normalizedPlan.toolPlan?.steps.length, }); - return plan; + return normalizedPlan; }; const plannerMessages = buildPlannerMessages(derived, messages); const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, { deadlineMs: getRemainingDeadlineMs(), jsonMode: true, maxTokens: getPlannerMaxTokens(), - modelOverride: getTier1PlannerModel(), role: "planner", routingDecision: derived.routingDecision, temperature: 0, @@ -356,7 +555,6 @@ export class OnboardingOrchestrator { deadlineMs: getRemainingDeadlineMs(), jsonMode: true, maxTokens: getPlannerMaxTokens(), - modelOverride: getTier1PlannerModel(), role: "planner", routingDecision: derived.routingDecision, temperature: 0, @@ -376,7 +574,7 @@ export class OnboardingOrchestrator { return finishTrace(retryPlan); } - const fallbackPlan = getDefaultPlan(derived); + const fallbackPlan = getDefaultPlan(derived, verificationFocus); logger.debug( { intent: derived.intent.kind, @@ -403,7 +601,6 @@ export class OnboardingOrchestrator { options, tools, maxTokens, - getTier0Model(), ); } @@ -428,16 +625,19 @@ export class OnboardingOrchestrator { ); } const toolPlanResult = await this.runProgrammaticToolPlan( + derived, plan, finalResponderMessages, options, ); - if (toolPlanResult && !toolPlanResult.fallbackToLlm) { + if (toolPlanResult && shouldInjectToolPlanResult(derived, toolPlanResult)) { finalResponderMessages = injectSystemAddendum( finalResponderMessages, buildToolPlanAddendum(toolPlanResult), ); + } + if (toolPlanResult && shouldSuppressResponderTools(derived, plan, toolPlanResult)) { responderTools = []; } @@ -447,7 +647,6 @@ export class OnboardingOrchestrator { options, responderTools, options.maxTokens ?? getTier1MaxTokens(), - getTier1ResponderModel(), ); } @@ -472,16 +671,19 @@ export class OnboardingOrchestrator { ); } const toolPlanResult = await this.runProgrammaticToolPlan( + derived, plan, finalResponderMessages, options, ); - if (toolPlanResult && !toolPlanResult.fallbackToLlm) { + if (toolPlanResult && shouldInjectToolPlanResult(derived, toolPlanResult)) { finalResponderMessages = injectSystemAddendum( finalResponderMessages, buildToolPlanAddendum(toolPlanResult), ); + } + if (toolPlanResult && shouldSuppressResponderTools(derived, plan, toolPlanResult)) { responderTools = []; } @@ -491,7 +693,6 @@ export class OnboardingOrchestrator { options, responderTools, options.maxTokens ?? getTier2MaxTokens(), - getTier1ResponderModel(), ); if (!draft) { @@ -517,12 +718,19 @@ export class OnboardingOrchestrator { } private async runProgrammaticToolPlan( + derived: DerivedState, plan: PlannerOutput, _messages: ChatMessage[], options: OrchestratorRunOptions, ): Promise { if ( - !getAgentProgrammaticToolCalling() || + !getAgentProgrammaticToolCalling() && + derived.intent.kind !== "discord-verify-question" + ) { + return null; + } + + if ( !plan.toolPlan || !options.request || !options.tools?.length diff --git a/src/skills/onboarding/runtime/tool-selector.ts b/src/skills/onboarding/runtime/tool-selector.ts index a6cc3a7..07c7e4d 100644 --- a/src/skills/onboarding/runtime/tool-selector.ts +++ b/src/skills/onboarding/runtime/tool-selector.ts @@ -26,7 +26,11 @@ export function selectToolsForDerived( if (derived.intent.kind === 'discord-verify-question') { return maybeUseToolSearch( - tools.filter((tool) => tool.name === 'verify-discord-nickname'), + tools.filter( + (tool) => + tool.name === 'verify-discord-nickname' || + tool.name === 'check-onboarding-progress', + ), ); } diff --git a/src/skills/onboarding/state/classifier.test.ts b/src/skills/onboarding/state/classifier.test.ts index a24ed4b..09dd86e 100644 --- a/src/skills/onboarding/state/classifier.test.ts +++ b/src/skills/onboarding/state/classifier.test.ts @@ -52,6 +52,15 @@ describe("OnboardingIntentClassifier", () => { expect(result.source).toBe("regex"); }); + it("classifies resident application completion reports as discord verification", async () => { + const result = await classifier.classify( + "저 디코는 어떻게 들어가요? 주민신청까지 해뒀습니다", + "user-message", + ); + expect(result.intent.kind).toBe("discord-verify-question"); + expect(result.source).toBe("regex"); + }); + it("classifies identity room questions as help-how", async () => { const result = await classifier.classify( "본인인증 방은 뭐에요?", diff --git a/src/tools/moderation/lookup-rule.test.ts b/src/tools/moderation/lookup-rule.test.ts index ecbcb67..30ca422 100644 --- a/src/tools/moderation/lookup-rule.test.ts +++ b/src/tools/moderation/lookup-rule.test.ts @@ -1,6 +1,13 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; import type { MemoryService } from "../../memory/memory-types.js"; -import { lookupRuleTool } from "./lookup-rule.js"; +import { + lookupRuleTool, + resolveCommunityRulesPath, +} from "./lookup-rule.js"; const failingMemoryService = { longTerm: { @@ -9,6 +16,26 @@ const failingMemoryService = { } as unknown as MemoryService; describe("lookupRuleTool", () => { + it("falls back to the source rules resource when imported from dist", () => { + const root = mkdtempSync(join(tmpdir(), "community-rules-resource-")); + const sourceResourceDir = join( + root, + "src", + "skills", + "moderation", + "resources", + ); + mkdirSync(sourceResourceDir, { recursive: true }); + const sourceRulesPath = join(sourceResourceDir, "community-rules.md"); + writeFileSync(sourceRulesPath, "## ads\n광고 금지\n"); + + const moduleUrl = pathToFileURL( + join(root, "dist", "tools", "moderation", "lookup-rule.js"), + ).href; + + expect(resolveCommunityRulesPath(moduleUrl, root)).toBe(sourceRulesPath); + }); + it("returns the matching community rule section", async () => { const result = await lookupRuleTool.execute({ topic: "ads" }, {}); diff --git a/src/tools/moderation/lookup-rule.ts b/src/tools/moderation/lookup-rule.ts index ff1703a..d277e4c 100644 --- a/src/tools/moderation/lookup-rule.ts +++ b/src/tools/moderation/lookup-rule.ts @@ -1,4 +1,5 @@ -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { z } from "zod"; import { @@ -40,10 +41,31 @@ type RuleTopic = | "sanctions" | "spam"; -const RULES_PATH = fileURLToPath( - new URL("../../skills/moderation/resources/community-rules.md", import.meta.url), -); -const RULES_TEXT = readFileSync(RULES_PATH, "utf8"); +const RULES_RELATIVE_PATH = "skills/moderation/resources/community-rules.md"; + +export function resolveCommunityRulesPath( + moduleUrl = import.meta.url, + cwd = process.cwd(), +): string { + const moduleDir = dirname(fileURLToPath(moduleUrl)); + const candidates = [ + resolve(moduleDir, "../../", RULES_RELATIVE_PATH), + resolve(moduleDir, "../../../src", RULES_RELATIVE_PATH), + resolve(cwd, "dist", RULES_RELATIVE_PATH), + resolve(cwd, "src", RULES_RELATIVE_PATH), + ]; + const found = candidates.find((candidate) => existsSync(candidate)); + + if (!found) { + throw new Error( + `Missing moderation community rules resource. Tried: ${candidates.join(", ")}`, + ); + } + + return found; +} + +const RULES_TEXT = readFileSync(resolveCommunityRulesPath(), "utf8"); function extractRuleSection(topic: RuleTopic): string { const pattern = new RegExp(`^## ${topic}\\n([\\s\\S]*?)(?=\\n## |$)`, "mu"); diff --git a/src/tools/onboarding/check-onboarding-progress.test.ts b/src/tools/onboarding/check-onboarding-progress.test.ts index 71b7bf6..24a0b20 100644 --- a/src/tools/onboarding/check-onboarding-progress.test.ts +++ b/src/tools/onboarding/check-onboarding-progress.test.ts @@ -44,6 +44,8 @@ describe("checkOnboardingProgressTool", () => { expect(result.step1Done).toBe(true); expect(result.overallDone).toBe(false); + expect(result.discordProfileNicknameDone).toBe(false); + expect(result.discordProfileMessageDone).toBe(false); expect(result.remainingSteps).toEqual([ "discord-profile-nickname", "discord-profile-message", @@ -104,6 +106,8 @@ describe("checkOnboardingProgressTool", () => { expect(result.step1Done).toBe(true); expect(result.overallDone).toBe(true); + expect(result.discordProfileNicknameDone).toBe(null); + expect(result.discordProfileMessageDone).toBe(null); expect(result.remainingSteps).toEqual([]); }); }); diff --git a/src/tools/onboarding/check-onboarding-progress.ts b/src/tools/onboarding/check-onboarding-progress.ts index 10d1552..6fec443 100644 --- a/src/tools/onboarding/check-onboarding-progress.ts +++ b/src/tools/onboarding/check-onboarding-progress.ts @@ -18,6 +18,8 @@ const argsSchema = z.object({ }); const resultSchema = z.object({ + discordProfileMessageDone: z.boolean().nullable(), + discordProfileNicknameDone: z.boolean().nullable(), nicknameDone: z.boolean(), photoDone: z.boolean(), step1Done: z.boolean(), @@ -62,6 +64,8 @@ export const checkOnboardingProgressTool: Tool< : null; return { + discordProfileMessageDone: session?.discordProfileMessageVerified ?? null, + discordProfileNicknameDone: session?.discordProfileNicknameVerified ?? null, nicknameDone: session?.nicknameVerified ?? false, photoDone: session?.photoVerified ?? false, step1Done: isStep1Completed(session), diff --git a/src/tools/onboarding/verify-discord-nickname.test.ts b/src/tools/onboarding/verify-discord-nickname.test.ts index 6d1b767..3701c58 100644 --- a/src/tools/onboarding/verify-discord-nickname.test.ts +++ b/src/tools/onboarding/verify-discord-nickname.test.ts @@ -54,6 +54,16 @@ describe("verifyDiscordNicknameTool", () => { username: "priri", }, ], + residentApplicationMessageMatches: [ + { + authorId: "discord-1", + channelId: "resident-channel-1", + content: "프리리#KR1/95/E", + createdAt: "2026-04-17T00:12:00.000Z", + matchedBy: "author-id", + messageId: "message-1", + }, + ], joinedAfterMatches: [], }; const request = vi.fn( @@ -81,12 +91,88 @@ describe("verifyDiscordNicknameTool", () => { expect(result.status).toBe("matched-renamed"); expect(result.message).toContain("올바른 형식으로 설정되어 있어요"); + expect(result.message).toContain("주민신청 메시지도 확인됐어요"); expect(result.discordMemberId).toBe("discord-1"); + expect(result.residentApplicationMessageStatus).toBe("confirmed"); + expect(result.residentApplicationMessageId).toBe("message-1"); + expect(result.residentApplicationMessageContent).toBe("프리리#KR1/95/E"); + expect(result.residentApplicationMessageExpectedContent).toBe("프리리#KR1/95/E"); expect(session.discordMemberId).toBe("discord-1"); + expect(session.discordProfileNicknameVerified).toBe(true); + expect(session.discordProfileMessageVerified).toBe(true); + expect(session.discordProfileMessageVerifiedAt).toBe( + "2026-04-17T00:12:00.000Z", + ); expect(request).toHaveBeenCalledTimes(1); + expect(request.mock.calls[0]?.[1]).toMatchObject({ + includeResidentApplicationMessages: true, + searchName: "프리리", + }); expect(setJson).toHaveBeenCalledTimes(1); }); + it("does not confirm resident application messages that differ from the server nickname", async () => { + const setJson = vi.fn().mockResolvedValue(undefined); + const response: DiscordMemberLookupResponse = { + status: "ok", + nicknameMatches: [ + { + memberId: "discord-1", + joinedAt: "2026-04-17T00:10:00.000Z", + globalName: "프리리", + guildNickname: "프리리#KR1/95/E", + matchedBy: "guild-nickname", + username: "priri", + }, + ], + residentApplicationMessageMatches: [ + { + authorId: "discord-1", + channelId: "resident-channel-1", + content: "프리리/E", + createdAt: "2026-04-17T00:12:00.000Z", + matchedBy: "author-id", + messageId: "message-1", + }, + ], + joinedAfterMatches: [], + }; + const request = vi.fn( + ( + _subject: string, + _payload: TRequest, + _options?: { timeoutMs?: number }, + ): Promise => Promise.resolve(response as TResponse), + ); + const session = makeSession(); + + const result = await verifyDiscordNicknameTool.execute( + {}, + { + chatroomId: session.chatroomId, + garnetClient: { + setJson, + } as never, + natsClient: { + request: request as never, + }, + session, + userId: session.userId, + }, + ); + + expect(result.status).toBe("matched-renamed"); + expect(result.residentApplicationMessageStatus).toBe("content-mismatch"); + expect(result.residentApplicationMessageContent).toBe("프리리/E"); + expect(result.residentApplicationMessageExpectedContent).toBe("프리리#KR1/95/E"); + expect(result.residentApplicationMessageCorrectionGuide).toContain( + "주민신청 메시지는 서버프로필 닉네임과 동일하게 작성해야 해요", + ); + expect(result.message).toContain("현재 메시지는 '프리리/E'"); + expect(session.discordProfileNicknameVerified).toBe(true); + expect(session.discordProfileMessageVerified).toBeUndefined(); + }); + it("does not mark renamed when only the raw kakao nickname matches literally", async () => { const response: DiscordMemberLookupResponse = { status: "ok", @@ -121,7 +207,51 @@ describe("verifyDiscordNicknameTool", () => { ); expect(result.status).toBe("matched-pending-rename"); - expect(result.message).toContain("이름#태그/출생연도/티어"); + expect(result.message).toContain("디스코드 닉네임이 형식과 맞지 않습니다."); + expect(result.message).toContain("현재 소환사님의 서버프로필은 '프리리/95/여/E'"); + expect(result.message).toContain("닉네임#태그/출생연도/티어로 수정해 주세요."); + expect(result.recommendedNickname).toBe("프리리#태그/95/E"); + expect(result.correctionGuide).toBe(result.message); + expect(result.guidance).toContain("이름#태그/출생연도/티어"); + }); + + it("preserves an existing discord tag when suggesting a corrected nickname", async () => { + const response: DiscordMemberLookupResponse = { + status: "ok", + nicknameMatches: [ + { + memberId: "discord-1", + globalName: "프리리", + guildNickname: "프리리#KR1/95/여/E", + joinedAt: "2026-04-17T00:10:00.000Z", + matchedBy: "global-name", + username: "priri", + }, + ], + joinedAfterMatches: [], + }; + const request = vi.fn( + ( + _subject: string, + _payload: TRequest, + _options?: { timeoutMs?: number }, + ): Promise => Promise.resolve(response as TResponse), + ); + + const result = await verifyDiscordNicknameTool.execute( + {}, + { + natsClient: { + request: request as never, + }, + session: makeSession(), + }, + ); + + expect(result.status).toBe("matched-pending-rename"); + expect(result.recommendedNickname).toBe("프리리#KR1/95/E"); + expect(result.correctionGuide).toContain("현재 소환사님의 서버프로필은 '프리리#KR1/95/여/E'"); + expect(result.correctionGuide).toContain("할리#프렌치불독/88/D 와 같이"); }); it("falls back to joined-after candidates and returns not-confirmed when ambiguous", async () => { diff --git a/src/tools/onboarding/verify-discord-nickname.ts b/src/tools/onboarding/verify-discord-nickname.ts index 98ee2ff..dca0d0e 100644 --- a/src/tools/onboarding/verify-discord-nickname.ts +++ b/src/tools/onboarding/verify-discord-nickname.ts @@ -8,6 +8,7 @@ import type { DiscordMemberCandidate, DiscordMemberLookupRequest, DiscordMemberLookupResponse, + DiscordResidentApplicationMessageCandidate, } from '@common/nats-subjects'; import { getContext, getRequestLogger } from '@obs/context'; import { @@ -20,6 +21,7 @@ import { getKakaotalkNickname, matchesDiscordNicknameRule, parseKakaoNickname, + type ParsedKakaoNickname, } from '@skills/onboarding/session'; import type { Tool, ToolContext } from '../tool.js'; @@ -50,6 +52,8 @@ const resultSchema = z.object({ searchName: z.string().optional(), discordMemberId: z.string().optional(), currentDisplayName: z.string().optional(), + recommendedNickname: z.string().optional(), + correctionGuide: z.string().optional(), formatExample: z.string().optional(), guidance: z.string().optional(), issueMessages: z.array(z.string()).optional(), @@ -57,6 +61,19 @@ const resultSchema = z.object({ matchSource: z.enum(['joined-after', 'nickname']).optional(), matchedBy: z.enum(['global-name', 'guild-nickname', 'joined-after', 'username']).optional(), nicknameMatchCount: z.number().int().nonnegative().optional(), + residentApplicationMessageAuthorId: z.string().optional(), + residentApplicationMessageContent: z.string().optional(), + residentApplicationMessageCorrectionGuide: z.string().optional(), + residentApplicationMessageCreatedAt: z.string().optional(), + residentApplicationMessageExpectedContent: z.string().optional(), + residentApplicationMessageId: z.string().optional(), + residentApplicationMessageMatchCount: z.number().int().nonnegative().optional(), + residentApplicationMessageMatchedBy: z + .enum(['author-id', 'author-name', 'content', 'joined-after']) + .optional(), + residentApplicationMessageStatus: z + .enum(['ambiguous', 'confirmed', 'content-mismatch', 'lookup-unavailable', 'not-confirmed']) + .optional(), }); function pickDisplayName(candidate: DiscordMemberCandidate): string { @@ -65,34 +82,249 @@ function pickDisplayName(candidate: DiscordMemberCandidate): string { ); } -function resolveTimeoutMs(): number | null { +function extractExistingDiscordTag( + currentGuildNickname: string | undefined, + parsed: ParsedKakaoNickname, +): string | null { + const normalized = currentGuildNickname?.trim(); + if (!normalized) { + return null; + } + + const prefix = `${parsed.baseName}#`; + if (!normalized.startsWith(prefix)) { + return null; + } + + const tag = normalized.slice(prefix.length).split('/')[0]?.trim(); + return tag || null; +} + +function buildRecommendedNickname( + parsed: ParsedKakaoNickname, + currentGuildNickname: string | undefined, +): string { + const tag = extractExistingDiscordTag(currentGuildNickname, parsed) ?? '태그'; + return `${parsed.baseName}#${tag}/${parsed.year}/${parsed.tier}`; +} + +function buildCorrectionGuide(args: { + currentDisplayName: string; + currentGuildNickname?: string | undefined; +}): string { + const formatGuide = `${DISCORD_NICKNAME_FORMAT_EXAMPLE} 와 같이 닉네임#태그/출생연도/티어로 수정해 주세요.`; + + if (args.currentGuildNickname?.trim()) { + return `디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '${args.currentGuildNickname.trim()}' 으로 확인됩니다. ${formatGuide}`; + } + + return `디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '${args.currentDisplayName}' 으로 확인됩니다. ${formatGuide}`; +} + +type ResidentApplicationMessageStatus = + | 'ambiguous' + | 'confirmed' + | 'content-mismatch' + | 'lookup-unavailable' + | 'not-confirmed'; + +interface ResidentApplicationMessageResult { + candidate?: DiscordResidentApplicationMessageCandidate | undefined; + correctionGuide?: string | undefined; + expectedContent?: string | undefined; + matchCount?: number | undefined; + status: ResidentApplicationMessageStatus; +} + +function resolveResidentApplicationMessageResult( + matches: DiscordResidentApplicationMessageCandidate[] | undefined, +): ResidentApplicationMessageResult { + if (!matches) { + return { status: 'lookup-unavailable' }; + } + + if (matches.length === 0) { + return { matchCount: 0, status: 'not-confirmed' }; + } + + if (matches.length > 1) { + return { matchCount: matches.length, status: 'ambiguous' }; + } + + return { + candidate: matches[0], + matchCount: 1, + status: 'confirmed', + }; +} + +function normalizeMessageContent(content: string | undefined): string { + return content?.replace(/\s+/gu, ' ').trim() ?? ''; +} + +function buildResidentApplicationMessageCorrectionGuide(args: { + actualContent: string; + expectedContent: string; +}): string { + return `주민신청 메시지는 서버프로필 닉네임과 동일하게 작성해야 해요. 현재 메시지는 '${args.actualContent}' 로 확인됩니다. '${args.expectedContent}' 로 다시 남겨주세요.`; +} + +function validateResidentApplicationMessageContent( + result: ResidentApplicationMessageResult, + expectedContent: string | undefined, +): ResidentApplicationMessageResult { + if (result.status !== 'confirmed' || !result.candidate || !expectedContent) { + return expectedContent ? { ...result, expectedContent } : result; + } + + const actualContent = normalizeMessageContent(result.candidate.content); + const normalizedExpected = normalizeMessageContent(expectedContent); + if (actualContent === normalizedExpected) { + return { + ...result, + expectedContent: normalizedExpected, + }; + } + + return { + ...result, + correctionGuide: buildResidentApplicationMessageCorrectionGuide({ + actualContent, + expectedContent: normalizedExpected, + }), + expectedContent: normalizedExpected, + status: 'content-mismatch', + }; +} + +function truncateMessageContent(content: string): string { + const normalized = content.replace(/\s+/gu, ' ').trim(); + return normalized.length > 160 ? `${normalized.slice(0, 157)}...` : normalized; +} + +function buildResidentApplicationMessageSummary( + result: ResidentApplicationMessageResult, +): string | null { + if (result.status === 'lookup-unavailable') { + return null; + } + + if (result.status === 'confirmed' && result.candidate) { + return `주민신청 메시지도 확인됐어요.`; + } + + if (result.status === 'content-mismatch' && result.correctionGuide) { + return result.correctionGuide; + } + + if (result.status === 'ambiguous') { + return '주민신청 메시지 후보가 여러 개라 하나로 확정하지 못했어요.'; + } + + return '주민신청 채널 메시지는 아직 확인되지 않았어요.'; +} + +function residentApplicationMessageFields( + result: ResidentApplicationMessageResult, +): Pick< + z.infer, + | 'residentApplicationMessageAuthorId' + | 'residentApplicationMessageContent' + | 'residentApplicationMessageCorrectionGuide' + | 'residentApplicationMessageCreatedAt' + | 'residentApplicationMessageExpectedContent' + | 'residentApplicationMessageId' + | 'residentApplicationMessageMatchCount' + | 'residentApplicationMessageMatchedBy' + | 'residentApplicationMessageStatus' +> { + const candidate = result.candidate; + return { + residentApplicationMessageCorrectionGuide: result.correctionGuide, + residentApplicationMessageExpectedContent: result.expectedContent, + residentApplicationMessageMatchCount: result.matchCount, + residentApplicationMessageStatus: result.status, + ...(candidate + ? { + residentApplicationMessageAuthorId: candidate.authorId, + residentApplicationMessageContent: truncateMessageContent(candidate.content), + residentApplicationMessageCreatedAt: candidate.createdAt, + residentApplicationMessageId: candidate.messageId, + residentApplicationMessageMatchedBy: candidate.matchedBy, + } + : {}), + }; +} + +interface TimeoutBudget { + configuredTimeoutMs: number; + deadlineAt?: string | undefined; + remainingMs?: number | undefined; + timeoutMs: number | null; +} + +function resolveTimeoutBudget(): TimeoutBudget { const configured = getDiscordVerifyTimeoutMs(); const deadlineAt = getContext()?.deadlineAt; if (!deadlineAt) { - return configured; + return { + configuredTimeoutMs: configured, + timeoutMs: configured, + }; } const remainingMs = Date.parse(deadlineAt) - Date.now() - 100; if (!Number.isFinite(remainingMs) || remainingMs <= 0) { - return null; + return { + configuredTimeoutMs: configured, + deadlineAt, + remainingMs, + timeoutMs: null, + }; } - return Math.min(configured, remainingMs); + return { + configuredTimeoutMs: configured, + deadlineAt, + remainingMs, + timeoutMs: Math.min(configured, remainingMs), + }; } -async function persistDiscordMemberId(memberId: string, ctx: ToolContext): Promise { +async function persistDiscordVerification( + input: { + memberId: string; + nicknameVerified: boolean; + residentApplicationMessageVerifiedAt?: string | undefined; + }, + ctx: ToolContext, +): Promise { if (!ctx.garnetClient || !ctx.session) { return; } + const now = new Date().toISOString(); const nextSession = { ...ctx.session, - discordMemberId: memberId, + discordMemberId: input.memberId, + ...(input.nicknameVerified + ? { + discordProfileNicknameVerified: true, + discordProfileNicknameVerifiedAt: + ctx.session.discordProfileNicknameVerifiedAt ?? now, + } + : {}), + ...(input.residentApplicationMessageVerifiedAt + ? { + discordProfileMessageVerified: true, + discordProfileMessageVerifiedAt: input.residentApplicationMessageVerifiedAt, + } + : {}), }; const key = `${getOnboardingSessionKeyPrefix()}:onboarding:${nextSession.chatroomId}:${nextSession.userId}`; await ctx.garnetClient.setJson(key, nextSession); - ctx.session.discordMemberId = memberId; + Object.assign(ctx.session, nextSession); } export const verifyDiscordNicknameTool: Tool< @@ -146,8 +378,7 @@ export const verifyDiscordNicknameTool: Tool< } const parsedKakaoNickname = parseKakaoNickname(expectedNickname); - const searchName = parsedKakaoNickname?.baseName ?? ''; - if (!searchName) { + if (!parsedKakaoNickname?.baseName) { logger.debug( { expectedNickname, @@ -161,6 +392,7 @@ export const verifyDiscordNicknameTool: Tool< expectedNickname, }; } + const searchName = parsedKakaoNickname.baseName; if (!ctx.natsClient) { logger.debug( @@ -179,11 +411,15 @@ export const verifyDiscordNicknameTool: Tool< }; } - const timeoutMs = resolveTimeoutMs(); + const timeoutBudget = resolveTimeoutBudget(); + const timeoutMs = timeoutBudget.timeoutMs; if (!timeoutMs) { logger.debug( { + configuredTimeoutMs: timeoutBudget.configuredTimeoutMs, + deadlineAt: timeoutBudget.deadlineAt, expectedNickname, + remainingMs: timeoutBudget.remainingMs, searchName, }, '[VerifyDiscordNickname] No timeout budget left for lookup', @@ -201,6 +437,7 @@ export const verifyDiscordNicknameTool: Tool< try { const payload: DiscordMemberLookupRequest = { chatroomId: ctx.chatroomId ?? ctx.session.chatroomId, + includeResidentApplicationMessages: true, joinedAfter: ctx.session.joinedAt, searchName, userId: ctx.userId ?? ctx.session.userId, @@ -209,6 +446,9 @@ export const verifyDiscordNicknameTool: Tool< logger.debug( { payload, + configuredTimeoutMs: timeoutBudget.configuredTimeoutMs, + deadlineAt: timeoutBudget.deadlineAt, + remainingMs: timeoutBudget.remainingMs, subject: getDiscordMemberLookupSubject(), timeoutMs, }, @@ -257,11 +497,18 @@ export const verifyDiscordNicknameTool: Tool< const nicknameMatches = lookupResult.nicknameMatches; const joinedAfterMatches = lookupResult.joinedAfterMatches; + const unvalidatedResidentApplicationMessageResult = resolveResidentApplicationMessageResult( + lookupResult.residentApplicationMessageMatches, + ); logger.debug( { expectedNickname, joinedAfterMatches: joinedAfterMatches.length, nicknameMatches: nicknameMatches.length, + residentApplicationMessageMatchCount: + unvalidatedResidentApplicationMessageResult.matchCount, + residentApplicationMessageStatus: + unvalidatedResidentApplicationMessageResult.status, searchName, }, '[VerifyDiscordNickname] Lookup response received', @@ -293,6 +540,10 @@ export const verifyDiscordNicknameTool: Tool< expectedNickname, joinedAfterMatchCount: joinedAfterMatches.length, nicknameMatchCount: nicknameMatches.length, + residentApplicationMessageMatchCount: + unvalidatedResidentApplicationMessageResult.matchCount, + residentApplicationMessageStatus: + unvalidatedResidentApplicationMessageResult.status, reason: reason ?? (nicknameMatches.length === 0 ? 'nickname-search-none' : 'nickname-search-ambiguous'), @@ -310,29 +561,63 @@ export const verifyDiscordNicknameTool: Tool< searchName, joinedAfterMatchCount: joinedAfterMatches.length, nicknameMatchCount: nicknameMatches.length, + ...residentApplicationMessageFields(unvalidatedResidentApplicationMessageResult), }; } - await persistDiscordMemberId(candidate.memberId, ctx); - const currentDisplayName = pickDisplayName(candidate); const currentGuildNickname = candidate.guildNickname?.trim(); const isRenamed = matchesDiscordNicknameRule(currentGuildNickname, expectedNickname); const status = isRenamed ? 'matched-renamed' : 'matched-pending-rename'; + const recommendedNickname = buildRecommendedNickname(parsedKakaoNickname, currentGuildNickname); + const residentApplicationMessageExpectedContent = isRenamed + ? currentGuildNickname + : recommendedNickname.includes('#태그/') + ? undefined + : recommendedNickname; + const residentApplicationMessageResult = validateResidentApplicationMessageContent( + unvalidatedResidentApplicationMessageResult, + residentApplicationMessageExpectedContent, + ); + await persistDiscordVerification( + { + memberId: candidate.memberId, + nicknameVerified: isRenamed, + residentApplicationMessageVerifiedAt: + residentApplicationMessageResult.status === 'confirmed' + ? residentApplicationMessageResult.candidate?.createdAt + : undefined, + }, + ctx, + ); + + const correctionGuide = buildCorrectionGuide({ + currentDisplayName, + currentGuildNickname, + }); const discordFormatGuide = `디스코드 서버 닉네임은 이름#태그/출생연도/티어 형식이에요. 카카오톡 형식과 달리 성별은 빼고 태그(#)가 들어가요. 예: ${DISCORD_NICKNAME_FORMAT_EXAMPLE}`; + const residentApplicationMessageSummary = buildResidentApplicationMessageSummary( + residentApplicationMessageResult, + ); const message = isRenamed ? '디스코드 서버 닉네임이 올바른 형식으로 설정되어 있어요. 확인 완료!' - : currentGuildNickname - ? `디스코드에서 유저를 찾았지만 서버 닉네임이 아직 올바른 형식으로 확인되지는 않았어요. 현재 서버 닉네임은 "${currentGuildNickname}"으로 보여요. ${discordFormatGuide}` - : `디스코드에서 유저를 찾았지만 서버 닉네임이 아직 설정되지 않았거나 확인되지 않았어요. 현재 표시 이름은 "${currentDisplayName}"으로 보여요. ${discordFormatGuide}`; + : correctionGuide; + const finalMessage = [message, residentApplicationMessageSummary] + .filter(Boolean) + .join(' '); logger.debug( { currentDisplayName, + currentGuildNickname, discordMemberId: candidate.memberId, expectedNickname, matchSource, matchedBy: candidate.matchedBy, + recommendedNickname, + residentApplicationMessageMatchCount: + residentApplicationMessageResult.matchCount, + residentApplicationMessageStatus: residentApplicationMessageResult.status, status, }, '[VerifyDiscordNickname] Verification resolved', @@ -340,12 +625,16 @@ export const verifyDiscordNicknameTool: Tool< return { status, - message, + message: finalMessage, currentDisplayName, + correctionGuide: isRenamed ? undefined : correctionGuide, discordMemberId: candidate.memberId, expectedNickname, + guidance: isRenamed ? undefined : discordFormatGuide, matchSource, matchedBy: candidate.matchedBy, + recommendedNickname: isRenamed ? undefined : recommendedNickname, + ...residentApplicationMessageFields(residentApplicationMessageResult), searchName, }; },