From 8156e8c07ee67db0f461dded5851cd7ec695e798 Mon Sep 17 00:00:00 2001 From: dahyeon Date: Mon, 18 May 2026 19:33:11 -0700 Subject: [PATCH] Close multi-agent plan and add router shadow reporting --- CLAUDE.md | 44 +++ docs/multi-agent-architecture.md | 178 ++++++++++++ docs/optimization-roadmap.md | 66 +++++ docs/resolution-hash-sunset-procedure.md | 101 +++++++ docs/service-evaluation-operations.md | 305 ++++++++++++++------- package.json | 1 + src/config.test.ts | 10 + src/config.ts | 44 +-- src/eval/cli.ts | 167 ++++++----- src/eval/router-shadow-report.test.ts | 178 ++++++++++++ src/eval/router-shadow-report.ts | 249 +++++++++++++++++ src/skills/supervisor/capability-prompt.ts | 33 ++- src/skills/supervisor/schema.ts | 1 + src/skills/supervisor/skill.test.ts | 78 +++++- src/skills/supervisor/skill.ts | 52 +++- 15 files changed, 1303 insertions(+), 204 deletions(-) create mode 100644 CLAUDE.md create mode 100644 docs/multi-agent-architecture.md create mode 100644 docs/optimization-roadmap.md create mode 100644 docs/resolution-hash-sunset-procedure.md create mode 100644 src/eval/router-shadow-report.test.ts create mode 100644 src/eval/router-shadow-report.ts diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..1c2bac1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,44 @@ +# CLAUDE.md + +`lolympus-agent`는 NATS 메시지 드리븐 멀티 에이전트 시스템입니다. 자세한 구조는 [docs/multi-agent-architecture.md](docs/multi-agent-architecture.md)를 먼저 읽으세요. + +## 작업 시 자주 쓰는 명령 + +| 명령 | 용도 | +| ------------------------------------------------- | ------------------------------------------ | +| `npm run typecheck` | TypeScript 타입 검증 | +| `npm run lint` | ESLint와 SKILL.md 일관성 검증 | +| `npm run lint:skills` | SKILL.md와 코드 manifest 검증 | +| `npm test` | 전체 테스트 | +| `npm run eval -- --set=onboarding/cases` | onboarding golden 평가 | +| `npm run eval:moderation -- --skip-llm-judge` | moderation golden smoke | +| `npm run eval:handoff -- --skip-llm-judge` | handoff 경로 회귀 | +| `npm run eval:router-shadow-report -- --since=7d` | static router와 learned router shadow 비교 | +| `npm run review:list -- --limit=5` | supervisor review queue 확인 | + +## 핵심 디렉토리 + +- `src/skills//` - skill별 단일 정의 위치. `SKILL.md`, intent, prompt data, tools, runtime을 함께 둔다. +- `src/agent/dispatch.ts` - production과 eval이 공유하는 handoff dispatch loop. +- `src/eval/` - golden, shadow archive, proposal, tournament, router training/reporting. +- `src/operations/` - supervisor review queue 운영 CLI. +- `src/observability/` - logger, tracing, metrics, alert, Slack notifier. +- `docs/service-evaluation-operations.md` - 운영 runbook. + +## 변경 시 주의 + +- SKILL.md를 바꾸면 `npm run lint:skills`를 실행한다. +- prompt, routing, handoff를 바꾸면 onboarding, moderation, handoff eval smoke를 함께 확인한다. +- 새 skill은 `src/skills//`에 정의하고 factory/registry에 등록한 뒤 golden case를 추가한다. +- plan 문서(`docs/refactor-skill-ssot*.md`)는 작업 지시와 기록이다. 공개 main에 병합할 때는 별도 요청이 없는 한 제외한다. + +## 운영 + +- Review queue 처리: [docs/service-evaluation-operations.md](docs/service-evaluation-operations.md) +- Architecture: [docs/multi-agent-architecture.md](docs/multi-agent-architecture.md) +- 고도화 시작 조건: [docs/optimization-roadmap.md](docs/optimization-roadmap.md) +- Resolution hash sunset: [docs/resolution-hash-sunset-procedure.md](docs/resolution-hash-sunset-procedure.md) + +## 상태 + +운영 가능한 멀티 에이전트 기반은 Iter8 (2026-05-18)에 완료되었습니다. 이후 고도화는 [docs/optimization-roadmap.md](docs/optimization-roadmap.md)의 trigger 조건에 따라 별도 track으로 진행합니다. diff --git a/docs/multi-agent-architecture.md b/docs/multi-agent-architecture.md new file mode 100644 index 0000000..ed1988a --- /dev/null +++ b/docs/multi-agent-architecture.md @@ -0,0 +1,178 @@ +# lolympus-agent Multi-Agent Architecture + +Last verified: Iter8 implementation branch, 2026-05-18. + +## 1. 개요 + +`lolympus-agent`는 Lolympus 커뮤니티 운영을 돕는 NATS 메시지 드리븐 멀티 skill agent다. 실시간 요청은 `Agent.handleRequest()`로 들어오고, skill registry가 요청 skill을 찾아 `Skill.handle()`을 실행한다. 품질 관리는 shadow evaluation, MongoDB archive, human review, proposal/tournament loop로 이어진다. + +운영 목표는 세 가지다. + +- 실시간 응답 경로는 짧고 예측 가능하게 유지한다. +- skill, prompt, intent, resource, tool 정의는 각 `src/skills//` 폴더에 모은다. +- production과 eval은 같은 dispatch, handoff, budget 규칙을 사용한다. + +## 2. 아키텍처 다이어그램 + +### 실시간 메시지 흐름 + +```text +NATS lolympus-agent.request + -> src/main.ts + -> Agent.handleRequest() + -> src/agent/dispatch.ts + -> SkillRegistry.get(request.skill) + -> Skill.handle(request, history) + -> AgentResponse +``` + +### Handoff 흐름 + +```text +Skill response + -> control token/parser or metadata.handoffRequest + -> transition-matrix validation + -> BudgetTracker + -> buildHandoffPacket() + -> dispatch target skill + -> supervisor fallback when blocked +``` + +`src/agent/dispatch.ts`는 production과 eval에서 모두 쓰인다. handoff chain, depth, last request는 metadata와 logger context에 남는다. + +### Eval 흐름 + +```text +real-time response + -> ShadowEvalBus + -> lolympus-agent:background + -> ShadowWorker + -> lolympus-agent:eval:results + -> aggregation/archive/drift/review + -> propose/tournament/promote +``` + +### Operation 흐름 + +```text +supervisor handoff + -> lolympus-agent:review-queue + -> review:list/show/resolve/stats + -> resolution stream + compatibility hash + -> Slack/NATS alert +``` + +## 3. Skill 모델 + +모든 skill은 `src/skills//` 아래에 둔다. 필수 진입점은 다음과 같다. + +| 파일 | 역할 | +| --------------------------------- | --------------------------------------- | +| `SKILL.md` | 사람이 읽는 capability card와 운영 설명 | +| `skill.ts` | `Skill` 구현체 | +| `intents.ts` | skill이 해석하는 intent 종류 | +| `prompt-data.ts` 또는 `prompts/*` | prompt layer, 예시, resource 연결 | +| `tools.ts` 또는 `resources/*` | skill 전용 tool/resource 정의 | + +`src/skills/skill-loader.ts`가 `SKILL.md`를 읽어 `SkillManifest`로 만들고, `src/skills/validate-manifests.ts`가 문서와 코드의 일관성을 검사한다. `handoffTargets`는 `src/skills/transition-matrix.ts`에서 검증되며 self-loop와 unknown target을 막는다. + +현재 핵심 skill은 `onboarding`, `moderation`, `supervisor`다. `supervisor`는 모든 skill의 fallback target이며, specialist가 처리하지 못한 요청을 review queue로 넘긴다. + +## 4. Dispatch 모델 + +Dispatch는 `src/agent/dispatch.ts`가 담당한다. + +- 응답 text 안의 control token은 parser가 제거하고 metadata로 구조화한다. +- skill이 직접 `metadata.handoffRequest`를 넣어도 같은 path로 처리된다. +- `HandoffPacket`은 원 요청, source response, handoff reason, evidence, source chain을 보존한다. +- `BudgetTracker`는 최대 handoff 수, 동일 skill 재방문, tool call, 비용 한도를 강제한다. +- transition matrix가 허용하지 않는 target은 supervisor로 reroute된다. +- budget이 막히면 `same-agent-revisit`, `max-hop`, `high-risk-action` 같은 reason으로 supervisor fallback이 만들어진다. + +이 구조 덕분에 eval handoff golden도 실제 production dispatch와 같은 규칙을 검증한다. + +## 5. Eval 모델 + +`src/eval/`은 네 계층으로 나뉜다. + +| 계층 | 주요 파일 | 설명 | +| ------ | ------------------------------------ | ----------------------------------- | +| Golden | `src/eval/golden/*/cases.jsonl` | 사람이 관리하는 고정 회귀 케이스 | +| Runner | `runner.ts`, `handoff-dispatch.ts` | skill-aware eval 실행 | +| Judges | `judges/*` | rule, LLM, process, handoff 판단 | +| Loop | `archive.ts`, `promote.ts`, `loop/*` | archive 기반 개선 후보와 tournament | + +Router 학습은 `src/eval/router-training.ts`가 MongoDB archive에서 training sample을 만들고, `src/skills/onboarding/runtime/router.ts`가 learned model을 저장한다. Iter8에서 추가된 `src/eval/router-shadow-report.ts`는 supervisor outcome과 shadow eval을 조인해 static router와 learned router의 divergence를 보고한다. + +## 6. Observability + +관측성은 세 갈래다. + +- Logger context: requestId, skill, handoff chain, depth를 포함한다. +- Metrics: `src/observability/metrics.ts`가 Prometheus text exposition을 제공한다. +- Alerts: `src/observability/alert.ts`가 NATS와 Slack으로 운영 이벤트를 발행한다. + +대표 metric: + +| Metric | 용도 | +| ----------------------------- | ----------------------------- | +| `agent_handoff_total` | skill 간 handoff rate | +| `agent_handoff_loop_total` | loop guardrail 발생 | +| `agent_human_override_total` | supervisor/manual review 진입 | +| `agent_validator_fail_total` | validator 실패 | +| `agent_time_to_resolution_ms` | skill별 처리 시간 | +| `agent_token_cost_usd` | model/skill 비용 추정 | + +`METRICS_PORT`의 `/metrics`와 `/healthz`를 scrape할 수 있다. Slack alert는 `SLACK_ALERT_WEBHOOK_URL`이 있을 때만 전송된다. + +## 7. 데이터 저장소 + +### Garnet/Redis + +| Key | Type | 용도 | +| ----------------------------------- | ------ | ------------------------------ | +| `lolympus-agent:background` | Stream | shadow eval과 summary job | +| `lolympus-agent:eval:results` | Stream | shadow eval 결과 | +| `lolympus-agent:eval:agg:*` | Hash | slice별 aggregate | +| `lolympus-agent:eval:review:queue` | Stream | eval human review | +| `lolympus-agent:review-queue` | Stream | supervisor review | +| `lolympus-agent:review-resolutions` | Stream | supervisor resolution audit | +| `lolympus-agent:review-resolved:*` | Hash | legacy compatibility read path | +| `lolympus-agent:alert:dedupe:*` | String | alert dedupe/rate limit | + +### MongoDB + +| Collection | 용도 | +| -------------------- | ------------------------------ | +| `eval_archive` | shadow eval 장기 archive | +| `eval_review_labels` | 사람 review label archive | +| `eval_tournaments` | prompt variant tournament 결과 | + +Conversation, blackboard, summary state는 Garnet key와 TTL로 관리한다. + +## 8. 확장 가이드 + +새 skill을 추가할 때는 다음 순서로 진행한다. + +1. `src/skills//` 폴더를 만들고 `SKILL.md`, `skill.ts`, 필요한 `intents.ts`, `prompt-data.ts`, `tools.ts`를 둔다. +2. `SKILL.md`에 `name`, `domain`, `riskTier`, `canHandle`, `cannotHandle`, `handoffTargets`, `outputContract`, `triggers`를 작성한다. +3. factory/registry에 skill을 등록하고 transition matrix가 target을 인식하게 한다. +4. 필요한 resource는 skill 폴더의 `resources/`에 두고 prompt builder가 읽게 한다. +5. 최소 golden case를 `src/eval/golden//cases.jsonl`에 추가한다. +6. `npm run lint:skills`, `npm test`, 해당 skill eval smoke를 실행한다. +7. handoff가 필요한 skill이면 `src/eval/golden/handoff/cases.jsonl`에 교차 skill 케이스를 추가한다. + +## 9. References + +| 문서 | 의미 | +| ------------------------------------------------------------ | ------------------------------------------------------- | +| [refactor-skill-ssot.md](refactor-skill-ssot.md) | Iter1, skill/prompt/intent SSOT 시작 | +| [refactor-skill-ssot-iter2.md](refactor-skill-ssot-iter2.md) | Iter2, tool/eval path skill-aware화 | +| [refactor-skill-ssot-iter3.md](refactor-skill-ssot-iter3.md) | Iter3, eval pipeline과 SKILL.md 검증 강화 | +| [refactor-skill-ssot-iter4.md](refactor-skill-ssot-iter4.md) | Iter4, 멀티 에이전트 운영 조건 정의 | +| [refactor-skill-ssot-iter5.md](refactor-skill-ssot-iter5.md) | Iter5, bounded graph와 structured handoff | +| [refactor-skill-ssot-iter6.md](refactor-skill-ssot-iter6.md) | Iter6, supervisor review loop와 router training signals | +| [refactor-skill-ssot-iter7.md](refactor-skill-ssot-iter7.md) | Iter7, observability와 Slack alert | +| [refactor-skill-ssot-iter8.md](refactor-skill-ssot-iter8.md) | Iter8, closing iteration | + +운영 runbook은 [service-evaluation-operations.md](service-evaluation-operations.md), 고도화 trigger는 [optimization-roadmap.md](optimization-roadmap.md), resolution hash sunset은 [resolution-hash-sunset-procedure.md](resolution-hash-sunset-procedure.md)를 참고한다. diff --git a/docs/optimization-roadmap.md b/docs/optimization-roadmap.md new file mode 100644 index 0000000..3d43e74 --- /dev/null +++ b/docs/optimization-roadmap.md @@ -0,0 +1,66 @@ +# Optimization Roadmap + +Last verified: Iter8 implementation branch, 2026-05-18. + +운영 가능한 멀티 에이전트 기반은 Iter8에서 닫았다. 이 문서는 다음 고도화를 추측으로 시작하지 않기 위한 trigger 표다. Trigger는 권장 시작점이며, 최종 착수는 사람이 실제 비용, 운영 부담, 사용자 영향도를 보고 결정한다. + +## Trigger Summary + +| 항목 | Trigger 조건 | 측정 지표 | Plan 작성 시작 시점 | +| ------------------------- | ------------------------------------------------------- | ----------------------------------------- | ------------------------------- | +| Cost-aware routing | 월 LLM 비용이 운영 예산을 초과 | `agent_token_cost_usd` 합계 | 비용 초과가 2개월 연속 발생 | +| Parallel fan-out | handoff 평균 hop이 2.0 이상 또는 timeout 빈발 | `agent_handoff_count` p50/p95 | hop >= 2.0이 1주 지속 | +| Cross-room context | 방 사이의 동일 사용자/패턴 이슈가 반복 | 신규 metric 필요 | sensor 먼저 추가 후 데이터 확인 | +| Resolution hash sunset | dual-write 안정 2개월 + stream/hash divergence 0 | sunset 점검 script 또는 spot check | 조건 충족 후 sunset plan 실행 | +| Review queue Web UI | 일일 review queue 진입이 10건 이상 또는 운영자 2명 이상 | `agent_human_override_total` daily | 1개월 지속 | +| Runtime skill discovery | 신규 skill 배포가 주 1회 이상 | 배포 기록, `lint:skills` 변경 빈도 | 빈도 조건 충족 | +| Multi-skill orchestration | handoff chain 평균이 3 hop 이상 | `agent_handoff_count` p90, handoff golden | 평균 3 hop 이상 반복 | +| Supervisor 자동 학습 | router shadow agreement rate가 80% 미만 | `eval:router-shadow-report` | 2개 이상 window에서 80% 미만 | + +## Cost-aware routing + +왜 지금 안 하는가: 현재는 비용보다 correctness와 운영 가능성이 더 큰 리스크다. tier별 routing과 learned router가 이미 비용을 일부 통제한다. + +시작 시 검토할 것: model별 token cost 산정, skill별 비용 dashboard, router decision의 false negative 비용, fallback 품질이다. 관련 코드는 `src/observability/metrics.ts`, `src/skills/onboarding/runtime/router.ts`, `src/eval/router-training.ts`다. + +## Parallel fan-out + +왜 지금 안 하는가: handoff graph가 아직 작고, 병렬 실행은 비용과 race condition을 늘린다. + +시작 시 검토할 것: fan-out 대상 skill의 idempotency, response merge policy, budget accounting, timeout policy다. 관련 코드는 `src/agent/dispatch.ts`, `src/agent/budget.ts`, `src/eval/golden/handoff/cases.jsonl`다. + +## Cross-room context + +왜 지금 안 하는가: room 간 context가 실제로 필요한 빈도와 개인정보 리스크가 아직 수치화되지 않았다. + +시작 시 검토할 것: 사용자 단위 aggregation metric, retention/TTL, opt-in 정책, PII redaction이다. 먼저 sensor와 metric을 추가하고 한 달 이상 관찰한다. + +## Resolution hash sunset + +왜 지금 안 하는가: Iter6에서 stream audit와 hash compatibility dual-write를 막 붙였다. 바로 제거하면 운영 rollback 경로가 좁아진다. + +시작 시 검토할 것: stream과 hash의 일치율, `review:show`와 `review:stats` read path, TTL 이후 복구 가능성이다. 절차는 [resolution-hash-sunset-procedure.md](resolution-hash-sunset-procedure.md)를 따른다. + +## Review queue Web UI + +왜 지금 안 하는가: CLI로 충분한 규모인지 확인되지 않았다. Web UI는 인증, audit, 권한, 배포 표면을 늘린다. + +시작 시 검토할 것: 일일 queue 양, 운영자 수, 자주 쓰는 resolution action, note template, audit export 필요성이다. CLI는 `src/operations/cli.ts`가 기준이다. + +## Runtime skill discovery + +왜 지금 안 하는가: 현재 skill 수가 적고 정적 registry가 변경 영향 파악에 유리하다. + +시작 시 검토할 것: skill package format, hot reload, SKILL.md validation timing, transition matrix reload, 배포 rollback 방식이다. + +## Multi-skill orchestration + +왜 지금 안 하는가: 지금의 handoff는 sequential dispatch로 충분하다. orchestration graph를 일반화하면 디버깅 난도가 크게 올라간다. + +시작 시 검토할 것: graph DSL, response merge, partial failure, supervisor override, eval judge 확장이다. + +## Supervisor 자동 학습 + +왜 지금 안 하는가: supervisor routing과 userMessage는 이제 막 안정 가드를 갖췄다. 자동 학습은 충분한 resolution label과 shadow divergence evidence가 필요하다. + +시작 시 검토할 것: `npm run eval:router-shadow-report -- --since=7d`, supervisor resolution action 분포, target별 false route, human note taxonomy다. diff --git a/docs/resolution-hash-sunset-procedure.md b/docs/resolution-hash-sunset-procedure.md new file mode 100644 index 0000000..04db61d --- /dev/null +++ b/docs/resolution-hash-sunset-procedure.md @@ -0,0 +1,101 @@ +# Supervisor Resolution Hash Sunset Procedure + +Last verified: Iter8 implementation branch, 2026-05-18. + +Supervisor resolution은 현재 두 곳에 기록된다. + +- Audit stream: `lolympus-agent:review-resolutions` +- Compatibility hash: `lolympus-agent:review-resolved:{reviewId}` + +Hash는 기존 read path와 rollback을 위한 compatibility layer다. 실제 sunset은 최소 1-2개월 운영 후 아래 조건을 만족할 때만 진행한다. + +## 선결 조건 + +- Dual-write 운영 기간이 2개월 이상이다. +- Stream과 hash의 resolution 데이터 일치율이 100%다. +- `review:list`, `review:show`, `review:resolve`, `review:stats`가 stream 기반 확인으로 대체 가능한 상태다. +- 최근 2주간 supervisor resolution 관련 incident가 없다. +- Sunset 실행 전 main branch의 전체 회귀가 통과한다. + +권장 점검: + +```bash +npm run review:stats +npm run review:list -- --include-resolved --limit=20 +npm run typecheck +npm test +``` + +필요하면 별도 consistency script를 만든다. + +```text +scripts/verify-dual-write-consistency.ts +``` + +이 script는 `lolympus-agent:review-resolutions`의 최신 resolution과 `lolympus-agent:review-resolved:{reviewId}` hash를 비교해 divergence를 출력해야 한다. + +## Sunset 단계 + +### Step 1. Hash write 중단 + +`src/skills/supervisor/review-queue.ts`의 `resolveSupervisorReview()`에서 `setHash()`와 `expireKey()` 호출을 제거한다. Stream append는 유지한다. + +검증: + +```bash +npm run typecheck +npm test -- src/skills/supervisor/review-queue.test.ts src/operations/cli.test.ts +``` + +Rollback: 해당 commit revert. 기존 hash TTL 동안 read fallback은 유지된다. + +### Step 2. Stream-only write로 1개월 운영 + +새 resolution은 stream에만 쌓이고, 기존 hash는 TTL까지 남는다. 이 기간 동안 `review:list/show/stats`가 정상 동작하는지 확인한다. + +관찰 항목: + +- unresolved item이 비정상적으로 늘지 않는가 +- resolution history가 누락되지 않는가 +- 운영자가 `review:show`에서 이전 resolution을 확인할 수 있는가 + +Rollback: `setHash()`와 `expireKey()`를 복구한다. + +### Step 3. Hash read fallback 제거 + +`getSupervisorReview()`와 `listSupervisorReview()`에서 hash lookup을 제거하고, resolution stream join으로만 상태를 계산한다. + +검증: + +```bash +npm run review:list -- --include-resolved --limit=5 +npm run review:stats +npm test -- src/skills/supervisor/review-queue.test.ts src/operations/cli.test.ts +``` + +Rollback: hash lookup 코드를 복구한다. Step 1 이후 새 데이터는 hash에 없으므로 rollback 전에 stream 기반 backfill이 필요한지 판단한다. + +### Step 4. 기존 hash 정리 + +TTL이 짧으면 자연 만료를 기다린다. 운영상 즉시 정리가 필요하면 cleanup script를 별도 PR로 만든다. + +주의: cleanup script는 반드시 dry-run을 기본값으로 둔다. + +### Step 5. 상수와 문서 정리 + +`SUPERVISOR_RESOLVED_HASH_PREFIX` 상수와 관련 env/문서 언급을 제거한다. `docs/service-evaluation-operations.md`의 Redis key schema도 갱신한다. + +검증: + +```bash +rg "review-resolved|SUPERVISOR_RESOLVED_HASH_PREFIX" src docs +npm run lint +npm test +``` + +## Rollback 원칙 + +- Step 1-2는 단순 revert로 rollback 가능하다. +- Step 3 이후 rollback은 stream-to-hash backfill 필요 여부를 먼저 판단한다. +- Step 4 cleanup 이후에는 hash 데이터 복구를 기대하지 않는다. +- 모든 단계는 작은 PR로 나누고, 각 PR은 독립 revert 가능해야 한다. diff --git a/docs/service-evaluation-operations.md b/docs/service-evaluation-operations.md index 88d91e0..3358fc0 100644 --- a/docs/service-evaluation-operations.md +++ b/docs/service-evaluation-operations.md @@ -6,10 +6,10 @@ 운영 프로세스는 크게 두 개다. -| 프로세스 | 엔트리포인트 | 역할 | -|---|---|---| -| agent | `src/main.ts` | NATS request/reply를 받아 실시간 온보딩 응답 생성 | -| worker | `src/workers/background.ts` | Redis Stream background job 처리, shadow eval, archive, drift, review routing | +| 프로세스 | 엔트리포인트 | 역할 | +| -------- | --------------------------- | ----------------------------------------------------------------------------- | +| agent | `src/main.ts` | NATS request/reply를 받아 실시간 온보딩 응답 생성 | +| worker | `src/workers/background.ts` | Redis Stream background job 처리, shadow eval, archive, drift, review routing | PM2 기준으로는 `ecosystem.config.cjs`에서 agent와 worker를 함께 띄운다. @@ -88,17 +88,17 @@ Tool loop는 `responseMode=llm-with-tools`일 때 작동한다. Built-in tool은 Worker는 `lolympus-agent:background` Redis Stream을 consumer group `bg-worker`로 읽는다. -| Job type | 처리 | -|---|---| -| `shadow-eval` | 실시간 응답을 rule/LLM/process 기준으로 평가 | -| `summarize` | 오래된 대화 일부를 요약해 summary slot에 저장 | +| Job type | 처리 | +| ------------- | --------------------------------------------- | +| `shadow-eval` | 실시간 응답을 rule/LLM/process 기준으로 평가 | +| `summarize` | 오래된 대화 일부를 요약해 summary slot에 저장 | Worker는 pending job을 복구하고, delivery count가 한도를 넘으면 DLQ로 이동한다. -| Stream | 용도 | -|---|---| -| `lolympus-agent:background` | background job queue | -| `lolympus-agent:background:dlq` | 반복 실패 job 저장 | +| Stream | 용도 | +| ------------------------------- | -------------------- | +| `lolympus-agent:background` | background job queue | +| `lolympus-agent:background:dlq` | 반복 실패 job 저장 | ## 6. Shadow Evaluation 흐름 @@ -123,14 +123,14 @@ lolympus-agent:eval:results Worker는 일정 간격으로 eval result를 가공한다. -| 작업 | 기본 주기 | 결과 | -|---|---:|---| -| aggregation | `EVAL_AGG_TICK_MS` 5분 | slice별 Redis Hash metric | -| archive | `EVAL_ARCHIVE_TICK_MS` 1시간 | MongoDB `eval_archive` | -| drift detection | `EVAL_DRIFT_TICK_MS` 24시간 | `eval-drift` alert | -| promotion | `EVAL_PROMOTE_TICK_MS` 24시간 | golden candidate jsonl | -| review routing | `EVAL_REVIEW_TICK_MS` 5분 | human review queue | -| review label archive | archive tick과 함께 | MongoDB `eval_review_labels` | +| 작업 | 기본 주기 | 결과 | +| -------------------- | ----------------------------: | ---------------------------- | +| aggregation | `EVAL_AGG_TICK_MS` 5분 | slice별 Redis Hash metric | +| archive | `EVAL_ARCHIVE_TICK_MS` 1시간 | MongoDB `eval_archive` | +| drift detection | `EVAL_DRIFT_TICK_MS` 24시간 | `eval-drift` alert | +| promotion | `EVAL_PROMOTE_TICK_MS` 24시간 | golden candidate jsonl | +| review routing | `EVAL_REVIEW_TICK_MS` 5분 | human review queue | +| review label archive | archive tick과 함께 | MongoDB `eval_review_labels` | Slice 정의: @@ -149,29 +149,29 @@ tier × intent × phase 주요 Redis/Garnet key: -| Key | Type | 용도 | -|---|---|---| -| `lolympus-agent:background` | Stream | worker job queue | -| `lolympus-agent:background:dlq` | Stream | worker dead-letter queue | -| `lolympus-agent:eval:results` | Stream | shadow eval 결과 | -| `lolympus-agent:eval:baseline` | String | 최근 안정 baseline report | -| `lolympus-agent:eval:agg:{tier}:{intent}:{phase}` | Hash | slice별 count/score/latency/process metric | -| `lolympus-agent:eval:agg:cursor` | String | aggregation cursor | -| `lolympus-agent:eval:archive:cursor` | String | eval archive cursor | -| `lolympus-agent:eval:review:queue` | Stream | human review 대기 큐 | -| `lolympus-agent:eval:review:labeled` | Stream | human review label | -| `lolympus-agent:eval:review:cursor` | String | review routing cursor | -| `lolympus-agent:eval:review:dedupe:{evalStreamId}` | String | 동일 eval result 중복 라우팅 방지 | -| `lolympus-agent:eval:review:label-archive:cursor` | String | review label archive cursor | -| `lolympus-agent:alert:dedupe:{kind}` | String | alert rate limit | +| Key | Type | 용도 | +| -------------------------------------------------- | ------ | ------------------------------------------ | +| `lolympus-agent:background` | Stream | worker job queue | +| `lolympus-agent:background:dlq` | Stream | worker dead-letter queue | +| `lolympus-agent:eval:results` | Stream | shadow eval 결과 | +| `lolympus-agent:eval:baseline` | String | 최근 안정 baseline report | +| `lolympus-agent:eval:agg:{tier}:{intent}:{phase}` | Hash | slice별 count/score/latency/process metric | +| `lolympus-agent:eval:agg:cursor` | String | aggregation cursor | +| `lolympus-agent:eval:archive:cursor` | String | eval archive cursor | +| `lolympus-agent:eval:review:queue` | Stream | human review 대기 큐 | +| `lolympus-agent:eval:review:labeled` | Stream | human review label | +| `lolympus-agent:eval:review:cursor` | String | review routing cursor | +| `lolympus-agent:eval:review:dedupe:{evalStreamId}` | String | 동일 eval result 중복 라우팅 방지 | +| `lolympus-agent:eval:review:label-archive:cursor` | String | review label archive cursor | +| `lolympus-agent:alert:dedupe:{kind}` | String | alert rate limit | ## 9. MongoDB Collections -| Collection | 생성/사용 모듈 | 용도 | -|---|---|---| -| `eval_archive` | `src/eval/archive.ts` | shadow eval 결과 장기 보관 | -| `eval_review_labels` | `src/eval/review/label-archive.ts` | 사람 라벨 장기 보관 | -| `eval_tournaments` | `src/eval/loop/tournament.ts` | prompt variant tournament 결과 | +| Collection | 생성/사용 모듈 | 용도 | +| -------------------- | ---------------------------------- | ------------------------------ | +| `eval_archive` | `src/eval/archive.ts` | shadow eval 결과 장기 보관 | +| `eval_review_labels` | `src/eval/review/label-archive.ts` | 사람 라벨 장기 보관 | +| `eval_tournaments` | `src/eval/loop/tournament.ts` | prompt variant tournament 결과 | `eval_archive`는 `evaluatedAt` 기준 TTL index를 가진다. 기본 보관 기간은 `EVAL_ARCHIVE_TTL_DAYS=30`이다. @@ -218,6 +218,45 @@ npm run eval:review -- --limit=20 --export-labeled=eval-reports/review-labels.js 라벨 결과는 `lolympus-agent:eval:review:labeled` Stream에 쌓이고 archive tick에서 `eval_review_labels`로 영속화된다. +### 10.1 Supervisor Review Queue + +Supervisor review queue는 skill handoff가 budget, confidence, policy, validation guardrail에 걸렸을 때 운영자가 확인하는 큐다. + +큐: + +```text +lolympus-agent:review-queue +``` + +Resolution audit: + +```text +lolympus-agent:review-resolutions +``` + +기본 workflow: + +1. Slack 또는 NATS에서 `supervisor-handoff` alert를 확인한다. +2. 최근 queue를 본다. +3. 개별 review를 열어 handoff packet, source chain, reason을 확인한다. +4. 운영자 답변, reroute, dismiss 중 하나로 resolution을 기록한다. +5. `review:stats`로 pending이 줄었는지 확인한다. + +```bash +npm run review:list -- --limit=10 +npm run review:show -- --history +npm run review:resolve -- --action=manual-reply --note="운영자 확인 완료" +npm run review:stats +``` + +특정 reason만 볼 때: + +```bash +npm run review:list -- --reason=low-confidence --limit=10 +``` + +Resolution history는 stream에 쌓이며, compatibility hash는 sunset 전까지 read fallback으로 유지된다. Sunset 조건과 절차는 [resolution-hash-sunset-procedure.md](resolution-hash-sunset-procedure.md)를 따른다. + ## 11. Calibration 흐름 Calibration은 사람 점수와 LLM judge 점수의 상관을 계산한다. @@ -345,66 +384,118 @@ Variant 적용은 자동 머지가 아니다. 사람이 tournament 결과와 sam ## 14. Eval CLI 요약 -| 명령 | 용도 | -|---|---| -| `npm run eval` | golden set 실행 | -| `npm run eval:capture` | MongoDB events에서 golden scaffold 추출 | -| `npm run eval:baseline` | 안정 baseline 저장 | -| `npm run eval:report` | archive slice report 출력 | -| `npm run eval:promote` | golden candidate 추출 | -| `npm run eval:review` | human review CLI | -| `npm run eval:calibrate` | 사람/LLM judge calibration | -| `npm run eval:propose` | prompt/tool improvement proposal 생성 | -| `npm run eval:tournament` | prompt variant A/B tournament 실행 | -| `npm run eval:tournament-results` | tournament 결과 조회 | +| 명령 | 용도 | +| ----------------------------------- | ---------------------------------------------- | +| `npm run eval` | golden set 실행 | +| `npm run eval:capture` | MongoDB events에서 golden scaffold 추출 | +| `npm run eval:baseline` | 안정 baseline 저장 | +| `npm run eval:report` | archive slice report 출력 | +| `npm run eval:promote` | golden candidate 추출 | +| `npm run eval:review` | human review CLI | +| `npm run eval:calibrate` | 사람/LLM judge calibration | +| `npm run eval:propose` | prompt/tool improvement proposal 생성 | +| `npm run eval:tournament` | prompt variant A/B tournament 실행 | +| `npm run eval:tournament-results` | tournament 결과 조회 | +| `npm run eval:train-router` | archive 기반 learned router weights 생성 | +| `npm run eval:router-shadow-report` | static router와 learned router divergence 보고 | + +### 14.1 자주 쓰는 명령 Cheat Sheet + +평가: + +```bash +npm run eval -- --set=onboarding/cases --skip-llm-judge +npm run eval:moderation -- --skip-llm-judge +npm run eval:handoff -- --skip-llm-judge +npm run eval:report -- --worst=20 --since=7d +npm run eval:propose -- --slice=tier=1,intent=status-question,phase=deadline-near +npm run eval:tournament -- --variants=control, --max-matches=50 +``` + +학습과 shadow 비교: + +```bash +npm run eval:train-router -- --output=.local/router-weights.json --force +npm run eval:router-shadow-report -- --since=7d --weights=.local/router-weights.json +``` + +운영: + +```bash +npm run review:list -- --limit=10 +npm run review:show -- --history +npm run review:resolve -- --action=dismiss --note="중복" +npm run review:stats +``` + +검증: + +```bash +npm run typecheck +npm run lint +npm run lint:skills +npm test +``` ## 15. 주요 Env -| Env | 기본값 | 용도 | -|---|---:|---| -| `EVAL_STREAM_MAXLEN` | 10000 | eval results Stream maxlen | -| `EVAL_AGG_TICK_MS` | 300000 | aggregation tick | -| `EVAL_AGG_TTL_SEC` | 604800 | slice metric TTL | -| `EVAL_ARCHIVE_TICK_MS` | 3600000 | archive tick | -| `EVAL_ARCHIVE_TTL_DAYS` | 30 | archive TTL days | -| `EVAL_DRIFT_TICK_MS` | 86400000 | drift tick | -| `EVAL_PROMOTE_TICK_MS` | 86400000 | promotion tick | -| `EVAL_REVIEW_TICK_MS` | 300000 | review routing tick | -| `EVAL_REVIEW_QUEUE_MAXLEN` | 500 | review queue maxlen | -| `EVAL_REVIEW_LABEL_MAXLEN` | 5000 | review label Stream maxlen | -| `EVAL_REVIEW_DEDUPE_TTL_SEC` | 604800 | review dedupe TTL | -| `EVAL_CALIBRATION_MIN_CORR` | 0.5 | calibration gate | -| `REVIEWER_ID` | git user.email | human review 식별자 | -| `LLM_PROPOSE_MODEL` | `LLM_JUDGE_MODEL` | proposal model | -| `LLM_PROPOSE_MAX_TOKENS` | 1024 | proposal max tokens | -| `EVAL_TOURNAMENT_MAX_MATCHES` | 50 | tournament match cap | -| `EVAL_TOURNAMENT_MIN_CASES` | 10 | proposal 최소 archive cases | -| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 | -| `METRICS_PORT` | 9464 | Prometheus `/metrics` HTTP endpoint port | -| `SLACK_ALERT_WEBHOOK_URL` | unset | alert Slack incoming webhook URL | -| `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | Slack webhook POST timeout | -| `ALERT_THRESHOLD_P95_LATENCY_MS` | 3000 | 글로벌 p95 latency alert 기준 | -| `ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT` | 10 | 글로벌 eval score drop alert 기준 | -| `ALERT_THRESHOLD_LLM_ERROR_RATE` | 0.1 | 글로벌 LLM judge error rate alert 기준 | -| `ALERT_THRESHOLD_DRIFT_PCT` | 15 | 글로벌 drift alert 기준 | +| Env | 기본값 | 용도 | +| --------------------------------------- | ----------------: | ---------------------------------------- | +| `EVAL_STREAM_MAXLEN` | 10000 | eval results Stream maxlen | +| `EVAL_AGG_TICK_MS` | 300000 | aggregation tick | +| `EVAL_AGG_TTL_SEC` | 604800 | slice metric TTL | +| `EVAL_ARCHIVE_TICK_MS` | 3600000 | archive tick | +| `EVAL_ARCHIVE_TTL_DAYS` | 30 | archive TTL days | +| `EVAL_DRIFT_TICK_MS` | 86400000 | drift tick | +| `EVAL_PROMOTE_TICK_MS` | 86400000 | promotion tick | +| `EVAL_REVIEW_TICK_MS` | 300000 | review routing tick | +| `EVAL_REVIEW_QUEUE_MAXLEN` | 500 | review queue maxlen | +| `EVAL_REVIEW_LABEL_MAXLEN` | 5000 | review label Stream maxlen | +| `EVAL_REVIEW_DEDUPE_TTL_SEC` | 604800 | review dedupe TTL | +| `EVAL_CALIBRATION_MIN_CORR` | 0.5 | calibration gate | +| `REVIEWER_ID` | git user.email | human review 식별자 | +| `LLM_PROPOSE_MODEL` | `LLM_JUDGE_MODEL` | proposal model | +| `LLM_PROPOSE_MAX_TOKENS` | 1024 | proposal max tokens | +| `EVAL_TOURNAMENT_MAX_MATCHES` | 50 | tournament match cap | +| `EVAL_TOURNAMENT_MIN_CASES` | 10 | proposal 최소 archive cases | +| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 | +| `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path | +| `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 | +| `METRICS_PORT` | 9464 | Prometheus `/metrics` HTTP endpoint port | +| `SLACK_ALERT_WEBHOOK_URL` | unset | alert Slack incoming webhook URL | +| `SLACK_WEBHOOK_TIMEOUT_MS` | 3000 | Slack webhook POST timeout | +| `ALERT_THRESHOLD_P95_LATENCY_MS` | 3000 | 글로벌 p95 latency alert 기준 | +| `ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT` | 10 | 글로벌 eval score drop alert 기준 | +| `ALERT_THRESHOLD_LLM_ERROR_RATE` | 0.1 | 글로벌 LLM judge error rate alert 기준 | +| `ALERT_THRESHOLD_DRIFT_PCT` | 15 | 글로벌 drift alert 기준 | ## 16. Alert와 Monitoring Worker는 다음 종류의 alert를 보낼 수 있다. -| Alert kind | 의미 | -|---|---| -| `latency` | 최근 shadow eval latency p95 초과 | -| `llm-error-rate` | LLM judge 실패율 상승 | -| `eval-score-drop` | baseline 대비 eval score 하락 | -| `eval-drift` | 24h vs 7d slice score drift | -| `endpoint-down` | LLM circuit breaker open | -| `archive-stalled` | MongoDB archive 연속 실패 | +| Alert kind | 의미 | +| -------------------- | --------------------------------- | +| `latency` | 최근 shadow eval latency p95 초과 | +| `llm-error-rate` | LLM judge 실패율 상승 | +| `eval-score-drop` | baseline 대비 eval score 하락 | +| `eval-drift` | 24h vs 7d slice score drift | +| `endpoint-down` | LLM circuit breaker open | +| `archive-stalled` | MongoDB archive 연속 실패 | +| `supervisor-handoff` | supervisor review queue 진입 | Alert는 `lolympus-agent.alert` NATS subject로 발행되고, dedupe key로 rate limit된다. `SLACK_ALERT_WEBHOOK_URL`이 설정되어 있으면 같은 alert가 Slack incoming webhook으로도 전송된다. Slack 전송은 NATS publish 이후 비동기로 실행되므로 Slack 장애가 agent 응답 경로를 막지 않는다. +Alert 처리 workflow: + +1. Slack 메시지의 `kind`, `skill`, `metric`, `value`, `threshold`, `window`를 확인한다. +2. `supervisor-handoff`면 `review:list`와 `review:show`로 packet을 확인한다. +3. `latency`면 `/metrics`의 p95 panel과 최근 deploy를 확인한다. +4. `llm-error-rate`나 `endpoint-down`이면 LLM endpoint env와 circuit breaker 로그를 확인한다. +5. `eval-score-drop` 또는 `eval-drift`면 `eval:report`, archive sample, golden smoke를 확인한다. +6. 조치 후 dedupe window가 끝나도 반복되는지 본다. + ### 16.1 Skill별 threshold Alert threshold는 글로벌 env를 기본으로 쓰고, skill별 env가 있으면 우선 적용한다. @@ -430,9 +521,9 @@ skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한 Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다. -| Endpoint | 응답 | -|---|---| -| `/healthz` | `ok` | +| Endpoint | 응답 | +| ---------- | -------------------------- | +| `/healthz` | `ok` | | `/metrics` | Prometheus text exposition | Prometheus scrape 예시: @@ -447,18 +538,18 @@ scrape_configs: 권장 dashboard panel: -| Metric | 권장 panel | -|---|---| -| `agent_handoff_total` | skill/from/to/reason별 handoff rate | -| `agent_handoff_loop_total` | skill별 loop 발생 수 | -| `agent_human_override_total` | supervisor/manual review 유입 | -| `agent_missed_escalation_total` | skill별 missed escalation | -| `agent_routing_accuracy_total` | eval routing accuracy | -| `agent_unnecessary_escalation_total` | skill별 unnecessary escalation | -| `agent_validator_fail_total` | skill/stage별 validator fail | -| `agent_handoff_count` | skill별 request당 handoff histogram | -| `agent_time_to_resolution_ms` | skill별 처리 시간 histogram | -| `agent_token_cost_usd` | skill/model별 token cost histogram | +| Metric | 권장 panel | +| ------------------------------------ | ----------------------------------- | +| `agent_handoff_total` | skill/from/to/reason별 handoff rate | +| `agent_handoff_loop_total` | skill별 loop 발생 수 | +| `agent_human_override_total` | supervisor/manual review 유입 | +| `agent_missed_escalation_total` | skill별 missed escalation | +| `agent_routing_accuracy_total` | eval routing accuracy | +| `agent_unnecessary_escalation_total` | skill별 unnecessary escalation | +| `agent_validator_fail_total` | skill/stage별 validator fail | +| `agent_handoff_count` | skill별 request당 handoff histogram | +| `agent_time_to_resolution_ms` | skill별 처리 시간 histogram | +| `agent_token_cost_usd` | skill/model별 token cost histogram | PromQL 예시: @@ -468,6 +559,16 @@ histogram_quantile(0.95, sum by (skill, le) (rate(agent_time_to_resolution_ms_bu sum by (skill, stage) (rate(agent_validator_fail_total[5m])) ``` +운영 SLI 권장 시작점: + +| SLI | 권장 시작값 | +| -------------------------- | -------------------: | +| request p95 latency | 3초 이하 | +| supervisor escalation rate | skill별 일 10건 이하 | +| LLM judge error rate | 10% 이하 | +| eval drift | 15% 이하 | +| archive stalled | 3회 연속 실패 0건 | + ### 16.3 Slack webhook 설정 Slack incoming webhook URL을 env로 설정한다. diff --git a/package.json b/package.json index 2c0f1d3..95dc145 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "eval:calibrate": "node --import tsx src/eval/cli.ts calibrate", "eval:train-prm": "node --import tsx src/eval/cli.ts train-prm", "eval:train-router": "node --import tsx src/eval/cli.ts train-router", + "eval:router-shadow-report": "node --import tsx src/eval/cli.ts router-shadow-report", "eval:propose": "node --import tsx src/eval/cli.ts propose", "eval:propose:moderation": "node --import tsx src/eval/cli.ts propose --skill=moderation", "eval:tournament": "node --import tsx src/eval/cli.ts tournament", diff --git a/src/config.test.ts b/src/config.test.ts index a620b79..f92a3e9 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -7,6 +7,7 @@ import { getAlertThresholdP95LatencyMs, getSlackAlertWebhookUrl, getSlackWebhookTimeoutMs, + getSupervisorLlmResponseEnabled, } from "./config.js"; const ENV_KEYS = [ @@ -20,6 +21,7 @@ const ENV_KEYS = [ "ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING", "SLACK_ALERT_WEBHOOK_URL", "SLACK_WEBHOOK_TIMEOUT_MS", + "AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", ] as const; describe("config", () => { @@ -59,4 +61,12 @@ describe("config", () => { expect(getSlackAlertWebhookUrl()).toBe("https://hooks.slack.test/x"); expect(getSlackWebhookTimeoutMs()).toBe(1234); }); + + it("enables supervisor LLM user messages by default with env override", () => { + expect(getSupervisorLlmResponseEnabled()).toBe(true); + + vi.stubEnv("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", "false"); + + expect(getSupervisorLlmResponseEnabled()).toBe(false); + }); }); diff --git a/src/config.ts b/src/config.ts index 1c1bdf1..9037c2a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,20 +7,14 @@ type LogLevel = | "trace" | "silent"; -function getPositiveNumber( - envKey: string, - fallback: number, -): number { +function getPositiveNumber(envKey: string, fallback: number): number { const raw = process.env[envKey]?.trim(); if (!raw) return fallback; const parsed = Number(raw); return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } -function getRatioNumber( - envKey: string, - fallback: number, -): number { +function getRatioNumber(envKey: string, fallback: number): number { const raw = process.env[envKey]?.trim(); if (!raw) return fallback; const parsed = Number(raw); @@ -29,8 +23,13 @@ function getRatioNumber( : fallback; } -function normalizeSkillEnvSuffix(skill: string | undefined): string | undefined { - const suffix = skill?.trim().toUpperCase().replace(/[^A-Z0-9]/gu, "_"); +function normalizeSkillEnvSuffix( + skill: string | undefined, +): string | undefined { + const suffix = skill + ?.trim() + .toUpperCase() + .replace(/[^A-Z0-9]/gu, "_"); return suffix || undefined; } @@ -282,6 +281,10 @@ export function getSupervisorLlmConfidenceThreshold(): number { return getRatioNumber("AGENT_SUPERVISOR_LLM_CONFIDENCE_THRESHOLD", 0.6); } +export function getSupervisorLlmResponseEnabled(): boolean { + return getBoolean("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", true); +} + export function getEvalCalibrationMinCorr(): number { return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5); } @@ -334,7 +337,11 @@ export function getAgentSkillMdMode(): boolean { return getBoolean("AGENT_SKILL_MD_MODE", false); } -export function getAgentRouterMode(): "off" | "static" | "learned-v1" | "learned-v2" { +export function getAgentRouterMode(): + | "off" + | "static" + | "learned-v1" + | "learned-v2" { const raw = process.env["AGENT_ROUTER_MODE"]?.trim(); if ( raw === "off" || @@ -456,7 +463,9 @@ export function getOpenRouterEnabled(): boolean { } export function getOpenRouterApiUrl(): string { - return process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1"; + return ( + process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1" + ); } export function getTier0Model(): string { @@ -573,10 +582,7 @@ export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number { getNodeEnv() === "prod" ? 0.5 : 1.0, ); case 2: - return getRatioNumber( - "SHADOW_EVAL_SAMPLING_RATE_TIER2", - 1.0, - ); + return getRatioNumber("SHADOW_EVAL_SAMPLING_RATE_TIER2", 1.0); } } @@ -597,11 +603,7 @@ export function getAlertThresholdEvalScoreDropPct(skill?: string): number { } export function getAlertThresholdLlmErrorRate(skill?: string): number { - return getRatioNumberForSkill( - "ALERT_THRESHOLD_LLM_ERROR_RATE", - skill, - 0.1, - ); + return getRatioNumberForSkill("ALERT_THRESHOLD_LLM_ERROR_RATE", skill, 0.1); } export function getAlertThresholdDriftPct(skill?: string): number { diff --git a/src/eval/cli.ts b/src/eval/cli.ts index 03e9c41..0e286c1 100644 --- a/src/eval/cli.ts +++ b/src/eval/cli.ts @@ -49,30 +49,37 @@ function parseSlice(value: string | undefined): SliceKey { }; } -function printRunSummary(report: { - generatedAt: string; - summary: { - cases: number; - passRate: number; - ruleAverageScore: number; - averageLatencyMs: number; - llmAverageOverall: number | null; - llmJudgedCases: number; - }; - baselineDelta?: { - passRate: number | null; - ruleAverageScore: number | null; - llmAverageOverall: number | null; - averageLatencyMs: number | null; - } | undefined; -}, reportPath: string): void { +function printRunSummary( + report: { + generatedAt: string; + summary: { + cases: number; + passRate: number; + ruleAverageScore: number; + averageLatencyMs: number; + llmAverageOverall: number | null; + llmJudgedCases: number; + }; + baselineDelta?: + | { + passRate: number | null; + ruleAverageScore: number | null; + llmAverageOverall: number | null; + averageLatencyMs: number | null; + } + | undefined; + }, + reportPath: string, +): void { const llmLine = report.summary.llmAverageOverall === null ? "n/a" : `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`; console.info(`[eval] generatedAt=${report.generatedAt}`); - console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`); + console.info( + `[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`, + ); console.info( `[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`, ); @@ -229,12 +236,9 @@ async function main(): Promise { } if (command === "run") { - const { loadEvalCases, resolveEvalSetPath, runEval } = await import( - "./runner.js" - ); - const { publishEvalReportIfPossible } = await import( - "./runner.js" - ); + const { loadEvalCases, resolveEvalSetPath, runEval } = + await import("./runner.js"); + const { publishEvalReportIfPossible } = await import("./runner.js"); const setName = getFlagString(flags, "set") ?? "onboarding/cases"; const resolvedSet = resolveEvalSetPath(setName); const skill = getFlagString(flags, "skill") ?? resolvedSet.skill; @@ -269,7 +273,8 @@ async function main(): Promise { if (command === "baseline") { const { updateEvalBaseline } = await import("./runner.js"); const reportPath = getFlagString(flags, "report") ?? positionals[0]; - const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath); + const { baseline, baselinePath, storedInRedis } = + await updateEvalBaseline(reportPath); console.info( `[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`, ); @@ -298,14 +303,15 @@ async function main(): Promise { } if (command === "promote") { - const { appendCandidatesToJsonl, selectPromotionCandidates } = await import( - "./promote.js" - ); + const { appendCandidatesToJsonl, selectPromotionCandidates } = + await import("./promote.js"); const mongo = await connectMongoForEval(); try { const candidates = await selectPromotionCandidates(mongo.connection, { limit: parsePositiveInteger(getFlagString(flags, "limit")), - minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")), + minLowScore: parsePositiveNumber( + getFlagString(flags, "min-low-score"), + ), }); if (hasFlag(flags, "dry-run")) { @@ -336,7 +342,8 @@ async function main(): Promise { } if (command === "review") { - const { parseReviewReason, runReviewCli } = await import("./review/cli.js"); + const { parseReviewReason, runReviewCli } = + await import("./review/cli.js"); const garnet = await connectGarnetForEval(); try { const result = await runReviewCli(garnet.garnetClient, { @@ -356,9 +363,8 @@ async function main(): Promise { if (command === "calibrate") { const { getEvalCalibrationMinCorr } = await import("../config.js"); - const { computeCalibrationReport } = await import( - "./review/calibration.js" - ); + const { computeCalibrationReport } = + await import("./review/calibration.js"); const mongo = await connectMongoForEval(); const garnet = await connectGarnetForEval(); try { @@ -392,19 +398,13 @@ async function main(): Promise { } if (command === "train-prm") { - const { - getEvalPrmMinLabels, - getEvalPrmWeightsPath, - } = await import("../config.js"); + const { getEvalPrmMinLabels, getEvalPrmWeightsPath } = + await import("../config.js"); const { getEvalArchiveCollection } = await import("./archive.js"); - const { queryArchivedReviewLabels } = await import( - "./review/label-archive.js" - ); - const { - buildPrmTrainingSamples, - savePrmModel, - trainRidgePrmModel, - } = await import("./judges/prm.js"); + const { queryArchivedReviewLabels } = + await import("./review/label-archive.js"); + const { buildPrmTrainingSamples, savePrmModel, trainRidgePrmModel } = + await import("./judges/prm.js"); const mongo = await connectMongoForEval(); try { const labels = await queryArchivedReviewLabels(mongo.connection, { @@ -460,16 +460,13 @@ async function main(): Promise { } if (command === "train-router") { - const { - getAgentRouterWeightsPath, - getEvalPrmMinLabels, - } = await import("../config.js"); + const { getAgentRouterWeightsPath, getEvalPrmMinLabels } = + await import("../config.js"); const { getEvalArchiveCollection } = await import("./archive.js"); - const { buildRouterTrainingSamples } = await import("./router-training.js"); - const { - saveLearnedRouterModel, - trainLearnedRouterModel, - } = await import("@skills/onboarding/runtime/router"); + const { buildRouterTrainingSamples } = + await import("./router-training.js"); + const { saveLearnedRouterModel, trainLearnedRouterModel } = + await import("@skills/onboarding/runtime/router"); const mongo = await connectMongoForEval(); try { const archiveLimit = @@ -501,7 +498,9 @@ async function main(): Promise { const model = trainLearnedRouterModel(samples, { epochs: parsePositiveInteger(getFlagString(flags, "epochs")), - learningRate: parsePositiveNumber(getFlagString(flags, "learning-rate")), + learningRate: parsePositiveNumber( + getFlagString(flags, "learning-rate"), + ), l2: parsePositiveNumber(getFlagString(flags, "l2")), }); saveLearnedRouterModel(outputPath, model); @@ -514,6 +513,46 @@ async function main(): Promise { return; } + if (command === "router-shadow-report") { + const { getAgentRouterWeightsPath } = await import("../config.js"); + const { loadLearnedRouterModel } = + await import("@skills/onboarding/runtime/router"); + const { buildRouterShadowReport, formatRouterShadowReport } = + await import("./router-shadow-report.js"); + const sinceMs = parseDurationMs(getFlagString(flags, "since")); + const untilFlag = getFlagString(flags, "until"); + const untilMs = untilFlag ? parseDurationMs(untilFlag) : 0; + const now = Date.now(); + const weightsPath = + getFlagString(flags, "weights") ?? getAgentRouterWeightsPath(); + const reportOptions = { + since: new Date(now - sinceMs), + until: new Date(now - untilMs), + weightsPath, + }; + + if (!loadLearnedRouterModel(weightsPath)) { + const report = await buildRouterShadowReport( + { readStreamRange: () => Promise.resolve([]) }, + reportOptions, + ); + console.info(formatRouterShadowReport(report)); + return; + } + + const garnet = await connectGarnetForEval(); + try { + const report = await buildRouterShadowReport( + garnet.garnetClient, + reportOptions, + ); + console.info(formatRouterShadowReport(report)); + } finally { + await garnet.close(); + } + return; + } + if (command === "propose") { const { createEvalLlmClient } = await import("./runner.js"); const { @@ -559,23 +598,22 @@ async function main(): Promise { if (command === "tournament") { const { createEvalLlmClient } = await import("./runner.js"); - const { - loadTournamentCases, - runTournament, - saveTournamentResult, - } = await import("./loop/tournament.js"); + const { loadTournamentCases, runTournament, saveTournamentResult } = + await import("./loop/tournament.js"); const variantIds = parseCsv( getFlagString(flags, "variants") ?? positionals[0], ); if (variantIds.length < 2) { throw new Error( - 'eval:tournament requires --variants=control,', + "eval:tournament requires --variants=control,", ); } const mongo = await connectMongoForEval(); try { - const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches")); + const maxMatches = parsePositiveInteger( + getFlagString(flags, "max-matches"), + ); const skill = getFlagString(flags, "skill"); const cases = await loadTournamentCases(mongo.connection, { caseIds: parseCsv(getFlagString(flags, "case-ids")), @@ -616,10 +654,7 @@ async function main(): Promise { throw new Error(`Unknown eval command: ${command}`); } catch (error) { - logger.error( - { err: error }, - `[eval] "${command}" failed`, - ); + logger.error({ err: error }, `[eval] "${command}" failed`); process.exitCode = 1; } } diff --git a/src/eval/router-shadow-report.test.ts b/src/eval/router-shadow-report.test.ts new file mode 100644 index 0000000..0ce3df9 --- /dev/null +++ b/src/eval/router-shadow-report.test.ts @@ -0,0 +1,178 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +import { describe, expect, it } from "vitest"; + +import type { StreamEntry } from "@common/garnet"; +import { + ROUTER_FEATURE_NAMES, + ROUTING_DECISION_KINDS, + type LearnedRouterModel, +} from "@skills/onboarding/runtime/router"; +import { + SUPERVISOR_RESOLUTIONS_STREAM_KEY, + SUPERVISOR_REVIEW_STREAM_KEY, +} from "@skills/supervisor/review-queue"; +import { BACKGROUND_STREAM_KEY } from "./shadow-bus.js"; +import { + buildRouterShadowReport, + formatRouterShadowReport, +} from "./router-shadow-report.js"; + +class FakeGarnet { + calls = 0; + + constructor(private readonly streams: Record = {}) {} + + readStreamRange(key: string): Promise { + this.calls += 1; + return Promise.resolve(this.streams[key] ?? []); + } +} + +function writeModel(): string { + const weights = ROUTING_DECISION_KINDS.map(() => + ROUTER_FEATURE_NAMES.map(() => 0), + ); + const tier2Index = ROUTING_DECISION_KINDS.indexOf("tier2-full"); + const tier2Weights = weights[tier2Index]; + if (tier2Weights) { + tier2Weights[0] = 10; + } + const model: LearnedRouterModel = { + decisionKinds: [...ROUTING_DECISION_KINDS], + featureNames: [...ROUTER_FEATURE_NAMES], + trainedAt: "2026-05-18T00:00:00.000Z", + version: 1, + weights, + }; + const path = join( + mkdtempSync(join(tmpdir(), "router-shadow-")), + "model.json", + ); + writeFileSync(path, `${JSON.stringify(model)}\n`); + return path; +} + +function streamEntry(id: string, message: Record): StreamEntry { + return { id, message }; +} + +describe("buildRouterShadowReport", () => { + it("reports missing weights without reading Garnet streams", async () => { + const garnet = new FakeGarnet(); + + const report = await buildRouterShadowReport(garnet, { + since: new Date("2026-05-18T00:00:00.000Z"), + until: new Date("2026-05-18T01:00:00.000Z"), + weightsPath: "/path/that/does/not/exist.json", + }); + + expect(report).toMatchObject({ + agreementRate: 1, + status: "weights-missing", + totalSamples: 0, + }); + expect(report.notes[0]).toContain("Router weights missing"); + expect(garnet.calls).toBe(0); + }); + + it("compares static and learned router decisions from supervisor outcomes", async () => { + const weightsPath = writeModel(); + const garnet = new FakeGarnet({ + [BACKGROUND_STREAM_KEY]: [ + streamEntry("10-0", { + classified: JSON.stringify({ + confidence: 0.7, + intent: { kind: "status-question" }, + source: "llm", + }), + createdAt: "2026-05-18T00:00:30.000Z", + derived: JSON.stringify({ + allowShortResponse: false, + intent: { kind: "status-question" }, + phase: { kind: "deadline-near", remaining: [] }, + responseMode: "llm", + routingDecision: { kind: "tier1-direct", skipPlanner: false }, + shouldRespond: true, + suggestedMaxTokens: 512, + tier: 1, + }), + input: JSON.stringify({ + request: { + chatroomId: "room-1", + nickname: "tester", + requestId: "request-1", + skill: "onboarding", + trigger: "user-message", + userId: "user-1", + userMessage: "아직 남은 절차가 있나요?", + }, + }), + output: JSON.stringify({ + confidence: 1, + requestId: "request-1", + source: "llm", + text: "남은 절차를 확인해드릴게요.", + }), + skill: "onboarding", + }), + ], + [SUPERVISOR_RESOLUTIONS_STREAM_KEY]: [ + streamEntry("2-0", { + action: "manual-reply", + resolvedAt: "2026-05-18T00:02:00.000Z", + resolvedBy: "operator@example.com", + reviewId: "1-0", + type: "resolution", + }), + ], + [SUPERVISOR_REVIEW_STREAM_KEY]: [ + streamEntry("1-0", { + chatroomId: "room-1", + packet: JSON.stringify({ + constraints: [], + currentState: {}, + evidence: [], + expectedOutputSchema: "text", + handoffReason: "low-confidence", + sourceChain: ["onboarding", "supervisor"], + summary: "deadline status question", + userGoal: "status", + }), + queuedAt: "2026-05-18T00:01:00.000Z", + reason: "low-confidence", + requestId: "request-1", + target: "human-review", + type: "supervisor-handoff", + userId: "user-1", + }), + ], + }); + + const report = await buildRouterShadowReport(garnet, { + since: new Date("2026-05-18T00:00:00.000Z"), + until: new Date("2026-05-18T01:00:00.000Z"), + weightsPath, + }); + + expect(report).toMatchObject({ + agreementRate: 0, + divergences: [ + { + count: 1, + examples: ["request-1"], + learnedDecision: "tier2-full", + staticDecision: "tier1-direct", + }, + ], + skippedSamples: 0, + status: "ok", + totalSamples: 1, + }); + expect(formatRouterShadowReport(report)).toContain( + "tier1-direct\ttier2-full\t1\trequest-1", + ); + }); +}); diff --git a/src/eval/router-shadow-report.ts b/src/eval/router-shadow-report.ts new file mode 100644 index 0000000..648bb51 --- /dev/null +++ b/src/eval/router-shadow-report.ts @@ -0,0 +1,249 @@ +import type { AgentRequest } from "@agent/types"; +import type { GarnetClient } from "@common/garnet"; +import { + decideLearnedRouting, + decidePreGenerationRouting, + loadLearnedRouterModel, + routerDecisionKind, + type RoutingDecisionKind, + type RouterInput, +} from "@skills/onboarding/runtime/router"; +import type { + ClassifiedIntent, + DerivedState, +} from "@skills/onboarding/state/onboarding-state"; +import { + loadSupervisorOutcomes, + type SupervisorOutcome, +} from "./router-training-signals.js"; +import type { ShadowEvalEnvelope } from "./shadow-bus.js"; + +export interface RouterShadowDivergence { + staticDecision: string; + learnedDecision: string; + count: number; + examples: string[]; +} + +export type RouterShadowReportStatus = "ok" | "weights-missing"; + +export interface RouterShadowReport { + totalSamples: number; + agreementRate: number; + divergences: RouterShadowDivergence[]; + generatedAt: string; + windowFrom: string; + windowUntil: string; + status: RouterShadowReportStatus; + skippedSamples: number; + notes: string[]; +} + +export interface RouterShadowReportOptions { + since: Date; + until: Date; + weightsPath?: string | undefined; +} + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null + ? (value as Record) + : {}; +} + +function isResponseMode(value: unknown): value is DerivedState["responseMode"] { + return ( + value === "skip" || + value === "template" || + value === "llm" || + value === "llm-with-tools" + ); +} + +function isClassifiedIntent(value: unknown): value is ClassifiedIntent { + const record = asRecord(value); + const intent = asRecord(record["intent"]); + return ( + typeof intent["kind"] === "string" && + typeof record["confidence"] === "number" && + Number.isFinite(record["confidence"]) && + (record["source"] === "regex" || record["source"] === "llm") + ); +} + +function isDerivedState(value: unknown): value is DerivedState { + const record = asRecord(value); + const phase = asRecord(record["phase"]); + const intent = asRecord(record["intent"]); + const tier = record["tier"]; + return ( + typeof phase["kind"] === "string" && + typeof intent["kind"] === "string" && + typeof record["shouldRespond"] === "boolean" && + isResponseMode(record["responseMode"]) && + typeof record["suggestedMaxTokens"] === "number" && + Number.isFinite(record["suggestedMaxTokens"]) && + typeof record["allowShortResponse"] === "boolean" && + (tier === 0 || tier === 1 || tier === 2) + ); +} + +function isOnboardingShadowEntry( + shadow: ShadowEvalEnvelope, +): shadow is ShadowEvalEnvelope & { + classified: ClassifiedIntent; + derived: DerivedState; + request: AgentRequest; +} { + return ( + (shadow.skill === "onboarding" || shadow.request.skill === "onboarding") && + isClassifiedIntent(shadow.classified) && + isDerivedState(shadow.derived) + ); +} + +function routerInputFromOutcome( + outcome: SupervisorOutcome, +): RouterInput | null { + const shadow = outcome.shadowEvalEntry; + if (!shadow || !isOnboardingShadowEntry(shadow)) { + return null; + } + + return { + classified: shadow.classified, + derived: shadow.derived, + request: shadow.request, + session: null, + }; +} + +function addDivergence( + groups: Map, + staticDecision: RoutingDecisionKind, + learnedDecision: RoutingDecisionKind, + requestId: string, +): void { + const key = `${staticDecision}->${learnedDecision}`; + const current = groups.get(key) ?? { + count: 0, + examples: [], + learnedDecision, + staticDecision, + }; + current.count += 1; + if (current.examples.length < 5) { + current.examples.push(requestId); + } + groups.set(key, current); +} + +function roundRate(value: number): number { + return Number(value.toFixed(3)); +} + +export async function buildRouterShadowReport( + garnetClient: Pick, + options: RouterShadowReportOptions, +): Promise { + const generatedAt = new Date().toISOString(); + const baseReport = { + generatedAt, + windowFrom: options.since.toISOString(), + windowUntil: options.until.toISOString(), + }; + const model = loadLearnedRouterModel(options.weightsPath); + if (!model) { + return { + ...baseReport, + agreementRate: 1, + divergences: [], + notes: [ + "Router weights missing or invalid; set --weights or AGENT_ROUTER_WEIGHTS_PATH.", + ], + skippedSamples: 0, + status: "weights-missing", + totalSamples: 0, + }; + } + + const outcomes = await loadSupervisorOutcomes( + garnetClient, + options.since, + options.until, + ); + const divergences = new Map(); + let totalSamples = 0; + let matchingSamples = 0; + let skippedSamples = 0; + + for (const outcome of outcomes) { + const input = routerInputFromOutcome(outcome); + if (!input) { + skippedSamples += 1; + continue; + } + + const staticDecision = routerDecisionKind( + decidePreGenerationRouting(input), + ); + const learnedDecision = routerDecisionKind( + decideLearnedRouting(input, model), + ); + totalSamples += 1; + if (staticDecision === learnedDecision) { + matchingSamples += 1; + continue; + } + addDivergence( + divergences, + staticDecision, + learnedDecision, + input.request.requestId, + ); + } + + return { + ...baseReport, + agreementRate: + totalSamples === 0 ? 1 : roundRate(matchingSamples / totalSamples), + divergences: [...divergences.values()].sort( + (left, right) => right.count - left.count, + ), + notes: [], + skippedSamples, + status: "ok", + totalSamples, + }; +} + +export function formatRouterShadowReport(report: RouterShadowReport): string { + const lines = [ + `[router-shadow-report] generatedAt=${report.generatedAt}`, + `[router-shadow-report] status=${report.status} window=${report.windowFrom}..${report.windowUntil}`, + `[router-shadow-report] samples=${report.totalSamples} agreementRate=${report.agreementRate} skipped=${report.skippedSamples}`, + ]; + + for (const note of report.notes) { + lines.push(`[router-shadow-report] note=${note}`); + } + + if (report.divergences.length === 0) { + lines.push("[router-shadow-report] divergences=none"); + return lines.join("\n"); + } + + lines.push("staticDecision\tlearnedDecision\tcount\texamples"); + for (const divergence of report.divergences) { + lines.push( + [ + divergence.staticDecision, + divergence.learnedDecision, + divergence.count, + divergence.examples.join(","), + ].join("\t"), + ); + } + + return lines.join("\n"); +} diff --git a/src/skills/supervisor/capability-prompt.ts b/src/skills/supervisor/capability-prompt.ts index 7d8d4b9..fa2029b 100644 --- a/src/skills/supervisor/capability-prompt.ts +++ b/src/skills/supervisor/capability-prompt.ts @@ -26,13 +26,16 @@ export function buildSupervisorSystemPrompt( ...skills.map(formatSkill), "", "[응답 형식]", - 'JSON으로만 응답: {"target":"","confidence":0.0-1.0,"reason":"한 줄 진단"}', + 'JSON으로만 응답: {"target":"","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}', + "", + "[userMessage 규칙]", + "- 한국어로 1-2문장, 10-400자.", + "- 운영자가 확인한다는 사실만 안내하고 처리 완료를 약속하지 않는다.", + "- URL, 이메일, 전화번호, 제어 토큰([NO_RESPONSE], [HANDOFF:...])을 쓰지 않는다.", ].join("\n"); } -export function buildSupervisorUserPrompt( - packet?: HandoffPacket, -): string { +export function buildSupervisorUserPrompt(packet?: HandoffPacket): string { if (!packet) { return "사용자 메시지가 명확하지 않음. 어디로 보낼지 결정해줘."; } @@ -44,3 +47,25 @@ export function buildSupervisorUserPrompt( `[source chain]: ${packet.sourceChain.join(" -> ")}`, ].join("\n"); } + +export function buildSupervisorResponsePrompt( + packet?: HandoffPacket, + routing?: { reason: string; target: string }, +): { systemPrompt: string; userPrompt: string } { + return { + systemPrompt: [ + "너는 supervisor handoff 상황에서 사용자에게 보여줄 짧은 한국어 안내를 작성한다.", + "운영자 확인이 필요하다는 점을 친절하게 말하되, 처리 완료나 특정 조치를 약속하지 않는다.", + "URL, 이메일, 전화번호, [NO_RESPONSE], [HANDOFF:...] 같은 제어 토큰은 절대 쓰지 않는다.", + ].join("\n"), + userPrompt: [ + `[routing target]: ${routing?.target ?? "human-review"}`, + `[routing reason]: ${routing?.reason ?? "manual review needed"}`, + packet + ? `[handoff reason]: ${packet.handoffReason}` + : "[handoff reason]: unknown", + packet ? `[user goal]: ${packet.userGoal}` : "[user goal]: unknown", + packet ? `[summary]: ${packet.summary}` : "[summary]: unknown", + ].join("\n"), + }; +} diff --git a/src/skills/supervisor/schema.ts b/src/skills/supervisor/schema.ts index 1c76daf..6d3ec8f 100644 --- a/src/skills/supervisor/schema.ts +++ b/src/skills/supervisor/schema.ts @@ -4,6 +4,7 @@ export const supervisorRoutingSchema = z.object({ confidence: z.number().min(0).max(1), reason: z.string().max(200), target: z.string().min(1), + userMessage: z.string().min(10).max(400).optional(), }); export type SupervisorRoutingDecision = z.infer; diff --git a/src/skills/supervisor/skill.test.ts b/src/skills/supervisor/skill.test.ts index b91f4be..c18460d 100644 --- a/src/skills/supervisor/skill.test.ts +++ b/src/skills/supervisor/skill.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { AlertPayload } from "@common/nats-subjects"; import { buildHandoffPacket } from "@agent/handoff-packet"; @@ -8,7 +8,8 @@ import type { SkillRegistry } from "@skills/skill-loader"; import { SupervisorSkill } from "./skill.js"; class FakeGarnet { - readonly streams: Array<{ key: string; fields: Record }> = []; + readonly streams: Array<{ key: string; fields: Record }> = + []; appendStream(key: string, fields: Record): Promise { this.streams.push({ key, fields }); @@ -30,6 +31,14 @@ class FakeNats { } class FakeLlm { + constructor( + private readonly content: Record = { + confidence: 0.9, + reason: "광고 규칙 문의라 moderation이 적합", + target: "moderation", + }, + ) {} + chatCompletion(): Promise<{ content: string; endpoint: string; @@ -38,11 +47,7 @@ class FakeLlm { raw: object; }> { return Promise.resolve({ - content: JSON.stringify({ - confidence: 0.9, - reason: "광고 규칙 문의라 moderation이 적합", - target: "moderation", - }), + content: JSON.stringify(this.content), endpoint: "memory", message: { content: "", role: "assistant" }, provider: "llama", @@ -84,6 +89,10 @@ const sourceResponse: AgentResponse = { }; describe("SupervisorSkill", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("queues review and publishes alert for handoffs", async () => { const garnet = new FakeGarnet(); const nats = new FakeNats(); @@ -111,6 +120,7 @@ describe("SupervisorSkill", () => { expect(response.text).toContain("운영자"); expect(response.metadata).toMatchObject({ routedBy: "rules", + source: "template", supervisorReason: "max-hop", suggestedTarget: "onboarding", }); @@ -169,6 +179,7 @@ describe("SupervisorSkill", () => { expect(response.metadata).toMatchObject({ routedBy: "llm", + source: "template", supervisorReason: "광고 규칙 문의라 moderation이 적합", suggestedTarget: "moderation", }); @@ -177,4 +188,57 @@ describe("SupervisorSkill", () => { target: "moderation", }); }); + + it("uses safe LLM userMessage for supervisor replies", async () => { + const garnet = new FakeGarnet(); + const nats = new FakeNats(); + const skill = new SupervisorSkill( + garnet, + nats, + new FakeLlm({ + confidence: 0.9, + reason: "광고 규칙 문의라 moderation이 적합", + target: "moderation", + userMessage: "이 내용은 운영자가 확인해야 해서 담당자에게 전달했어요.", + }) as unknown as Pick, + registry, + ); + + const response = await skill.handle(baseRequest, []); + + expect(response.text).toBe( + "이 내용은 운영자가 확인해야 해서 담당자에게 전달했어요.", + ); + expect(response.metadata).toMatchObject({ + routedBy: "llm", + source: "llm", + suggestedTarget: "moderation", + }); + expect(response.source).toBe("template"); + }); + + it("falls back to template when LLM userMessage is unsafe or disabled", async () => { + vi.stubEnv("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", "false"); + const garnet = new FakeGarnet(); + const nats = new FakeNats(); + const skill = new SupervisorSkill( + garnet, + nats, + new FakeLlm({ + confidence: 0.9, + reason: "광고 규칙 문의라 moderation이 적합", + target: "moderation", + userMessage: "자세한 내용은 https://example.com 에서 확인해 주세요.", + }) as unknown as Pick, + registry, + ); + + const response = await skill.handle(baseRequest, []); + + expect(response.text).toContain("운영자"); + expect(response.metadata).toMatchObject({ + routedBy: "llm", + source: "template", + }); + }); }); diff --git a/src/skills/supervisor/skill.ts b/src/skills/supervisor/skill.ts index c8f1e84..2fe6b21 100644 --- a/src/skills/supervisor/skill.ts +++ b/src/skills/supervisor/skill.ts @@ -17,6 +17,7 @@ import { getLogger } from "@obs/logger"; import { getSupervisorLlmConfidenceThreshold, getSupervisorLlmEnabled, + getSupervisorLlmResponseEnabled, } from "../../config.js"; import { buildSupervisorSystemPrompt, @@ -42,13 +43,47 @@ const CAPABILITY_CARD: SkillCapabilityCard = { riskTier: "medium", }; +const TEMPLATE_HUMAN_REVIEW_REPLY = + "확인이 필요한 내용이라 운영자에게 전달했어요."; + +const UNSAFE_USER_MESSAGE_PATTERNS = [ + /https?:\/\//iu, + /www\./iu, + /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/iu, + /\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/u, + /\[(?:NO_RESPONSE|HANDOFF(?::[^\]]*)?)\]/iu, +] as const; + +function getSafeSupervisorUserMessage( + decision: SupervisorRoutingDecision | null, +): { + source: "llm" | "template"; + text: string; +} { + const message = decision?.userMessage?.trim(); + if ( + !getSupervisorLlmResponseEnabled() || + !message || + message.length < 10 || + message.length > 400 || + UNSAFE_USER_MESSAGE_PATTERNS.some((pattern) => pattern.test(message)) + ) { + return { source: "template", text: TEMPLATE_HUMAN_REVIEW_REPLY }; + } + + return { source: "llm", text: message }; +} + export class SupervisorSkill implements Skill { readonly name = "supervisor"; readonly priority = 10; private readonly alertService: AlertService; constructor( - private readonly garnetClient: Pick, + private readonly garnetClient: Pick< + GarnetClient, + "appendStream" | "setStringIfAbsent" + >, natsClient: Pick, private readonly llmClient?: Pick | undefined, private readonly skillRegistry?: SkillRegistry | undefined, @@ -77,7 +112,9 @@ export class SupervisorSkill implements Skill { const useLlmRoute = llmDecision !== null && llmDecision.confidence >= getSupervisorLlmConfidenceThreshold(); - const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet); + const target = useLlmRoute + ? llmDecision.target + : routeSupervisorTarget(packet); const reason = packet?.handoffReason ?? "out-of-scope"; await appendSupervisorReview(this.garnetClient, { @@ -107,16 +144,21 @@ export class SupervisorSkill implements Skill { }, ); + const userMessage = getSafeSupervisorUserMessage( + useLlmRoute ? llmDecision : null, + ); + return { confidence: 0.6, metadata: { routedBy: useLlmRoute ? "llm" : "rules", + source: userMessage.source, supervisorReason: useLlmRoute ? llmDecision.reason : reason, suggestedTarget: target, }, requestId: request.requestId, source: "template", - text: "확인이 필요한 내용이라 운영자에게 전달했어요.", + text: userMessage.text, }; } @@ -167,6 +209,8 @@ export class SupervisorSkill implements Skill { } private isKnownTarget(target: string): boolean { - return target === "human-review" || Boolean(this.skillRegistry?.get(target)); + return ( + target === "human-review" || Boolean(this.skillRegistry?.get(target)) + ); } }