diff --git a/IMDS-README.md b/IMDS-README.md new file mode 100644 index 0000000..40c3d31 --- /dev/null +++ b/IMDS-README.md @@ -0,0 +1,544 @@ +# Machine Env Service + +IMDSv2-compatible local metadata daemon for homelab machines. Each machine runs one daemon, and local processes or containers can fetch machine-scoped metadata and allowlisted Infisical secrets through HTTP without holding Infisical credentials. + +## API Reference + +Use `BASE_URL=http://169.254.169.254` on a host with the link-local listener. For local development use `BASE_URL=http://127.0.0.1:8675`. For Docker Desktop on macOS use `BASE_URL=http://host.docker.internal:8675`. + +Issue an IMDSv2-style token: + +```bash +BASE_URL=http://169.254.169.254 + +TOKEN=$( + curl -fsS -X PUT \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60" \ + "$BASE_URL/latest/api/token" +) +``` + +Token TTL must be between `1` and `21600` seconds. All metadata, dynamic data, user-data, and secret endpoints require: + +```text +X-aws-ec2-metadata-token: +``` + +| Method | Path | Description | +| ------ | -------------------------------------------- | ---------------------------------------------------------------------------------- | +| `PUT` | `/latest/api/token` | Issue an opaque metadata token. Requires `X-aws-ec2-metadata-token-ttl-seconds`. | +| `GET` | `/latest/meta-data/` | List top-level metadata keys. | +| `GET` | `/latest/meta-data/environment` | Infisical environment configured for this machine. | +| `GET` | `/latest/meta-data/hostname` | Machine hostname. | +| `GET` | `/latest/meta-data/local-hostname` | Local hostname. | +| `GET` | `/latest/meta-data/local-ipv4` | Primary local IPv4. | +| `GET` | `/latest/meta-data/mac` | Primary MAC address. | +| `GET` | `/latest/meta-data/machine-id` | Machine ID from config or OS. | +| `GET` | `/latest/meta-data/role` | Optional machine role. Returns `404` if unset. | +| `GET` | `/latest/meta-data/tags/` | List tag namespaces. Currently `instance/`. | +| `GET` | `/latest/meta-data/tags/` | Read a machine tag by key. | +| `GET` | `/latest/meta-data/tags/instance/` | List machine tag keys. | +| `GET` | `/latest/meta-data/tags/instance/` | Read a machine tag by key. | +| `GET` | `/latest/meta-data/secrets/` | List allowlisted secret keys currently present in cache. | +| `GET` | `/latest/meta-data/secrets/` | Read one allowlisted secret value as plain text. | +| `GET` | `/latest/meta-data/secrets-bundle` | Read all cached allowlisted secrets as JSON. Requires `ALLOW_SECRETS_BUNDLE=true`. | +| `GET` | `/latest/dynamic/` | List dynamic-data categories. | +| `GET` | `/latest/dynamic/instance-identity/` | List instance identity fields. | +| `GET` | `/latest/dynamic/instance-identity/document` | Machine identity document as JSON. | +| `GET` | `/latest/user-data` | Configured user-data as plain text. Returns `404` if unset. | + +### Curl Examples + +List top-level metadata: + +```bash +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/meta-data/" +``` + +Read basic machine metadata: + +```bash +curl -fsS -H "X-aws-ec2-metadata-token: $TOKEN" "$BASE_URL/latest/meta-data/hostname" +curl -fsS -H "X-aws-ec2-metadata-token: $TOKEN" "$BASE_URL/latest/meta-data/local-ipv4" +curl -fsS -H "X-aws-ec2-metadata-token: $TOKEN" "$BASE_URL/latest/meta-data/mac" +curl -fsS -H "X-aws-ec2-metadata-token: $TOKEN" "$BASE_URL/latest/meta-data/machine-id" +curl -fsS -H "X-aws-ec2-metadata-token: $TOKEN" "$BASE_URL/latest/meta-data/environment" +``` + +Read role: + +```bash +curl -i \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/meta-data/role" +``` + +List and read tags: + +```bash +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/meta-data/tags/instance/" + +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/meta-data/tags/instance/site" +``` + +Read the identity document: + +```bash +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/dynamic/instance-identity/document" +``` + +Read user-data: + +```bash +curl -i \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/user-data" +``` + +List and read secrets: + +```bash +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/meta-data/secrets/" + +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/meta-data/secrets/FOO" +``` + +Read the optional secret bundle: + +```bash +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + "$BASE_URL/latest/meta-data/secrets-bundle" +``` + +Expected failure cases: + +```bash +# Missing token: 401 +curl -i "$BASE_URL/latest/meta-data/" + +# SSRF guard: 403 +curl -i \ + -H "X-Forwarded-For: 1.2.3.4" \ + "$BASE_URL/latest/meta-data/" + +# Token endpoint with wrong method: 405 +curl -i "$BASE_URL/latest/api/token" +``` + +## Configuration + +The daemon reads its own dotenv-style config file. + +Default paths: + +```text +Linux: /etc/machine-env-service/env +macOS: /usr/local/etc/machine-env-service/env +``` + +Override with: + +```bash +MACHINE_ENV_CONFIG=/path/to/env +``` + +Minimum config: + +```env +INFISICAL_ACCESS_TOKEN= +INFISICAL_PROJECT_ID= +INFISICAL_ENVIRONMENT=prod + +KNOWN_SECRET_KEYS=FOO,BAR +METADATA_PORT=8675 +REFRESH_INTERVAL_SEC=60 +``` + +Optional machine metadata: + +```env +MACHINE_ID= +MACHINE_HOSTNAME= +MACHINE_LOCAL_HOSTNAME= +MACHINE_LOCAL_IPV4= +MACHINE_MAC= +MACHINE_ROLE=homelab-node +MACHINE_TAGS=site=home,tier=prod +USER_DATA= +USER_DATA_FILE= +``` + +If machine fields are empty, the daemon derives hostname, IPv4, MAC, and machine-id from the OS where possible. + +Secret allowlist variables are checked in this order: + +```text +KNOWN_SECRET_KEYS +EXPOSED_SECRET_KEYS +SECRET_ALLOWLIST +SECRET_WHITELIST +``` + +Values can be comma-separated or whitespace-separated. + +## Local Development + +Build: + +```bash +npm ci +npm run build +``` + +Run locally without binding `169.254.169.254:80`: + +```bash +NODE_ENV=dev LINK_LOCAL_LISTENER_ENABLED=false npm start +``` + +Request a token: + +```bash +TOKEN=$( + curl -fsS -X PUT \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60" \ + http://127.0.0.1:8675/latest/api/token +) +``` + +Fetch metadata: + +```bash +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + http://127.0.0.1:8675/latest/meta-data/hostname +``` + +Fetch a secret: + +```bash +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + http://127.0.0.1:8675/latest/meta-data/secrets/FOO +``` + +Or use: + +```bash +METADATA_BASE_URL=http://127.0.0.1:8675 ./examples/fetch-secret.sh FOO +``` + +## Ubuntu Install + +The included systemd units assume the app lives at `/opt/machine-env-service`. + +Install files: + +```bash +sudo mkdir -p /opt/machine-env-service +sudo chown -R "$USER:$USER" /opt/machine-env-service + +rsync -a \ + --exclude .git \ + --exclude node_modules \ + --exclude dist \ + --exclude env-vars \ + ./ /opt/machine-env-service/ + +cd /opt/machine-env-service +npm ci +npm run build +``` + +Create service user: + +```bash +sudo useradd --system --home /nonexistent --shell /usr/sbin/nologin machine-env || true +``` + +Create config: + +```bash +sudo install -d -o machine-env -g machine-env -m 0700 /etc/machine-env-service +sudo cp deploy/systemd/machine-env-service.env.example /etc/machine-env-service/env +sudo nano /etc/machine-env-service/env +sudo chown machine-env:machine-env /etc/machine-env-service/env +sudo chmod 0600 /etc/machine-env-service/env +``` + +Install and start systemd units: + +```bash +sudo cp deploy/systemd/machine-env-setup-link-local.service /etc/systemd/system/ +sudo cp deploy/systemd/machine-env-service.service /etc/systemd/system/ + +sudo systemctl daemon-reload +sudo systemctl enable --now machine-env-setup-link-local.service +sudo systemctl enable --now machine-env-service.service +``` + +Check logs: + +```bash +systemctl status machine-env-service.service +journalctl -u machine-env-service.service -f +``` + +Test the link-local endpoint: + +```bash +TOKEN=$( + curl -fsS -X PUT \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60" \ + http://169.254.169.254/latest/api/token +) + +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + http://169.254.169.254/latest/meta-data/ +``` + +## macOS Install + +The included launchd plists assume: + +```text +App path: /opt/machine-env-service +Node path: /usr/local/bin/node +Config path: /usr/local/etc/machine-env-service/env +``` + +Check your Node path first: + +```bash +command -v node +``` + +If it is not `/usr/local/bin/node`, either update `deploy/launchd/us.dahyeon.machine-env-service.plist` before installing or create a system-visible symlink: + +```bash +sudo mkdir -p /usr/local/bin +sudo ln -sf "$(command -v node)" /usr/local/bin/node +``` + +When Node comes from `nvm`, the symlink is the quickest local setup. A Homebrew Node installed at a stable path is usually better for a long-lived daemon. + +Install files: + +```bash +sudo mkdir -p /opt/machine-env-service +sudo chown -R "$(id -un):admin" /opt/machine-env-service + +rsync -a \ + --exclude .git \ + --exclude node_modules \ + --exclude dist \ + --exclude env-vars \ + ./ /opt/machine-env-service/ + +cd /opt/machine-env-service +npm ci +npm run build +``` + +Create config: + +```bash +sudo install -d -m 0700 /usr/local/etc/machine-env-service +sudo cp deploy/launchd/machine-env-service.env.example /usr/local/etc/machine-env-service/env +sudo nano /usr/local/etc/machine-env-service/env +sudo chown root:wheel /usr/local/etc/machine-env-service/env +sudo chmod 0600 /usr/local/etc/machine-env-service/env +``` + +Install and start launchd daemons: + +```bash +sudo cp deploy/launchd/us.dahyeon.machine-env-setup-link-local.plist /Library/LaunchDaemons/ +sudo cp deploy/launchd/us.dahyeon.machine-env-service.plist /Library/LaunchDaemons/ + +sudo chown root:wheel /Library/LaunchDaemons/us.dahyeon.machine-env-*.plist +sudo chmod 0644 /Library/LaunchDaemons/us.dahyeon.machine-env-*.plist + +sudo launchctl bootstrap system /Library/LaunchDaemons/us.dahyeon.machine-env-setup-link-local.plist +sudo launchctl bootstrap system /Library/LaunchDaemons/us.dahyeon.machine-env-service.plist +``` + +Check logs: + +```bash +sudo launchctl print system/us.dahyeon.machine-env-service +tail -f /var/log/machine-env-service.log +tail -f /var/log/machine-env-service.err.log +``` + +Test: + +```bash +TOKEN=$( + curl -fsS -X PUT \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60" \ + http://169.254.169.254/latest/api/token +) + +curl -fsS \ + -H "X-aws-ec2-metadata-token: $TOKEN" \ + http://169.254.169.254/latest/meta-data/hostname +``` + +Docker Desktop on macOS usually cannot route container traffic to host `169.254.169.254`. Use the LAN listener instead: + +```text +http://host.docker.internal:8675/latest/meta-data/... +``` + +## Update + +Restarting the daemon invalidates issued IMDSv2 tokens. Clients should request a new token after an update. + +If `/opt/machine-env-service` is a git checkout: + +```bash +cd /opt/machine-env-service +git pull +npm ci +npm run build +``` + +If you deploy from another checkout: + +```bash +rsync -a --delete \ + --exclude .git \ + --exclude node_modules \ + --exclude dist \ + --exclude env-vars \ + ./ /opt/machine-env-service/ + +cd /opt/machine-env-service +npm ci +npm run build +``` + +Do not overwrite the machine config file: + +```text +Linux: /etc/machine-env-service/env +macOS: /usr/local/etc/machine-env-service/env +``` + +Restart after code-only updates: + +```bash +sudo systemctl restart machine-env-service.service +``` + +```bash +sudo launchctl kickstart -k system/us.dahyeon.machine-env-service +``` + +If systemd unit files changed: + +```bash +sudo cp deploy/systemd/machine-env-setup-link-local.service /etc/systemd/system/ +sudo cp deploy/systemd/machine-env-service.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl restart machine-env-setup-link-local.service +sudo systemctl restart machine-env-service.service +``` + +If launchd plist files changed: + +```bash +sudo cp deploy/launchd/us.dahyeon.machine-env-setup-link-local.plist /Library/LaunchDaemons/ +sudo cp deploy/launchd/us.dahyeon.machine-env-service.plist /Library/LaunchDaemons/ +sudo chown root:wheel /Library/LaunchDaemons/us.dahyeon.machine-env-*.plist +sudo chmod 0644 /Library/LaunchDaemons/us.dahyeon.machine-env-*.plist + +sudo launchctl bootout system /Library/LaunchDaemons/us.dahyeon.machine-env-service.plist 2>/dev/null || true +sudo launchctl bootout system /Library/LaunchDaemons/us.dahyeon.machine-env-setup-link-local.plist 2>/dev/null || true +sudo launchctl bootstrap system /Library/LaunchDaemons/us.dahyeon.machine-env-setup-link-local.plist +sudo launchctl bootstrap system /Library/LaunchDaemons/us.dahyeon.machine-env-service.plist +``` + +Check after updating: + +```bash +systemctl status machine-env-service.service +journalctl -u machine-env-service.service -n 100 +``` + +```bash +sudo launchctl print system/us.dahyeon.machine-env-service +tail -n 100 /var/log/machine-env-service.err.log +``` + +## Operations + +Restart: + +```bash +sudo systemctl restart machine-env-service.service +``` + +```bash +sudo launchctl kickstart -k system/us.dahyeon.machine-env-service +``` + +Stop: + +```bash +sudo systemctl stop machine-env-service.service +``` + +```bash +sudo launchctl bootout system /Library/LaunchDaemons/us.dahyeon.machine-env-service.plist +``` + +## Troubleshooting + +If `curl http://169.254.169.254/latest/api/token` fails with `Couldn't connect to server`, check the link-local alias and daemon separately: + +```bash +ifconfig lo0 | grep 169.254.169.254 +sudo launchctl print system/us.dahyeon.machine-env-service +``` + +If launchd shows `last exit code = 78: EX_CONFIG` and the plist points to `/usr/local/bin/node`, verify that Node exists there: + +```bash +ls -l /usr/local/bin/node +command -v node +``` + +Fix a missing `/usr/local/bin/node` with: + +```bash +sudo mkdir -p /usr/local/bin +sudo ln -sf "$(command -v node)" /usr/local/bin/node +sudo launchctl kickstart -k system/us.dahyeon.machine-env-service +``` + +Then retry token issuance. + +## Security Notes + +- Keep the config file mode at `0600`. +- Do not commit `env-vars/*.env`, production config files, or access tokens. +- Only allowlist secrets that should be readable by local machine workloads. +- Block `METADATA_PORT` from untrusted networks. +- Requests with `X-Forwarded-For` are rejected. +- Metadata requests require IMDSv2-style tokens; token TTL is `1..21600` seconds. diff --git a/docs/portfolio-lolympus-agent.md b/docs/portfolio-lolympus-agent.md new file mode 100644 index 0000000..c9614d9 --- /dev/null +++ b/docs/portfolio-lolympus-agent.md @@ -0,0 +1,575 @@ +# Lolympus Agent 포트폴리오 정리본 + +> Confluence에 그대로 옮겨도 읽히도록 작성한 상세 설명 자료입니다. +> Mermaid 다이어그램은 Confluence Mermaid 매크로나 Mermaid 지원 플러그인에 붙여 넣으면 됩니다. + +## 1. 프로젝트 한 줄 소개 + +Lolympus Agent는 게임 커뮤니티의 신규 멤버 온보딩을 자동화하는 TypeScript 기반 LLM Agent입니다. +카카오톡 오픈채팅 본체 서비스가 NATS request/reply로 요청을 보내면, Agent가 Redis/Garnet에 저장된 온보딩 세션, 최근 대화, MongoDB 채팅방 맥락, LLM 응답, 도구 호출 결과를 조합해 신규 멤버에게 필요한 안내만 짧고 정확하게 반환합니다. + +핵심은 단순 챗봇이 아니라, 커뮤니티 규칙과 유저 상태를 기반으로 LLM을 제어하는 운영형 온보딩 에이전트라는 점입니다. + +## 2. 포트폴리오 요약 + +| 항목 | 내용 | +| --- | --- | +| 프로젝트명 | Lolympus Agent | +| 목적 | 게임 커뮤니티 신규 멤버 등록 절차 자동 안내 | +| 주요 기능 | 닉네임 형식 안내, 하트 인증 안내, 진행상황 안내, 마감 임박 안내, 디스코드 닉네임 확인, 잡담/타인 멘션 응답 스킵 | +| 핵심 기술 | TypeScript, Node.js, NATS, Redis/Garnet, MongoDB/Mongoose, OpenTelemetry, Pino, Zod, Vitest, LLM provider routing | +| 아키텍처 키워드 | Event-driven, request/reply agent, state-driven orchestration, tool calling, shadow evaluation, circuit breaker, background worker | +| 운영 포인트 | 요청 단위 tracing/logging, LLM endpoint circuit breaker, shadow eval sampling, alert subject 발행, Redis Stream 기반 worker/DLQ | + +## 3. 문제 정의 + +커뮤니티 신규 멤버 등록은 반복적인 안내가 많지만, 실제 응답은 단순 FAQ보다 복잡합니다. + +1. 유저마다 남은 절차가 다릅니다. + 예를 들어 닉네임은 완료했지만 하트 인증은 안 했을 수 있고, 1단계 이후 디스코드 프로필 단계가 남아 있을 수도 있습니다. + +2. 질문 의도가 짧고 모호합니다. + "이제 뭐 남았죠?", "어디서 해요?", "닉변 됐나요?" 같은 메시지는 세션 상태와 최근 대화 맥락을 함께 봐야 정확히 답할 수 있습니다. + +3. LLM만 사용하면 규칙 위반 가능성이 있습니다. + 디스코드 안내를 너무 일찍 하거나, 잘못된 닉네임 예시를 들거나, 유저가 묻지 않은 절차를 길게 붙이는 문제가 생길 수 있습니다. + +4. 운영 환경에서는 응답 시간과 장애 대응도 중요합니다. + LLM endpoint 장애, 네트워크 지연, 도구 호출 실패가 있어도 서비스 전체가 멈추지 않아야 합니다. + +이 프로젝트는 위 문제를 해결하기 위해 "결정론적 상태 판단 + LLM 응답 생성 + validator + fallback template + shadow eval" 구조로 설계했습니다. + +## 4. 전체 아키텍처 + +```mermaid +flowchart LR + Main["카카오톡 본체 서비스"] -->|"NATS request/reply
lolympus-agent.request"| Agent["Agent
src/agent/agent.ts"] + + Agent -->|"대화 조회/저장"| Conv[("Garnet/Redis
conversation history")] + Agent --> Skill["OnboardingSkill
src/skills/onboarding/skill.ts"] + + Skill -->|"온보딩 세션 조회"| Session[("Garnet/Redis
lolympus:onboarding:*")] + Skill -->|"최근 방 대화 조회"| Mongo[("MongoDB
events collection")] + Skill --> Classifier["Intent Classifier
regex first, LLM fallback"] + Skill --> Derived["State Derivation
phase, intent, tier"] + Skill --> Prompt["Prompt Builder
persona/policy/state/examples/context"] + Skill --> Orchestrator["Orchestrator
tier 0/1/2"] + Orchestrator --> LLM["LlmClient
provider routing + circuit breaker"] + LLM --> Provider1["CLI Proxy"] + LLM --> Provider2["OpenRouter"] + LLM --> Provider3["Local Llama"] + + Orchestrator --> Tools["Tool Loop
Zod schema tools"] + Tools -->|"Discord lookup"| Main + Tools -->|"rule/session/context lookup"| Session + + Skill --> Validator["Validator
state consistency, URL whitelist, length"] + Skill -->|"shadow eval job"| BgStream[("Redis Stream
lolympus-agent:background")] + BgStream --> Worker["Background Worker
src/workers/background.ts"] + Worker --> Eval[("Eval results stream")] + Worker --> Alert["NATS alert
lolympus-agent.alert"] + + Agent -->|"AgentResponse"| Main +``` + +## 5. 요청 처리 시퀀스 + +```mermaid +sequenceDiagram + autonumber + participant Main as 카카오톡 본체 + participant NATS as NATS + participant Agent as Agent + participant Conv as ConversationStore + participant Skill as OnboardingSkill + participant Redis as Garnet/Redis + participant Mongo as MongoDB RoomContext + participant LLM as LlmClient + participant Eval as ShadowEvalBus + + Main->>NATS: request(lolympus-agent.request, AgentRequest) + NATS->>Agent: raw message 전달 + Agent->>Agent: request schema 검증 + Agent->>Agent: AsyncLocalStorage request context 생성 + Agent->>Conv: getHistory(chatroomId, userId) + Conv->>Redis: lolympus-agent:conv:* 조회 + Redis-->>Conv: 최근 대화 + Conv-->>Agent: history + + opt userMessage가 있는 경우 + Agent->>Conv: appendMessage(user) + Conv->>Redis: conversation 저장 + end + + Agent->>Skill: handle(request, history) + Skill->>Redis: 온보딩 세션 조회 + Redis-->>Skill: OnboardingSession + Skill->>Skill: classify(trigger, userMessage) + Skill->>Skill: deriveState(session, trigger, intent) + + alt shouldRespond = false + Skill-->>Agent: empty response + else template mode + Skill->>Skill: template fallback 생성 + Skill-->>Agent: template response + else LLM mode + par context load + Skill->>Redis: blackboard 조회 + Skill->>Redis: summary 조회 + Skill->>Mongo: recent room messages 조회 + end + Skill->>Skill: PromptBuilder.build() + Skill->>LLM: chatCompletion(messages, tier/model/deadline) + LLM-->>Skill: draft response + Skill->>Skill: validateOnboardingResponse() + alt validator reject or LLM failure + Skill->>Skill: template fallback + end + Skill-->>Agent: final response + end + + opt 응답 text가 있는 경우 + Skill->>Redis: blackboard update + Skill->>Eval: shadow eval publish + Eval->>Redis: XADD lolympus-agent:background + Agent->>Conv: appendMessage(assistant) + end + + Agent->>NATS: respond(AgentResponse) + NATS-->>Main: response +``` + +## 6. 디스코드 닉네임 확인 시퀀스 + +디스코드 닉네임 확인은 LLM이 직접 추측하지 않도록 tool calling으로 분리했습니다. +LLM은 "조회가 필요한 상황"을 판단하고, 실제 확인은 `verify-discord-nickname` 도구가 NATS로 본체 서비스에 요청합니다. + +```mermaid +sequenceDiagram + autonumber + participant User as 유저 + participant Agent as OnboardingSkill + participant Orchestrator as Orchestrator + participant LLM as LLM + participant ToolLoop as ToolLoop + participant Tool as verify-discord-nickname + participant Main as 본체 Discord Lookup + participant Redis as Garnet/Redis + + User->>Agent: "디코 닉변 됐나요?" + Agent->>Agent: intent = discord-verify-question + Agent->>Agent: responseMode = llm-with-tools, tier = 1 + Agent->>Orchestrator: run(derived, messages, tools) + Orchestrator->>LLM: planner JSON 요청 + LLM-->>Orchestrator: suggestedTools = [verify-discord-nickname] + Orchestrator->>ToolLoop: responder with native/Hermes tools + ToolLoop->>LLM: tool 사용 가능한 메시지 전달 + LLM-->>ToolLoop: tool_call verify-discord-nickname + ToolLoop->>Tool: execute() + Tool->>Tool: 세션 닉네임 형식 검사 + Tool->>Main: NATS request(lolympus.discord.lookup-members) + Main-->>Tool: 후보 Discord member 목록 + Tool->>Tool: Kakao nickname 기반 Discord nickname rule 검증 + opt Discord memberId 확인됨 + Tool->>Redis: session.discordMemberId 저장 + end + Tool-->>ToolLoop: 구조화된 결과 반환 + ToolLoop->>LLM: tool result 전달 + LLM-->>ToolLoop: 최종 한국어 응답 + ToolLoop-->>Orchestrator: final response + Orchestrator-->>Agent: response +``` + +## 7. Shadow Eval 및 Alert 시퀀스 + +운영 트래픽 전체를 동기 평가하면 응답 지연이 커지므로, 실제 응답과 평가를 분리했습니다. +Agent는 샘플링된 요청만 Redis Stream에 넣고, Background Worker가 비동기로 validator와 LLM judge를 실행합니다. + +```mermaid +sequenceDiagram + autonumber + participant Skill as OnboardingSkill + participant Stream as Redis Stream + participant Worker as ShadowWorker + participant Rule as Rule Judge + participant LLMJudge as LLM Judge + participant Results as Eval Results Stream + participant Alert as AlertService + participant NATS as NATS + + Skill->>Stream: XADD type=shadow-eval + Worker->>Stream: XREADGROUP bg-worker + Worker->>Rule: validator 기반 rule judge + Worker->>LLMJudge: friendliness/accuracy/concision/scope 평가 + Worker->>Results: XADD shadow-eval-result + Worker->>Worker: 최근 5분 latency/score/error 집계 + + alt p95 latency, score drop, LLM error rate, circuit open 임계값 초과 + Worker->>Alert: sendAlert() + Alert->>Stream: dedupe key SET NX + Alert->>NATS: publish lolympus-agent.alert + end + + Worker->>Stream: XACK +``` + +## 8. 상태 모델 + +온보딩 응답은 "지금 유저가 어느 단계인지"와 "무엇을 물었는지"를 분리해 판단합니다. + +```mermaid +stateDiagram-v2 + [*] --> fresh + fresh --> both_pending: 세션 있음, 닉네임/사진 미완료 + both_pending --> nickname_pending: 사진 완료 + both_pending --> photo_pending: 닉네임 완료 + nickname_pending --> step1_completed: 닉네임 완료 + photo_pending --> step1_completed: 사진 완료 + step1_completed --> completed: 확장 추적 항목 모두 완료 + step1_completed --> deadline_near: 마감 임박 + both_pending --> nickname_rejected: nickname-invalid trigger + both_pending --> photo_rejected: photo-failed trigger + nickname_pending --> nickname_rejected: nickname-invalid trigger + photo_pending --> photo_rejected: photo-failed trigger + fresh --> timed_out: status = timed_out + completed --> [*] + timed_out --> [*] +``` + +주요 intent는 다음과 같습니다. + +| Intent | 의미 | 처리 방향 | +| --- | --- | --- | +| `greeting` | 인사 | 짧게 응답 | +| `acknowledgement` | 감사/확인 | 짧게 응답 | +| `status-question` | 남은 절차 질문 | 완료된 절차 제외, 남은 것만 안내 | +| `help-how` | 방법 질문 | 해당 단계 방법만 안내 | +| `help-where` | 위치 질문 | 위치/경로만 안내 | +| `nickname-question` | 카카오톡 닉네임 질문 | 형식/오류 원인 안내 | +| `photo-question` | 하트 인증 질문 | 하트 인증만 안내 | +| `discord-question` | 디스코드 일반 질문 | 1단계 완료 여부에 따라 제한 | +| `discord-verify-question` | 디스코드 닉네임 확인 | tool calling으로 실제 조회 | +| `chitchat`, `off-topic-to-other` | 잡담/타인 대상 메시지 | `[NO_RESPONSE]` 또는 빈 응답 | + +## 9. 핵심 구현 포인트 + +### 9.1 NATS 기반 Agent 레이어 + +관련 파일: `src/main.ts`, `src/agent/agent.ts`, `src/agent/types.ts` + +- `src/main.ts`에서 환경변수 로딩, logger/tracing 초기화, NATS/Garnet/MongoDB 연결, LLM client와 skill 등록을 수행합니다. +- Agent는 `lolympus-agent.request` subject를 구독하고 request/reply 패턴으로 응답합니다. +- 모든 요청은 `AsyncLocalStorage` 기반 request context로 감싸져 requestId, chatroomId, userId, trigger가 로그와 span에 자동 포함됩니다. +- 요청 전후로 conversation history를 Redis에 저장해 multi-turn 응답 품질을 유지합니다. + +### 9.2 Regex-first intent classifier + +관련 파일: `src/skills/onboarding/state/classifier.ts`, `src/skills/onboarding/message-filter.ts` + +Classifier는 비용과 latency를 줄이기 위해 regex를 먼저 사용합니다. + +- trigger가 명확한 이벤트는 바로 intent로 매핑합니다. 예: `nickname-invalid` -> `nickname-question` +- 인사, 감사, 진행상황 질문, 위치 질문, 디스코드 확인 질문 등 자주 나오는 표현은 regex로 처리합니다. +- regex confidence가 낮은 경우에만 LLM JSON classifier로 fallback합니다. + +이 구조는 운영상 유리합니다. 흔한 질문은 LLM 호출 없이 빠르게 처리하고, 애매한 질문만 LLM에 맡길 수 있습니다. + +### 9.3 Derived State로 응답 모드 결정 + +관련 파일: `src/skills/onboarding/state/derive.ts`, `src/skills/onboarding/state/transitions.ts` + +`deriveState()`는 세션, trigger, intent를 조합해 다음 값을 결정합니다. + +- `phase`: 현재 온보딩 단계 +- `shouldRespond`: 응답할지 스킵할지 +- `responseMode`: `skip`, `template`, `llm`, `llm-with-tools` +- `tier`: LLM orchestration 단계 +- `suggestedMaxTokens`: 상황별 token budget +- `allowShortResponse`: 짧은 응답 허용 여부 + +예를 들어 닉네임 오류 상태는 `llm-with-tools`, tier 1로 승격됩니다. 반대로 잡담이나 타인 멘션은 응답하지 않습니다. + +### 9.4 Layered Prompt Builder + +관련 파일: `src/skills/onboarding/prompts/builder.ts`, `src/skills/onboarding/prompts/layers.ts` + +Prompt는 하나의 거대한 system prompt가 아니라 여러 layer를 조립합니다. + +1. `personaLayer`: 커뮤니티 선배 역할과 핵심 규칙 +2. `policyLayer`: intent별 답변 정책 +3. `stateLayer`: 현재 세션 상태와 남은 절차 +4. `examplesLayer`: intent별 few-shot 예시 +5. `contextLayer`: 최근 대화, 요약, 방 분위기, blackboard 메모 + +이 방식의 장점은 변경 범위를 줄일 수 있다는 점입니다. +예를 들어 디스코드 응답 정책을 수정할 때 전체 prompt를 건드리지 않고 `policyLayer`와 관련 예시만 조정할 수 있습니다. + +### 9.5 Tier 기반 LLM Orchestration + +관련 파일: `src/skills/onboarding/runtime/orchestrator.ts`, `src/skills/onboarding/runtime/critic.ts` + +모든 요청을 동일하게 처리하지 않고 난이도에 따라 tier를 나눕니다. + +| Tier | 흐름 | 사용 상황 | +| --- | --- | --- | +| Tier 0 | Direct responder | 인사, 상태 질문, 일반 안내 | +| Tier 1 | Planner -> Responder | 닉네임 오류, 디스코드 확인, 마감 임박 | +| Tier 2 | Planner -> Responder -> Critic/Rewrite | 더 높은 정확도가 필요한 복잡 상황 | + +Orchestrator는 request deadline을 확인해 남은 시간이 부족하면 고비용 tier를 포기하고 더 빠른 경로로 내려갑니다. +즉, 품질과 응답 시간을 함께 관리합니다. + +### 9.6 LLM Provider Routing과 Circuit Breaker + +관련 파일: `src/llm/llm-client.ts`, `src/llm/routing.ts`, `src/llm/circuit.ts` + +LLM client는 provider 우선순위를 적용합니다. + +1. CLI Proxy +2. OpenRouter +3. Local Llama + +각 endpoint는 health check와 circuit breaker 상태를 거칩니다. +연속 실패가 발생하면 해당 endpoint를 cooldown 상태로 두고 다른 provider로 fallback합니다. + +### 9.7 Tool Calling 호환 계층 + +관련 파일: `src/tools/*`, `src/skills/onboarding/runtime/tool-loop.ts` + +Tool interface는 Zod schema를 포함합니다. + +```ts +interface Tool { + name: string; + description: string; + argsSchema: ZodType; + resultSchema: ZodType; + execute(args: A, ctx: ToolContext): Promise; +} +``` + +지원 도구: + +| Tool | 역할 | +| --- | --- | +| `validate-nickname` | 카카오톡 닉네임 형식 검사 | +| `check-onboarding-progress` | 현재 남은 절차 구조화 | +| `lookup-community-rule` | 나이 제한, 티어 코드, 하트 규칙 등 상수 조회 | +| `get-room-context` | 최근 채팅방 메시지 조회 | +| `verify-discord-nickname` | NATS로 본체에 Discord member lookup 요청 | + +Parser는 OpenAI native `tool_calls`와 Hermes `` 포맷을 모두 처리합니다. +덕분에 provider가 바뀌어도 Agent runtime은 같은 tool interface를 사용할 수 있습니다. + +### 9.8 LLM 응답 Validator + +관련 파일: `src/skills/onboarding/runtime/validator.ts` + +LLM 응답은 그대로 내보내지 않고 후처리와 검증을 거칩니다. + +- `` 제거 +- 허용되지 않은 URL 제거 +- 마크다운 제거 +- 최대 길이 제한 +- 빈 응답과 너무 짧은 응답 reject +- 디스코드 안내가 1단계 완료 전 과하게 나가는지 검사 +- 잘못된 티어 코드가 포함된 닉네임 예시 reject +- intent별 기대 키워드 soft warning +- 반복 phrase soft warning + +reject된 응답은 재시도하거나 template fallback으로 내려갑니다. + +### 9.9 Memory와 Context + +관련 파일: `src/conversation/conversation-store.ts`, `src/conversation/summarizer.ts`, `src/skills/onboarding/state/blackboard.ts`, `src/room-context/room-context-service.ts` + +대화 품질을 위해 세 가지 context를 사용합니다. + +| Context | 저장 위치 | 목적 | +| --- | --- | --- | +| Conversation history | `lolympus-agent:conv:{chatroomId}:{userId}` | 최근 user/assistant turn 유지 | +| Conversation summary | `lolympus-agent:summary:{chatroomId}:{userId}` | 긴 대화 압축 | +| Blackboard | `lolympus-agent:bb:{chatroomId}:{userId}` | 최근 intent, 열린 질문, 닉네임 시도, tone memory 저장 | +| Room context | MongoDB `events` collection | 최근 방 분위기와 타인 대상 메시지 판단 | + +5턴 이상 누적되면 background stream에 summarize job을 넣고 worker가 비동기로 요약합니다. + +### 9.10 Eval, Observability, Alert + +관련 파일: `src/eval/*`, `src/observability/*`, `src/workers/background.ts` + +운영 품질 관리를 위해 다음 구조를 갖고 있습니다. + +- Pino logger: 환경별 log level, secret redaction +- OpenTelemetry spans: agent, skill, classifier, LLM, tool 단위 span +- ShadowEvalBus: 실제 응답을 샘플링해 Redis Stream에 기록 +- ShadowWorker: rule judge, LLM judge, summary job 처리 +- AlertService: latency, eval score drop, LLM error rate, open circuit 발생 시 NATS alert 발행 +- DLQ: background job이 반복 실패하면 `lolympus-agent:background:dlq`로 이동 + +## 10. 주요 데이터 계약 + +### 10.1 Agent Request + +```ts +interface AgentRequest { + requestId: string; + skill: string; + chatroomId: string; + userId: string; + nickname: string; + trigger: string; + userMessage?: string; + context?: Record; +} +``` + +### 10.2 Agent Response + +```ts +interface AgentResponse { + requestId: string; + text: string; + confidence: number; + source: "llm" | "template"; + metadata?: Record; +} +``` + +### 10.3 Redis/Garnet Key 설계 + +| Key | 목적 | +| --- | --- | +| `lolympus:onboarding:{chatroomId}:{userId}` | 본체가 작성하는 온보딩 세션. Agent는 주로 read | +| `lolympus-agent:conv:{chatroomId}:{userId}` | 최근 대화 | +| `lolympus-agent:summary:{chatroomId}:{userId}` | 대화 요약 | +| `lolympus-agent:bb:{chatroomId}:{userId}` | 온보딩 blackboard | +| `lolympus-agent:background` | shadow eval 및 summarize job stream | +| `lolympus-agent:background:dlq` | worker 실패 job DLQ | +| `lolympus-agent:eval:results` | shadow eval 결과 | +| `lolympus-agent:circuit:*` | LLM endpoint circuit breaker 상태 | +| `lolympus-agent:alert:dedupe:*` | alert rate limit/dedupe | + +### 10.4 NATS Subject + +| Subject | 방향 | 목적 | +| --- | --- | --- | +| `lolympus-agent.request` | 본체 -> Agent | 온보딩 응답 요청 | +| `lolympus.discord.lookup-members` | Agent -> 본체 | Discord member 조회 | +| `lolympus-agent.alert` | Agent worker -> 본체 | 운영 alert | +| `lolympus-agent.eval.report` | Eval CLI -> 본체/관찰자 | eval report 발행 | + +## 11. 현재 리포지토리 기준 품질 지표 + +현재 저장된 리포지토리 스냅샷 기준입니다. 운영 KPI라기보다는 회귀 방지용 품질 자료로 보는 것이 안전합니다. + +| 항목 | 현재 값 | +| --- | --- | +| 테스트 파일 | `src/**/*.test.ts` 기준 16개 | +| 주요 테스트 영역 | LLM routing, alert, eval worker, judges, shadow bus, parser, classifier, state derive, prompt builder, validator, tool loop, orchestrator, built-in tools | +| Golden eval case | 3개 | +| 저장된 baseline pass rate | 1.0 | +| 저장된 baseline rule average score | 1.0 | +| 저장된 baseline average latency | 3ms | +| 저장된 baseline LLM judge case | 0개 | + +포트폴리오에는 다음처럼 쓰는 것이 좋습니다. + +> 현재는 rule-based golden eval을 smoke benchmark로 운영하고 있으며, shadow eval과 LLM judge를 통해 운영 트래픽 기반 품질 평가로 확장 가능한 구조를 구현했습니다. + +## 12. 포트폴리오에서 강조할 수 있는 기술적 성과 + +### 12.1 LLM을 제품 로직 안에 안전하게 넣은 구조 + +LLM에게 모든 판단을 맡기지 않고, `Intent -> DerivedState -> Prompt -> Orchestrator -> Validator -> Fallback` 파이프라인으로 감쌌습니다. +덕분에 커뮤니티 규칙, 유저 진행 상태, 응답 범위 제한을 코드 레벨에서 제어할 수 있습니다. + +### 12.2 비용과 latency를 고려한 hybrid classifier + +자주 나오는 짧은 메시지는 regex로 처리하고, 애매한 메시지만 LLM classifier로 넘깁니다. +이 방식은 LLM 호출 비용을 줄이고 p50 latency를 낮추는 데 유리합니다. + +### 12.3 Provider 장애를 고려한 LLM client + +Provider priority, health check, circuit breaker, fallback model을 한 레이어에 모았습니다. +외부 LLM endpoint가 불안정해도 서비스는 다른 endpoint 또는 template fallback으로 계속 응답할 수 있습니다. + +### 12.4 Tool calling으로 사실 확인 분리 + +디스코드 닉네임 확인처럼 실제 조회가 필요한 기능은 LLM 추측이 아니라 tool execution으로 처리합니다. +또한 OpenAI tool call과 Hermes tool call을 모두 파싱해 provider 교체 가능성을 높였습니다. + +### 12.5 운영 평가 루프 + +실제 응답을 shadow eval stream에 샘플링하고, background worker가 rule judge/LLM judge를 비동기로 수행합니다. +평가 결과가 나빠지거나 latency가 올라가면 alert subject로 알릴 수 있어, 단순 구현을 넘어 운영 가능한 LLM 시스템 형태를 갖췄습니다. + +## 13. Confluence 페이지 구성 추천 + +Confluence에는 아래 순서로 넣으면 읽기 좋습니다. + +1. 프로젝트 한 줄 소개 +2. 문제 정의 +3. 전체 아키텍처 다이어그램 +4. 요청 처리 시퀀스 다이어그램 +5. 핵심 구현 포인트 +6. 운영 안정성 및 평가 구조 +7. 데이터 계약과 저장소 설계 +8. 성과/품질 지표 +9. 배운 점과 개선 계획 + +## 14. 면접/리뷰에서 말하기 좋은 포인트 + +### Q. 왜 LLM만 쓰지 않고 상태 기반 파이프라인을 만들었나요? + +온보딩 안내는 "친절한 문장"보다 "현재 단계에 맞는 정확한 안내"가 더 중요했습니다. +LLM은 표현 생성에 강하지만, 디스코드 안내 가능 여부나 닉네임 형식 같은 규칙 판단은 코드가 더 안정적입니다. 그래서 상태 판단과 validator는 코드가 담당하고, LLM은 최종 문장 생성과 애매한 의도 해석에 집중하게 했습니다. + +### Q. 왜 tier를 나눴나요? + +모든 요청에 planner, tool, critic을 붙이면 품질은 올라갈 수 있지만 latency와 비용이 커집니다. +인사나 단순 진행상황 질문은 direct response로 빠르게 처리하고, 닉네임 오류나 디스코드 확인처럼 정확도가 중요한 경우만 tier 1 이상으로 승격했습니다. + +### Q. 환각은 어떻게 줄였나요? + +세 가지 방식으로 줄였습니다. + +1. prompt에 현재 세션 상태와 남은 절차를 명시합니다. +2. 실제 조회가 필요한 것은 tool calling으로 분리합니다. +3. 최종 응답을 validator로 검사하고, reject되면 재시도 또는 template fallback을 사용합니다. + +### Q. 운영 중 품질 저하는 어떻게 감지하나요? + +실제 응답을 Redis Stream에 shadow eval job으로 기록하고, worker가 rule judge와 LLM judge를 비동기로 실행합니다. +최근 5분 기준 p95 latency, LLM judge 점수 하락, judge error rate, LLM circuit open 상태를 감지해 NATS alert를 발행합니다. + +### Q. 이 프로젝트에서 가장 어려웠던 점은 무엇인가요? + +LLM 응답을 자연스럽게 만들면서도, 커뮤니티 규칙과 진행 상태를 벗어나지 않게 제어하는 점이 가장 어려웠습니다. +이를 해결하기 위해 prompt engineering만 하는 대신, 상태 모델, 도구, validator, fallback, eval을 하나의 파이프라인으로 묶었습니다. + +## 15. 개선 계획 + +현재 구조에서 다음을 추가하면 포트폴리오 완성도가 더 올라갑니다. + +| 개선 항목 | 기대 효과 | +| --- | --- | +| Golden set 30-50개로 확장 | 회귀 테스트 신뢰도 향상 | +| 운영 dashboard 추가 | latency, fallback rate, judge score 추이를 시각화 | +| OTLP exporter 연동 | Jaeger/Grafana Tempo 등으로 trace 분석 | +| prompt/version metadata 저장 | 응답 품질 저하 원인 추적 | +| eval case 자동 익명화 강화 | 실제 운영 데이터 기반 평가셋 구축 | +| 실패 응답 샘플 리뷰 페이지 | 운영자가 직접 prompt/tool 개선 포인트 확인 | + +## 16. 포트폴리오용 짧은 소개 문장 + +> TypeScript와 NATS 기반으로 게임 커뮤니티 신규 멤버 온보딩을 자동화하는 LLM Agent를 설계/구현했습니다. Redis/Garnet 세션 상태, MongoDB 채팅 맥락, regex-first intent classifier, tier 기반 LLM orchestration, Zod schema tool calling, validator/fallback, shadow eval worker를 결합해 단순 챗봇이 아니라 운영 가능한 온보딩 자동화 시스템으로 구성했습니다. + +## 17. 이력서용 bullet 예시 + +- NATS request/reply 기반 LLM Agent를 구현해 카카오톡 커뮤니티 신규 멤버 온보딩 안내를 자동화했습니다. +- Regex-first intent classifier와 상태 기반 `DerivedState` 모델을 설계해 응답 여부, LLM 사용 여부, tool calling 필요 여부를 결정했습니다. +- Tier 0/1/2 orchestration을 도입해 단순 질문은 빠르게, 검증이 필요한 질문은 planner/tool/critic 경로로 처리하도록 구성했습니다. +- Zod schema 기반 tool registry와 OpenAI/Hermes tool call parser를 구현해 provider에 덜 종속적인 tool execution 구조를 만들었습니다. +- LLM 응답 validator, URL whitelist, state consistency check, template fallback으로 커뮤니티 규칙 위반 가능성을 낮췄습니다. +- Redis Stream 기반 shadow eval worker와 alert pipeline을 구현해 운영 트래픽 기반 품질 평가 및 장애 감지 구조를 마련했습니다. diff --git a/docs/service-evaluation-operations.md b/docs/service-evaluation-operations.md new file mode 100644 index 0000000..16ea2f8 --- /dev/null +++ b/docs/service-evaluation-operations.md @@ -0,0 +1,510 @@ +# lolympus-agent Service and Evaluation Operations + +이 문서는 `lolympus-agent`의 운영 흐름과 evaluation 파이프라인을 한 곳에서 보기 위한 runbook이다. 실시간 응답 경로, background worker, shadow evaluation, human review, calibration, closed-loop prompt experiment까지 포함한다. + +## 1. 프로세스 구성 + +운영 프로세스는 크게 두 개다. + +| 프로세스 | 엔트리포인트 | 역할 | +|---|---|---| +| agent | `src/main.ts` | NATS request/reply를 받아 실시간 온보딩 응답 생성 | +| worker | `src/workers/background.ts` | Redis Stream background job 처리, shadow eval, archive, drift, review routing | + +PM2 기준으로는 `ecosystem.config.cjs`에서 agent와 worker를 함께 띄운다. + +```bash +npm run pm2:dev +npm run pm2:test +npm run pm2:prod +``` + +개발 환경 단독 실행은 기존 스크립트를 사용한다. + +```bash +npm run start:dev +``` + +## 2. Bootstrap 흐름 + +`src/main.ts`의 기동 순서는 다음과 같다. + +1. `loadEnvironmentFromNodeEnv()`로 `env-vars/{NODE_ENV}.env` 로드 +2. logger/tracing 초기화 +3. NATS 연결 +4. Garnet/Redis 연결 +5. LLM endpoint와 provider key 구성 +6. MongoDB 연결, 가능하면 room context 활성화 +7. `Agent` 생성 +8. `OnboardingSkill` 등록 +9. `lolympus-agent.request` 구독 시작 + +중요한 점은 NATS가 가장 먼저 외부 TLS 연결을 시도한다는 것이다. 따라서 bootstrap fatal error가 `certificate has expired`이고 `✓ NATS connected` 로그가 없었다면 1순위 원인은 `NATS_ENDPOINT` 인증서 만료다. + +## 3. 실시간 응답 흐름 + +본체는 NATS request/reply로 agent에 요청한다. + +```text +lolympus-agent.request + -> Agent.handleRequest + -> OnboardingSkill.handle + -> AgentResponse +``` + +`OnboardingSkill.handle()` 내부 흐름: + +1. Garnet에서 온보딩 세션 로드 +2. user message와 trigger 기반 intent classify +3. session, trigger, intent로 `DerivedState` 생성 +4. `shouldRespond=false`면 empty response 반환 +5. `responseMode=template`이면 template fallback 응답 +6. LLM 경로면 prompt build 후 orchestrator 실행 +7. tier에 따라 planner/responder/critic/tool-loop 사용 +8. validator로 응답 검증 +9. 최종 응답 반환 +10. blackboard 업데이트 +11. shadow eval event를 background stream에 비동기 발행 + +실시간 사용자 응답은 9번에서 끝난다. Evaluation은 11번 이후 worker가 비동기로 처리한다. + +## 4. Prompt와 Tool 경로 + +Prompt는 `src/skills/onboarding/prompts/builder.ts`가 만든다. + +기본 layer: + +- persona +- policy +- state +- examples +- context + +`PROMPT_VARIANT_ID`가 설정되면 `src/eval/proposals/{variantId}.json` manifest를 읽어 prompt layer diff를 적용한다. 운영 반영은 env로 variant를 선택하는 방식이며, proposal 생성과 tournament는 별도 CLI에서 수행한다. + +Tool loop는 `responseMode=llm-with-tools`일 때 작동한다. Built-in tool은 `src/tools/builtins/*`에 있다. + +## 5. Background Worker 흐름 + +Worker는 `lolympus-agent:background` Redis Stream을 consumer group `bg-worker`로 읽는다. + +| Job type | 처리 | +|---|---| +| `shadow-eval` | 실시간 응답을 rule/LLM/process 기준으로 평가 | +| `summarize` | 오래된 대화 일부를 요약해 summary slot에 저장 | + +Worker는 pending job을 복구하고, delivery count가 한도를 넘으면 DLQ로 이동한다. + +| Stream | 용도 | +|---|---| +| `lolympus-agent:background` | background job queue | +| `lolympus-agent:background:dlq` | 반복 실패 job 저장 | + +## 6. Shadow Evaluation 흐름 + +실시간 응답이 만들어지면 `ShadowEvalBus`가 background stream에 `shadow-eval` job을 발행한다. + +Worker는 이 job을 받아 다음을 수행한다. + +1. validator 재실행 +2. LLM judge 실행 +3. process trace가 있으면 process judge 실행 +4. latency, tier, intent, phase, judge 결과 저장 + +결과는 Redis Stream에 저장된다. + +```text +lolympus-agent:eval:results +``` + +이 Stream이 이후 evaluation 운영의 원천 데이터다. + +## 7. 주기 Evaluation 작업 + +Worker는 일정 간격으로 eval result를 가공한다. + +| 작업 | 기본 주기 | 결과 | +|---|---:|---| +| aggregation | `EVAL_AGG_TICK_MS` 5분 | slice별 Redis Hash metric | +| archive | `EVAL_ARCHIVE_TICK_MS` 1시간 | MongoDB `eval_archive` | +| drift detection | `EVAL_DRIFT_TICK_MS` 24시간 | `eval-drift` alert | +| promotion | `EVAL_PROMOTE_TICK_MS` 24시간 | golden candidate jsonl | +| review routing | `EVAL_REVIEW_TICK_MS` 5분 | human review queue | +| review label archive | archive tick과 함께 | MongoDB `eval_review_labels` | + +Slice 정의: + +```text +tier × intent × phase +``` + +예: + +```text +0:photo-question:photo-pending +1:discord-question:step1-completed +``` + +## 8. Redis Key Schema + +주요 Redis/Garnet key: + +| Key | Type | 용도 | +|---|---|---| +| `lolympus-agent:background` | Stream | worker job queue | +| `lolympus-agent:background:dlq` | Stream | worker dead-letter queue | +| `lolympus-agent:eval:results` | Stream | shadow eval 결과 | +| `lolympus-agent:eval:baseline` | String | 최근 안정 baseline report | +| `lolympus-agent:eval:agg:{tier}:{intent}:{phase}` | Hash | slice별 count/score/latency/process metric | +| `lolympus-agent:eval:agg:cursor` | String | aggregation cursor | +| `lolympus-agent:eval:archive:cursor` | String | eval archive cursor | +| `lolympus-agent:eval:review:queue` | Stream | human review 대기 큐 | +| `lolympus-agent:eval:review:labeled` | Stream | human review label | +| `lolympus-agent:eval:review:cursor` | String | review routing cursor | +| `lolympus-agent:eval:review:dedupe:{evalStreamId}` | String | 동일 eval result 중복 라우팅 방지 | +| `lolympus-agent:eval:review:label-archive:cursor` | String | review label archive cursor | +| `lolympus-agent:alert:dedupe:{kind}` | String | alert rate limit | + +## 9. MongoDB Collections + +| Collection | 생성/사용 모듈 | 용도 | +|---|---|---| +| `eval_archive` | `src/eval/archive.ts` | shadow eval 결과 장기 보관 | +| `eval_review_labels` | `src/eval/review/label-archive.ts` | 사람 라벨 장기 보관 | +| `eval_tournaments` | `src/eval/loop/tournament.ts` | prompt variant tournament 결과 | + +`eval_archive`는 `evaluatedAt` 기준 TTL index를 가진다. 기본 보관 기간은 `EVAL_ARCHIVE_TTL_DAYS=30`이다. + +## 10. Human Review 흐름 + +Review routing은 eval result 중 사람이 봐야 할 케이스를 큐에 넣는다. + +조건: + +- LLM judge overall이 낮음 +- validator와 LLM judge가 불일치 +- validator soft-warn +- rare slice + +큐: + +```text +lolympus-agent:eval:review:queue +``` + +라벨링: + +```bash +npm run eval:review -- --limit=20 +``` + +특정 reason만 보기: + +```bash +npm run eval:review -- --reason=low-score --limit=10 +``` + +stale pending item을 다시 큐에 넣기: + +```bash +npm run eval:review -- --reset-pending +``` + +세션 라벨을 JSONL로 저장: + +```bash +npm run eval:review -- --limit=20 --export-labeled=eval-reports/review-labels.jsonl +``` + +라벨 결과는 `lolympus-agent:eval:review:labeled` Stream에 쌓이고 archive tick에서 `eval_review_labels`로 영속화된다. + +## 11. Calibration 흐름 + +Calibration은 사람 점수와 LLM judge 점수의 상관을 계산한다. + +```bash +npm run eval:calibrate +``` + +CI나 수동 gate로 쓰려면: + +```bash +npm run eval:calibrate -- --check +``` + +`pairs < 10`이면 표본이 부족하므로 경고만 출력하고 실패 처리하지 않는다. 충분한 라벨이 쌓이면 `EVAL_CALIBRATION_MIN_CORR` 기준으로 LLM judge 신뢰도를 감시한다. + +## 12. Report와 Promotion + +Slice별 약한 영역 확인: + +```bash +npm run eval:report -- --worst=20 +``` + +최근 7일 대신 다른 window: + +```bash +npm run eval:report -- --worst=20 --since=24h +npm run eval:report -- --top=20 --since=30d +``` + +Golden candidate 확인: + +```bash +npm run eval:promote -- --dry-run +``` + +Candidate 파일에 추가: + +```bash +npm run eval:promote +``` + +기본 출력: + +```text +src/eval/golden/onboarding.candidate.jsonl +``` + +Promotion 우선순위: + +1. human-labeled fail +2. low-score +3. rule/LLM disagree +4. rare-slice + +candidate는 바로 golden으로 승격되는 것이 아니라 사람이 직접 검토해 `onboarding.jsonl`로 옮긴다. + +## 13. Closed Loop 흐름 + +Phase 9의 closed loop는 prompt/tool 개선안을 만들고 tournament로 비교하는 흐름이다. + +### 13.1 Proposal 생성 + +약한 slice를 지정한다. + +```bash +npm run eval:propose -- --slice=tier=1,intent=photo-question,phase=photo-pending +``` + +archive case 수가 부족해도 강제로 실행하려면: + +```bash +npm run eval:propose -- --slice=tier=1,intent=photo-question,phase=photo-pending --force +``` + +결과는 git-tracked proposal manifest로 저장된다. + +```text +src/eval/proposals/{variantId}.json +``` + +### 13.2 Tournament 실행 + +Control과 variant를 비교한다. + +```bash +npm run eval:tournament -- --variants=control, +``` + +특정 archive case만 지정: + +```bash +npm run eval:tournament -- --variants=control, --case-ids=1-0,2-0 +``` + +결과 조회: + +```bash +npm run eval:tournament-results +npm run eval:tournament-results -- --variant= +``` + +Tournament는 A/B 위치 편향을 줄이기 위해 variant 순서를 바꾼 match도 실행하고 ELO를 갱신한다. 결과는 MongoDB `eval_tournaments`에 저장된다. + +### 13.3 운영 반영 + +운영 prompt에 variant를 반영하려면 env를 설정한다. + +```env +PROMPT_VARIANT_ID= +``` + +반영 전에는 반드시 다음을 확인한다. + +```bash +npm run typecheck +npm run lint +npm run build +npm test +npm run eval -- --skip-llm-judge --no-publish-report +``` + +Variant 적용은 자동 머지가 아니다. 사람이 tournament 결과와 sample match를 보고 결정한다. + +## 14. Eval CLI 요약 + +| 명령 | 용도 | +|---|---| +| `npm run eval` | golden set 실행 | +| `npm run eval:capture` | MongoDB events에서 golden scaffold 추출 | +| `npm run eval:baseline` | 안정 baseline 저장 | +| `npm run eval:report` | archive slice report 출력 | +| `npm run eval:promote` | golden candidate 추출 | +| `npm run eval:review` | human review CLI | +| `npm run eval:calibrate` | 사람/LLM judge calibration | +| `npm run eval:propose` | prompt/tool improvement proposal 생성 | +| `npm run eval:tournament` | prompt variant A/B tournament 실행 | +| `npm run eval:tournament-results` | tournament 결과 조회 | + +## 15. 주요 Env + +| Env | 기본값 | 용도 | +|---|---:|---| +| `EVAL_STREAM_MAXLEN` | 10000 | eval results Stream maxlen | +| `EVAL_AGG_TICK_MS` | 300000 | aggregation tick | +| `EVAL_AGG_TTL_SEC` | 604800 | slice metric TTL | +| `EVAL_ARCHIVE_TICK_MS` | 3600000 | archive tick | +| `EVAL_ARCHIVE_TTL_DAYS` | 30 | archive TTL days | +| `EVAL_DRIFT_TICK_MS` | 86400000 | drift tick | +| `EVAL_PROMOTE_TICK_MS` | 86400000 | promotion tick | +| `EVAL_REVIEW_TICK_MS` | 300000 | review routing tick | +| `EVAL_REVIEW_QUEUE_MAXLEN` | 500 | review queue maxlen | +| `EVAL_REVIEW_LABEL_MAXLEN` | 5000 | review label Stream maxlen | +| `EVAL_REVIEW_DEDUPE_TTL_SEC` | 604800 | review dedupe TTL | +| `EVAL_CALIBRATION_MIN_CORR` | 0.5 | calibration gate | +| `REVIEWER_ID` | git user.email | human review 식별자 | +| `LLM_PROPOSE_MODEL` | `LLM_JUDGE_MODEL` | proposal model | +| `LLM_PROPOSE_MAX_TOKENS` | 1024 | proposal max tokens | +| `EVAL_TOURNAMENT_MAX_MATCHES` | 50 | tournament match cap | +| `EVAL_TOURNAMENT_MIN_CASES` | 10 | proposal 최소 archive cases | +| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 | + +## 16. Alert와 Monitoring + +Worker는 다음 종류의 alert를 보낼 수 있다. + +| Alert kind | 의미 | +|---|---| +| `latency` | 최근 shadow eval latency p95 초과 | +| `llm-error-rate` | LLM judge 실패율 상승 | +| `eval-score-drop` | baseline 대비 eval score 하락 | +| `eval-drift` | 24h vs 7d slice score drift | +| `endpoint-down` | LLM circuit breaker open | +| `archive-stalled` | MongoDB archive 연속 실패 | + +Alert는 `lolympus-agent.alert` NATS subject로 발행되고, dedupe key로 rate limit된다. + +## 17. 장애 확인 Runbook + +### 17.1 Bootstrap에서 `certificate has expired` + +증상: + +```text +[Bootstrap] Fatal startup error +message: certificate has expired +code: CERT_HAS_EXPIRED +``` + +`src/main.ts` 기준 NATS 연결이 첫 외부 TLS 연결이다. `✓ NATS connected` 로그가 없으면 `NATS_ENDPOINT` 인증서가 1순위 원인이다. + +확인: + +```bash +node --trace-warnings --import tsx src/main.ts +``` + +인증서 날짜 확인: + +```bash +openssl s_client -connect : -servername /dev/null \ + | openssl x509 -noout -dates -subject +``` + +`✓ NATS connected` 이후 같은 오류가 나면 다음 후보는 Garnet 또는 MongoDB다. + +### 17.2 LLM endpoint warning + +증상: + +```text +[LLM] Endpoint unavailable via circuit breaker +[LLM] Request error +[LLM] All endpoints failed +``` + +확인 순서: + +1. `CLI_PROXY_API_URLS` +2. `OPENROUTER_API_KEY`, `OPENROUTER_API_URL` +3. `AGENT_LLM_API_URLS` +4. `LLAMA_SERVER_API_URL` +5. provider별 API key + +`npm run eval -- --skip-llm-judge`는 LLM judge를 생략하지만 실제 응답 생성 LLM은 여전히 호출될 수 있다. endpoint가 모두 실패하면 template/fallback 경로로 통과할 수 있다. + +### 17.3 MongoDB 미설정 + +Worker가 다음과 비슷한 로그를 남긴다. + +```text +MongoDB connection string missing; eval archive/drift/promote disabled +``` + +영향: + +- 실시간 응답은 가능 +- shadow eval result Stream은 쌓임 +- archive, drift, promote, calibration, tournament는 제한됨 + +필요 env: + +```env +MONGODB_CONNECTION_STRING= +MONGODB_PROJECT_ID= +``` + +### 17.4 Review queue가 밀림 + +확인: + +```bash +npm run eval:review -- --limit=5 +``` + +stale pending item 정리: + +```bash +npm run eval:review -- --reset-pending +``` + +같은 케이스가 반복 라우팅되면 dedupe TTL과 cursor를 확인한다. + +```text +lolympus-agent:eval:review:cursor +lolympus-agent:eval:review:dedupe:{evalStreamId} +``` + +### 17.5 Archive stalled + +`archive-stalled` alert가 뜨면 MongoDB 연결과 index 생성 권한을 확인한다. + +관련 cursor: + +```text +lolympus-agent:eval:archive:cursor +lolympus-agent:eval:review:label-archive:cursor +``` + +cursor가 진행되지 않으면 worker가 다음 tick에서 재시도한다. + +## 18. 운영 원칙 + +- 실시간 응답 경로는 가능한 짧게 유지한다. +- Evaluation은 Redis Stream을 통해 비동기로 처리한다. +- Golden set 승격은 사람이 직접 결정한다. +- Prompt variant 운영 반영은 tournament 결과와 eval smoke를 본 뒤 결정한다. +- 자동 rollback이나 자동 merge는 하지 않는다. +- 모든 timestamp는 UTC ISO-8601을 사용한다. diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs index 57b8a36..3d3ca0f 100644 --- a/ecosystem.config.cjs +++ b/ecosystem.config.cjs @@ -1,17 +1,17 @@ module.exports = { apps: [ { - name: "agent", - script: "dist/main.js", + name: 'lolympus-agent', + script: 'dist/main.js', instances: 1, - max_memory_restart: "512M", + max_memory_restart: '512M', kill_timeout: 12000, }, { - name: "agent-worker", - script: "dist/workers/background.js", + name: 'lolympus-agent-worker', + script: 'dist/workers/background.js', instances: 1, - max_memory_restart: "512M", + max_memory_restart: '512M', kill_timeout: 12000, }, ], diff --git a/launch-server.ps1 b/launch-server.ps1 index 1b5f27e..472bc05 100644 --- a/launch-server.ps1 +++ b/launch-server.ps1 @@ -12,12 +12,8 @@ if (-not $env:NODE_ENV) { if ($isMacOS -or $isLinux) { $env:ENV_FILE = "$PSScriptRoot/env-vars/$env:NODE_ENV.env" - $env:AWS_CREDENTIALS_FILE = "$env:HOME/.aws/credentials" - $env:DAHYEON_GLOBAL_ENVIRONMENT_FILE = "$env:HOME/source/global-env-vars.txt" } elseif ($isWindows) { $env:ENV_FILE = "$PSScriptRoot\env-vars\$env:NODE_ENV.env" - $env:AWS_CREDENTIALS_FILE = "$env:USERPROFILE\.aws\credentials" - $env:DAHYEON_GLOBAL_ENVIRONMENT_FILE = "$env:USERPROFILE/source/global-env-vars.txt" } # 환경 변수 파일 확인 @@ -34,38 +30,16 @@ Get-Content $env:ENV_FILE | ForEach-Object { } } -# AWS 자격 증명 파일 확인 -if (-Not (Test-Path $env:AWS_CREDENTIALS_FILE)) { - Write-Error "The AWS credentials file '$env:AWS_CREDENTIALS_FILE' does not exist." - exit 1 +Get-ChildItem Env: | Where-Object { + $_.Name -eq "NODE_ENV" -or + $_.Name -like "IMDS_*" -or + $_.Name -like "METADATA_*" -or + $_.Name -eq "MACHINE_ENV_BASE_URL" } -# 공통 환경변수 파일 확인 -if (-Not (Test-Path $env:DAHYEON_GLOBAL_ENVIRONMENT_FILE)) { - Write-Error "The global environment variables file '$env:DAHYEON_GLOBAL_ENVIRONMENT_FILE' does not exist." - exit 1 -} - - -# AWS 자격 증명 파일에서 환경 변수 설정 -Get-Content $env:AWS_CREDENTIALS_FILE | Where-Object { $_ -match '^\s*([^#=\[]+?)\s*=\s*(.*)$' } | ForEach-Object { - $key, $value = $matches[1].Trim(), $matches[2].Trim() - $envKey = ($key -replace ' ', '_').ToUpper() - [System.Environment]::SetEnvironmentVariable($envKey, $value) -} - -# 공통 환경변수 파일에서 환경 변수 설정 -Get-Content $env:DAHYEON_GLOBAL_ENVIRONMENT_FILE | Where-Object { $_ -match '^\s*([^#=\[]+?)\s*=\s*(.*)$' } | ForEach-Object { - $key, $value = $matches[1].Trim(), $matches[2].Trim() - $envKey = ($key -replace ' ', '_').ToUpper() - [System.Environment]::SetEnvironmentVariable($envKey, $value) -} - -Get-ChildItem Env: | Where-Object { $_.Name -like "*AWS*" -or $_.Name -eq "NODE_ENV" } - if ($UsePm2) { - pm2 start ecosystem.config.cjs --only agent,agent-worker --update-env + pm2 start ecosystem.config.cjs --only lolympus-agent,lolympus-agent-worker --update-env } else { node ./dist/main.js } diff --git a/package.json b/package.json index 32d3315..ec85052 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,13 @@ "eval": "node --import tsx src/eval/cli.ts run --set=golden", "eval:capture": "node --import tsx src/eval/cli.ts capture", "eval:baseline": "node --import tsx src/eval/cli.ts baseline", + "eval:report": "node --import tsx src/eval/cli.ts report", + "eval:promote": "node --import tsx src/eval/cli.ts promote", + "eval:review": "node --import tsx src/eval/cli.ts review", + "eval:calibrate": "node --import tsx src/eval/cli.ts calibrate", + "eval:propose": "node --import tsx src/eval/cli.ts propose", + "eval:tournament": "node --import tsx src/eval/cli.ts tournament", + "eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results", "start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"", "start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"", "start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"", diff --git a/papers/2502.09054v1.pdf b/papers/2502.09054v1.pdf new file mode 100644 index 0000000..db317f3 Binary files /dev/null and b/papers/2502.09054v1.pdf differ diff --git a/papers/2602.12430v3.pdf b/papers/2602.12430v3.pdf new file mode 100644 index 0000000..c3897d6 Binary files /dev/null and b/papers/2602.12430v3.pdf differ diff --git a/papers/2605.06350v1.pdf b/papers/2605.06350v1.pdf new file mode 100644 index 0000000..8fe906e Binary files /dev/null and b/papers/2605.06350v1.pdf differ diff --git a/src/common/env-loader.ts b/src/common/env-loader.ts index 2d0e944..e3ddcde 100644 --- a/src/common/env-loader.ts +++ b/src/common/env-loader.ts @@ -34,6 +34,13 @@ function resolveEnvFile(mode: NodeEnvMode): string | null { return null; } +function loadDotenvFile(filePath: string, label: string): void { + const result = dotenv.config({ path: filePath, override: false }); + if (result.error) { + logger.warn(`[Env] Failed to load ${label} (${filePath}): ${result.error.message}`); + } +} + export function loadEnvironmentFromNodeEnv(): void { if (loaded) return; loaded = true; @@ -47,9 +54,5 @@ export function loadEnvironmentFromNodeEnv(): void { return; } - const result = dotenv.config({ path: envFile, override: false }); - if (result.error) { - logger.warn(`[Env] Failed to load ${envFile}: ${result.error.message}`); - return; - } + loadDotenvFile(envFile, `env-vars/${mode}.env`); } diff --git a/src/common/garnet.ts b/src/common/garnet.ts index 16263fc..fd9c010 100644 --- a/src/common/garnet.ts +++ b/src/common/garnet.ts @@ -188,6 +188,14 @@ export class GarnetClient { await this.getClient().hSet(key, fields); } + async expireKey(key: string, ttlSeconds: number): Promise { + if (ttlSeconds <= 0) { + return false; + } + + return Boolean(await this.getClient().expire(key, ttlSeconds)); + } + async appendStream( key: string, fields: Record, @@ -207,6 +215,31 @@ export class GarnetClient { return this.getClient().xAdd(key, "*", fields, trimOptions); } + async readStreamRange( + key: string, + start: string, + end: string, + options?: { count?: number | undefined }, + ): Promise { + const rangeOptions = + options?.count && options.count > 0 + ? { + COUNT: options.count, + } + : undefined; + const messages = await this.getClient().xRange( + key, + start, + end, + rangeOptions, + ); + + return messages.map((message) => ({ + id: String(message.id), + message: normalizeStreamFields(message.message), + })); + } + async createConsumerGroup( key: string, group: string, diff --git a/src/common/known-secret-keys.ts b/src/common/known-secret-keys.ts index 3c7b133..ed97491 100644 --- a/src/common/known-secret-keys.ts +++ b/src/common/known-secret-keys.ts @@ -1,4 +1,6 @@ export const KNOWN_SECRET_KEYS = [ + "INFISICAL_CLIENT_ID", + "INFISICAL_CLIENT_SECRET", "NATS_ENDPOINT", "NATS_PASS", "NATS_USER", diff --git a/src/common/nats-subjects.ts b/src/common/nats-subjects.ts index 635195d..8d6d237 100644 --- a/src/common/nats-subjects.ts +++ b/src/common/nats-subjects.ts @@ -7,7 +7,13 @@ export const DISCORD_MEMBER_LOOKUP_SUBJECT = "lolympus.discord.lookup-members"; export interface AlertPayload { - kind: "latency" | "eval-score-drop" | "llm-error-rate" | "endpoint-down"; + kind: + | "archive-stalled" + | "endpoint-down" + | "eval-drift" + | "eval-score-drop" + | "latency" + | "llm-error-rate"; metric: string; severity: "info" | "warn" | "critical"; suggestedAction?: string | undefined; diff --git a/src/common/secret-manager.ts b/src/common/secret-manager.ts index 0cbd6ce..dfa0fba 100644 --- a/src/common/secret-manager.ts +++ b/src/common/secret-manager.ts @@ -1,10 +1,19 @@ import { InfisicalSDK } from "@infisical/sdk"; import { KNOWN_SECRET_KEYS, type KnownSecretKey } from "./known-secret-keys.js"; + export type SecretMap = Partial>; export type RawSecretMap = Record; const INFISICAL_SITE_URL = "https://secret-manager.dahyeon.us/"; const KNOWN_SECRET_KEY_SET = new Set(KNOWN_SECRET_KEYS); +const IMDS_TOKEN_TTL_SECONDS = 60; +const DEFAULT_IMDS_TIMEOUT_MS = 1_500; +const DEFAULT_LOCAL_IMDS_BASE_URL = "http://127.0.0.1:8675"; +const DEFAULT_LINK_LOCAL_IMDS_BASE_URL = "http://169.254.169.254"; +const IMDS_TOKEN_TTL_HEADER = "X-aws-ec2-metadata-token-ttl-seconds"; +const IMDS_TOKEN_HEADER = "X-aws-ec2-metadata-token"; +const INFISICAL_CLIENT_ID_KEY = "INFISICAL_CLIENT_ID"; +const INFISICAL_CLIENT_SECRET_KEY = "INFISICAL_CLIENT_SECRET"; const infisicalClient = new InfisicalSDK({ siteUrl: INFISICAL_SITE_URL }); @@ -15,18 +24,10 @@ function isKnownSecretKey(value: string): value is KnownSecretKey { return KNOWN_SECRET_KEY_SET.has(value); } -function getInfisicalCredentials(): { +interface InfisicalCredentials { clientSecret: string; clientId: string; projectId: string; -} | null { - const clientSecret = process.env["INFISICAL_CLIENT_SECRET"] ?? ""; - const clientId = process.env["INFISICAL_CLIENT_ID"] ?? ""; - const projectId = process.env["INFISICAL_PROJECT_ID"] ?? ""; - if (!clientSecret || !clientId || !projectId) { - return null; - } - return { clientSecret, clientId, projectId }; } function getInfisicalEnvironment(): "dev" | "staging" | "prod" { @@ -35,6 +36,173 @@ function getInfisicalEnvironment(): "dev" | "staging" | "prod" { return "dev"; } +function isDisabled(value: string | undefined): boolean { + const normalized = value?.trim().toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes"; +} + +function getImdsBaseUrls(): string[] { + if ( + isDisabled(process.env["IMDS_DISABLED"]) || + isDisabled(process.env["METADATA_DISABLED"]) + ) { + return []; + } + + const configured = + process.env["METADATA_BASE_URL"]?.trim() || + process.env["IMDS_BASE_URL"]?.trim() || + process.env["MACHINE_ENV_BASE_URL"]?.trim(); + if (configured) { + return [configured.replace(/\/+$/u, "")]; + } + + const candidates = + process.env["NODE_ENV"] === "prod" + ? [DEFAULT_LINK_LOCAL_IMDS_BASE_URL, DEFAULT_LOCAL_IMDS_BASE_URL] + : [DEFAULT_LOCAL_IMDS_BASE_URL, DEFAULT_LINK_LOCAL_IMDS_BASE_URL]; + return [...new Set(candidates)]; +} + +function getImdsTimeoutMs(): number { + const raw = + process.env["METADATA_TIMEOUT_MS"]?.trim() || + process.env["IMDS_TIMEOUT_MS"]?.trim(); + if (!raw) { + return DEFAULT_IMDS_TIMEOUT_MS; + } + + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : DEFAULT_IMDS_TIMEOUT_MS; +} + +async function fetchWithTimeout( + url: string, + init: RequestInit, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), getImdsTimeoutMs()); + + try { + return await fetch(url, { + ...init, + signal: controller.signal, + }); + } finally { + clearTimeout(timeout); + } +} + +async function fetchImdsToken(baseUrl: string): Promise { + const response = await fetchWithTimeout(`${baseUrl}/latest/api/token`, { + method: "PUT", + headers: { + [IMDS_TOKEN_TTL_HEADER]: String(IMDS_TOKEN_TTL_SECONDS), + }, + }); + + if (!response.ok) { + return null; + } + + const token = await response.text(); + return token.trim() || null; +} + +async function fetchImdsText( + baseUrl: string, + token: string, + path: string, +): Promise { + const response = await fetchWithTimeout(`${baseUrl}${path}`, { + method: "GET", + headers: { + [IMDS_TOKEN_HEADER]: token, + }, + }); + + if (response.status === 404) { + return null; + } + + if (!response.ok) { + return null; + } + + return response.text(); +} + +async function fetchInfisicalCredentialsFromImds(): Promise< + Partial> +> { + for (const baseUrl of getImdsBaseUrls()) { + try { + const token = await fetchImdsToken(baseUrl); + if (!token) { + continue; + } + + const [clientId, clientSecret] = await Promise.all([ + fetchImdsText( + baseUrl, + token, + `/latest/meta-data/secrets/${encodeURIComponent(INFISICAL_CLIENT_ID_KEY)}`, + ), + fetchImdsText( + baseUrl, + token, + `/latest/meta-data/secrets/${encodeURIComponent(INFISICAL_CLIENT_SECRET_KEY)}`, + ), + ]); + + const trimmedClientId = clientId?.trim(); + const trimmedClientSecret = clientSecret?.trim(); + if (trimmedClientId || trimmedClientSecret) { + const credentials: Partial< + Pick + > = {}; + if (trimmedClientId) { + credentials.clientId = trimmedClientId; + } + if (trimmedClientSecret) { + credentials.clientSecret = trimmedClientSecret; + } + return credentials; + } + } catch { + continue; + } + } + + return {}; +} + +async function getInfisicalCredentials(): Promise { + const projectId = process.env["INFISICAL_PROJECT_ID"]?.trim() ?? ""; + if (!projectId) { + return null; + } + + const envClientId = process.env[INFISICAL_CLIENT_ID_KEY]?.trim(); + const envClientSecret = process.env[INFISICAL_CLIENT_SECRET_KEY]?.trim(); + const imdsCredentials: Partial< + Pick + > = + envClientId && envClientSecret + ? {} + : await fetchInfisicalCredentialsFromImds(); + const clientId = envClientId || imdsCredentials.clientId || ""; + const clientSecret = envClientSecret || imdsCredentials.clientSecret || ""; + + if (!clientId || !clientSecret) { + return null; + } + + return { clientId, clientSecret, projectId }; +} + export function getSecret(key: KnownSecretKey): string | undefined { return secrets?.[key] ?? process.env[key]; } @@ -43,11 +211,11 @@ export function getRawSecret(key: string): string | undefined { return rawSecrets?.[key] ?? process.env[key]; } -const fetchSecrets = async (): Promise<{ +const fetchSecretsFromInfisical = async (): Promise<{ known: SecretMap; raw: RawSecretMap; }> => { - const credentials = getInfisicalCredentials(); + const credentials = await getInfisicalCredentials(); if (!credentials) { return { known: {}, raw: {} }; } @@ -79,6 +247,13 @@ const fetchSecrets = async (): Promise<{ } }; +const fetchSecrets = async (): Promise<{ + known: SecretMap; + raw: RawSecretMap; +}> => { + return fetchSecretsFromInfisical(); +}; + const loaded = await fetchSecrets(); secrets = loaded.known; rawSecrets = loaded.raw; diff --git a/src/config.ts b/src/config.ts index bca16ad..252a2bd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -29,6 +29,14 @@ function getRatioNumber( : fallback; } +function getBoolean(envKey: string, fallback: boolean): boolean { + const raw = process.env[envKey]?.trim().toLowerCase(); + if (!raw) return fallback; + if (raw === "1" || raw === "true" || raw === "yes") return true; + if (raw === "0" || raw === "false" || raw === "no") return false; + return fallback; +} + export function getNodeEnv(): "dev" | "test" | "prod" { const value = process.env["NODE_ENV"]?.trim(); return value === "test" || value === "prod" ? value : "dev"; @@ -118,6 +126,94 @@ export function getEvalStreamMaxLen(): number { return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000); } +export function getEvalAggTickMs(): number { + return getPositiveNumber("EVAL_AGG_TICK_MS", 300_000); +} + +export function getEvalAggTtlSec(): number { + return getPositiveNumber("EVAL_AGG_TTL_SEC", 604_800); +} + +export function getEvalAggLatencySampleCap(): number { + return getPositiveNumber("EVAL_AGG_LATENCY_SAMPLE_CAP", 200); +} + +export function getEvalArchiveTickMs(): number { + return getPositiveNumber("EVAL_ARCHIVE_TICK_MS", 3_600_000); +} + +export function getEvalArchiveTtlDays(): number { + return getPositiveNumber("EVAL_ARCHIVE_TTL_DAYS", 30); +} + +export function getEvalArchiveBatchSize(): number { + return getPositiveNumber("EVAL_ARCHIVE_BATCH_SIZE", 500); +} + +export function getEvalDriftTickMs(): number { + return getPositiveNumber("EVAL_DRIFT_TICK_MS", 86_400_000); +} + +export function getEvalPromoteTickMs(): number { + return getPositiveNumber("EVAL_PROMOTE_TICK_MS", 86_400_000); +} + +export function getEvalPromoteLowScore(): number { + return getPositiveNumber("EVAL_PROMOTE_LOW_SCORE", 2); +} + +export function getEvalPromoteRareThreshold(): number { + return getPositiveNumber("EVAL_PROMOTE_RARE_THRESHOLD", 5); +} + +export function getEvalPromotePerCategoryLimit(): number { + return getPositiveNumber("EVAL_PROMOTE_PER_CATEGORY_LIMIT", 10); +} + +export function getEvalReviewTickMs(): number { + return getPositiveNumber("EVAL_REVIEW_TICK_MS", 300_000); +} + +export function getEvalReviewQueueMaxLen(): number { + return getPositiveNumber("EVAL_REVIEW_QUEUE_MAXLEN", 500); +} + +export function getEvalReviewLabelMaxLen(): number { + return getPositiveNumber("EVAL_REVIEW_LABEL_MAXLEN", 5_000); +} + +export function getEvalReviewPerCategoryLimit(): number { + return getPositiveNumber("EVAL_REVIEW_PER_CATEGORY_LIMIT", 5); +} + +export function getEvalReviewDedupeTtlSec(): number { + return getPositiveNumber("EVAL_REVIEW_DEDUPE_TTL_SEC", 604_800); +} + +export function getEvalCalibrationMinCorr(): number { + return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5); +} + +export function getLlmProposeModel(): string { + return process.env["LLM_PROPOSE_MODEL"]?.trim() || getLlmJudgeModel(); +} + +export function getLlmProposeMaxTokens(): number { + return getPositiveNumber("LLM_PROPOSE_MAX_TOKENS", 1_024); +} + +export function getEvalTournamentMaxMatches(): number { + return getPositiveNumber("EVAL_TOURNAMENT_MAX_MATCHES", 50); +} + +export function getEvalTournamentMinCases(): number { + return getPositiveNumber("EVAL_TOURNAMENT_MIN_CASES", 10); +} + +export function getPromptVariantId(): string | undefined { + return process.env["PROMPT_VARIANT_ID"]?.trim() || undefined; +} + export function getMaxConversationMessages(): number { return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20); } @@ -154,6 +250,18 @@ export function getLlmJudgeMaxTokens(): number { return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128); } +export function getEvalProcessLlmJudgeEnabled(): boolean { + return getBoolean("EVAL_PROCESS_LLM_JUDGE_ENABLED", false); +} + +export function getEvalProcessLlmJudgeSamplingRate(): number { + return getRatioNumber("EVAL_PROCESS_LLM_JUDGE_SAMPLING_RATE", 0.1); +} + +export function getEvalProcessLlmJudgeMaxTokens(): number { + return getPositiveNumber("EVAL_PROCESS_LLM_JUDGE_MAX_TOKENS", 256); +} + export function getSafetyMarginMs(): number { return getPositiveNumber("SAFETY_MARGIN_MS", 500); } @@ -303,6 +411,10 @@ export function getAlertThresholdLlmErrorRate(): number { return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1); } +export function getAlertThresholdDriftPct(): number { + return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15); +} + export function getAlertRateLimitSec(): number { return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300); } diff --git a/src/eval/aggregator.test.ts b/src/eval/aggregator.test.ts new file mode 100644 index 0000000..5ae7ebd --- /dev/null +++ b/src/eval/aggregator.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import type { StreamEntry } from "@common/garnet"; +import { + aggregateRecentEvalResults, + EVAL_AGG_CURSOR_KEY, + EVAL_RESULTS_STREAM_KEY, + getSliceAggregateKey, +} from "./aggregator.js"; + +class MemoryGarnet { + readonly expires = new Map(); + readonly hashes = new Map>(); + readonly strings = new Map(); + readonly streams = new Map(); + + expireKey(key: string, ttlSeconds: number): Promise { + this.expires.set(key, ttlSeconds); + return Promise.resolve(true); + } + + getHash(key: string): Promise> { + return Promise.resolve({ ...(this.hashes.get(key) ?? {}) }); + } + + readStreamRange( + key: string, + start: string, + _end: string, + ): Promise { + const entries = this.streams.get(key) ?? []; + if (!start.startsWith("(")) { + return Promise.resolve(entries); + } + + const exclusiveStart = start.slice(1); + return Promise.resolve(entries.filter((entry) => entry.id > exclusiveStart)); + } + + scanKeys(pattern: string): Promise { + const prefix = pattern.replace(/\*$/u, ""); + return Promise.resolve( + [...this.hashes.keys()].filter((key) => key.startsWith(prefix)), + ); + } + + setHash(key: string, fields: Record): Promise { + this.hashes.set(key, { ...fields }); + return Promise.resolve(); + } + + setString(key: string, value: string): Promise { + this.strings.set(key, value); + return Promise.resolve(); + } +} + +function resultEntry( + id: string, + fields: Partial>, +): StreamEntry { + return { + id, + message: { + type: "shadow-eval-result", + tier: "0", + intent: "status-question", + latencyMs: "100", + llmOverall: "4", + validatorStatus: "pass", + derived: JSON.stringify({ + tier: 0, + intent: { kind: "status-question" }, + phase: { kind: "photo-pending" }, + }), + ...fields, + }, + }; +} + +describe("aggregateRecentEvalResults", () => { + it("rolls stream entries up into per-slice hashes", async () => { + const garnet = new MemoryGarnet(); + garnet.streams.set(EVAL_RESULTS_STREAM_KEY, [ + resultEntry("1-0", {}), + resultEntry("2-0", { + latencyMs: "300", + llmOverall: "", + llmJudge: JSON.stringify({ + skipped: false, + scores: { overall: 2 }, + }), + validatorStatus: "reject", + }), + resultEntry("3-0", { + tier: "1", + intent: "photo-question", + latencyMs: "", + llmOverall: "", + validatorStatus: "soft-warn", + derived: JSON.stringify({ + tier: 1, + intent: { kind: "photo-question" }, + phase: { kind: "fresh" }, + }), + }), + ]); + + const result = await aggregateRecentEvalResults(garnet, { + sinceId: "0-0", + nowMs: Date.parse("2026-05-01T00:00:00.000Z"), + }); + + expect(result).toEqual({ lastSeenId: "3-0", updatedSlices: 2 }); + expect(garnet.strings.get(EVAL_AGG_CURSOR_KEY)).toBe("3-0"); + + const key = getSliceAggregateKey({ + tier: 0, + intent: "status-question", + phase: "photo-pending", + }); + expect(garnet.hashes.get(key)).toMatchObject({ + count: "2", + llmOverallSum: "6", + llmOverallCount: "2", + validatorRejectCount: "1", + latencySumMs: "400", + }); + expect(JSON.parse(garnet.hashes.get(key)?.["latencySamples"] ?? "[]")).toEqual([ + 100, + 300, + ]); + expect(garnet.expires.get(key)).toBeGreaterThan(0); + }); +}); diff --git a/src/eval/aggregator.ts b/src/eval/aggregator.ts new file mode 100644 index 0000000..c8c011d --- /dev/null +++ b/src/eval/aggregator.ts @@ -0,0 +1,396 @@ +import type { GarnetClient, StreamEntry } from "@common/garnet"; +import { + getEvalAggLatencySampleCap, + getEvalAggTtlSec, +} from "../config.js"; + +export const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results"; +export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor"; +const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:"; + +export interface SliceKey { + tier: 0 | 1 | 2; + intent: string; + phase: string; +} + +export interface SliceMetrics { + count: number; + llmOverallSum: number; + llmOverallCount: number; + validatorRejectCount: number; + validatorSoftWarnCount: number; + latencySumMs: number; + latencySamples: number[]; + processJudgeSum: number; + processJudgeCount: number; + updatedAt: string; +} + +export interface SliceReportRow { + slice: SliceKey; + count: number; + llmOverallAvg: number | null; + processJudgeAvg: number | null; + validatorRejectRate: number; + validatorSoftWarnRate: number; + p50LatencyMs: number | null; + p95LatencyMs: number | null; + updatedAt: string; +} + +type AggregatorGarnetClient = Pick< + GarnetClient, + | "expireKey" + | "getHash" + | "readStreamRange" + | "scanKeys" + | "setHash" + | "setString" +>; + +interface EvalResultObservation { + latencyMs: number | null; + llmOverall: number | null; + processJudgeScore: number | null; + slice: SliceKey; + validatorStatus: string | null; +} + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null + ? (value as Record) + : {}; +} + +function parseJsonRecord(value: string | undefined): Record { + if (!value?.trim()) { + return {}; + } + + try { + return asRecord(JSON.parse(value)); + } catch { + return {}; + } +} + +function parseFiniteNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value !== "string" || !value.trim()) { + return null; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function parseTier(value: unknown): 0 | 1 | 2 | null { + const parsed = parseFiniteNumber(value); + if (parsed === 0 || parsed === 1 || parsed === 2) { + return parsed; + } + return null; +} + +function getNestedString( + record: Record, + first: string, + second: string, +): string | null { + const nested = asRecord(record[first]); + const value = nested[second]; + return typeof value === "string" && value.trim() ? value : null; +} + +function extractLlmOverall(message: Record): number | null { + const direct = parseFiniteNumber(message["llmOverall"]); + if (direct !== null) { + return direct; + } + + const llmJudge = parseJsonRecord(message["llmJudge"]); + const scores = asRecord(llmJudge["scores"]); + return parseFiniteNumber(scores["overall"]); +} + +function extractProcessJudgeScore( + message: Record, +): number | null { + const processJudge = parseJsonRecord(message["processJudge"]); + return parseFiniteNumber(processJudge["score"]); +} + +function extractValidatorStatus(message: Record): string | null { + if (message["validatorStatus"]?.trim()) { + return message["validatorStatus"] ?? null; + } + + const validation = parseJsonRecord(message["validation"]); + const status = validation["status"]; + return typeof status === "string" && status.trim() ? status : null; +} + +function parseObservation(entry: StreamEntry): EvalResultObservation | null { + const derived = parseJsonRecord(entry.message["derived"]); + const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]); + const intent = + entry.message["intent"]?.trim() || + getNestedString(derived, "intent", "kind"); + const phase = + entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind"); + + if (tier === null || !intent || !phase) { + return null; + } + + return { + latencyMs: parseFiniteNumber(entry.message["latencyMs"]), + llmOverall: extractLlmOverall(entry.message), + processJudgeScore: extractProcessJudgeScore(entry.message), + slice: { tier, intent, phase }, + validatorStatus: extractValidatorStatus(entry.message), + }; +} + +function parseNumberField( + hash: Record, + field: keyof SliceMetrics, +): number { + const value = parseFiniteNumber(hash[field]); + return value ?? 0; +} + +function parseLatencySamples(hash: Record): number[] { + const raw = hash["latencySamples"]; + if (!raw?.trim()) { + return []; + } + + try { + const parsed = JSON.parse(raw) as unknown; + if (!Array.isArray(parsed)) { + return []; + } + return parsed.flatMap((item) => { + const value = parseFiniteNumber(item); + return value === null ? [] : [value]; + }); + } catch { + return []; + } +} + +function metricsFromHash(hash: Record): SliceMetrics { + return { + count: parseNumberField(hash, "count"), + llmOverallSum: parseNumberField(hash, "llmOverallSum"), + llmOverallCount: parseNumberField(hash, "llmOverallCount"), + validatorRejectCount: parseNumberField(hash, "validatorRejectCount"), + validatorSoftWarnCount: parseNumberField(hash, "validatorSoftWarnCount"), + latencySumMs: parseNumberField(hash, "latencySumMs"), + latencySamples: parseLatencySamples(hash), + processJudgeSum: parseNumberField(hash, "processJudgeSum"), + processJudgeCount: parseNumberField(hash, "processJudgeCount"), + updatedAt: hash["updatedAt"] ?? new Date(0).toISOString(), + }; +} + +function metricsToHash(metrics: SliceMetrics): Record { + return { + count: String(metrics.count), + llmOverallSum: String(metrics.llmOverallSum), + llmOverallCount: String(metrics.llmOverallCount), + validatorRejectCount: String(metrics.validatorRejectCount), + validatorSoftWarnCount: String(metrics.validatorSoftWarnCount), + latencySumMs: String(metrics.latencySumMs), + latencySamples: JSON.stringify(metrics.latencySamples), + processJudgeSum: String(metrics.processJudgeSum), + processJudgeCount: String(metrics.processJudgeCount), + updatedAt: metrics.updatedAt, + }; +} + +function updateMetrics( + current: SliceMetrics, + observation: EvalResultObservation, + nowMs: number, +): SliceMetrics { + const latencySamples = [...current.latencySamples]; + const latencySampleCap = getEvalAggLatencySampleCap(); + + if (observation.latencyMs !== null) { + latencySamples.push(observation.latencyMs); + if (latencySamples.length > latencySampleCap) { + latencySamples.splice(0, latencySamples.length - latencySampleCap); + } + } + + return { + count: current.count + 1, + llmOverallSum: + current.llmOverallSum + (observation.llmOverall ?? 0), + llmOverallCount: + current.llmOverallCount + (observation.llmOverall === null ? 0 : 1), + validatorRejectCount: + current.validatorRejectCount + + (observation.validatorStatus === "reject" ? 1 : 0), + validatorSoftWarnCount: + current.validatorSoftWarnCount + + (observation.validatorStatus === "soft-warn" ? 1 : 0), + latencySumMs: current.latencySumMs + (observation.latencyMs ?? 0), + latencySamples, + processJudgeSum: + current.processJudgeSum + (observation.processJudgeScore ?? 0), + processJudgeCount: + current.processJudgeCount + + (observation.processJudgeScore === null ? 0 : 1), + updatedAt: new Date(nowMs).toISOString(), + }; +} + +function streamRangeStart(sinceId: string): string { + const trimmed = sinceId.trim(); + if (!trimmed || trimmed === "0-0" || trimmed === "-") { + return "-"; + } + return `(${trimmed}`; +} + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +function calculatePercentile(values: number[], percentile: number): number | null { + if (values.length === 0) { + return null; + } + + const sorted = [...values].sort((left, right) => left - right); + const index = Math.min( + sorted.length - 1, + Math.max(0, Math.ceil(sorted.length * percentile) - 1), + ); + const selected = sorted[index]; + return selected === undefined ? null : round(selected); +} + +function average(sum: number, count: number): number | null { + return count > 0 ? round(sum / count) : null; +} + +function sliceKeyFromString(value: string): SliceKey | null { + const [tierRaw, intent, phase] = value.split(":"); + const tier = parseTier(tierRaw); + if (tier === null || !intent || !phase) { + return null; + } + + return { tier, intent, phase }; +} + +function sliceKeyFromRedisKey(key: string): SliceKey | null { + if (!key.startsWith(EVAL_AGG_SLICE_KEY_PREFIX)) { + return null; + } + + return sliceKeyFromString(key.slice(EVAL_AGG_SLICE_KEY_PREFIX.length)); +} + +export function sliceKeyToString(key: SliceKey): string { + return `${key.tier}:${key.intent}:${key.phase}`; +} + +export function getSliceAggregateKey(key: SliceKey): string { + return `${EVAL_AGG_SLICE_KEY_PREFIX}${sliceKeyToString(key)}`; +} + +export function sliceMetricsToReportRow( + slice: SliceKey, + metrics: SliceMetrics, +): SliceReportRow { + return { + slice, + count: metrics.count, + llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount), + processJudgeAvg: average( + metrics.processJudgeSum, + metrics.processJudgeCount, + ), + validatorRejectRate: + metrics.count === 0 + ? 0 + : round(metrics.validatorRejectCount / metrics.count), + validatorSoftWarnRate: + metrics.count === 0 + ? 0 + : round(metrics.validatorSoftWarnCount / metrics.count), + p50LatencyMs: calculatePercentile(metrics.latencySamples, 0.5), + p95LatencyMs: calculatePercentile(metrics.latencySamples, 0.95), + updatedAt: metrics.updatedAt, + }; +} + +export async function aggregateRecentEvalResults( + garnetClient: AggregatorGarnetClient, + options: { sinceId: string; nowMs: number }, +): Promise<{ lastSeenId: string; updatedSlices: number }> { + const entries = await garnetClient.readStreamRange( + EVAL_RESULTS_STREAM_KEY, + streamRangeStart(options.sinceId), + "+", + ); + + if (entries.length === 0) { + return { + lastSeenId: options.sinceId, + updatedSlices: 0, + }; + } + + const updatedKeys = new Set(); + let lastSeenId = options.sinceId; + + for (const entry of entries) { + lastSeenId = entry.id; + const observation = parseObservation(entry); + if (!observation) { + continue; + } + + const key = getSliceAggregateKey(observation.slice); + const current = metricsFromHash(await garnetClient.getHash(key)); + const next = updateMetrics(current, observation, options.nowMs); + await garnetClient.setHash(key, metricsToHash(next)); + await garnetClient.expireKey(key, getEvalAggTtlSec()); + updatedKeys.add(key); + } + + await garnetClient.setString(EVAL_AGG_CURSOR_KEY, lastSeenId); + + return { + lastSeenId, + updatedSlices: updatedKeys.size, + }; +} + +export async function readAggregatedSliceReportRows( + garnetClient: AggregatorGarnetClient, +): Promise { + const keys = await garnetClient.scanKeys(`${EVAL_AGG_SLICE_KEY_PREFIX}*`); + const rows: SliceReportRow[] = []; + + for (const key of keys) { + const slice = sliceKeyFromRedisKey(key); + if (!slice) { + continue; + } + + const metrics = metricsFromHash(await garnetClient.getHash(key)); + rows.push(sliceMetricsToReportRow(slice, metrics)); + } + + return rows; +} diff --git a/src/eval/anonymize.ts b/src/eval/anonymize.ts index e37ecab..4a8f447 100644 --- a/src/eval/anonymize.ts +++ b/src/eval/anonymize.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; const EVAL_ANON_SALT = "lolympus-agent-v1" as const; @@ -51,3 +52,32 @@ export function anonymizeRelativeTimestamp( const seconds = Math.round((timestamp - baseTimestamp.getTime()) / 1000); return `${seconds >= 0 ? "+" : ""}${seconds}s`; } + +function capText(value: string, maxLength = 600): string { + return value.length > maxLength ? value.slice(0, maxLength) : value; +} + +function anonymizeUnknown(value: unknown): unknown { + if (typeof value === "string") { + return capText(anonymizeText(value)); + } + if (Array.isArray(value)) { + return value.map((item) => anonymizeUnknown(item)); + } + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value).map(([key, nested]) => [ + key, + anonymizeUnknown(nested), + ]), + ); + } + + return value; +} + +export function anonymizeProcessTrace( + trace: ProcessTraceSnapshot, +): ProcessTraceSnapshot { + return anonymizeUnknown(trace) as ProcessTraceSnapshot; +} diff --git a/src/eval/archive.test.ts b/src/eval/archive.test.ts new file mode 100644 index 0000000..c820018 --- /dev/null +++ b/src/eval/archive.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; + +import type { StreamEntry } from "@common/garnet"; +import { + archiveEvalResults, + EVAL_ARCHIVE_CURSOR_KEY, + EVAL_ARCHIVE_COLLECTION, + type EvalArchiveEntry, +} from "./archive.js"; +import { EVAL_RESULTS_STREAM_KEY } from "./aggregator.js"; + +class MemoryGarnet { + readonly strings = new Map(); + + constructor(private readonly entries: StreamEntry[]) {} + + readStreamRange( + key: string, + start: string, + _end: string, + options?: { count?: number | undefined }, + ): Promise { + expect(key).toBe(EVAL_RESULTS_STREAM_KEY); + const filtered = start.startsWith("(") + ? this.entries.filter((entry) => entry.id > start.slice(1)) + : this.entries; + return Promise.resolve(filtered.slice(0, options?.count)); + } + + setString(key: string, value: string): Promise { + this.strings.set(key, value); + return Promise.resolve(); + } +} + +class MemoryCollection { + readonly docs = new Map(); + readonly indexes: Array> = []; + + bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> { + let upsertedCount = 0; + for (const operation of operations) { + const updateOne = (operation as { + updateOne: { + filter: { streamId: string }; + update: { $setOnInsert: EvalArchiveEntry }; + }; + }).updateOne; + const streamId = updateOne.filter.streamId; + if (!this.docs.has(streamId)) { + this.docs.set(streamId, updateOne.update.$setOnInsert); + upsertedCount += 1; + } + } + return Promise.resolve({ upsertedCount }); + } + + createIndex(keys: Record): Promise { + this.indexes.push(keys); + return Promise.resolve("idx"); + } + + find(): never { + throw new Error("not used"); + } +} + +class MemoryConnection { + readonly collectionInstance = new MemoryCollection(); + + collection(name: string): MemoryCollection { + expect(name).toBe(EVAL_ARCHIVE_COLLECTION); + return this.collectionInstance; + } +} + +function streamEntry(id: string): StreamEntry { + return { + id, + message: { + input: JSON.stringify({ + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "닉", + trigger: "user-message", + userMessage: "상태 알려줘", + }, + }), + output: JSON.stringify({ + requestId: "req-1", + text: "사진 인증만 남았어요.", + confidence: 0.9, + source: "llm", + }), + derived: JSON.stringify({ + phase: { kind: "photo-pending" }, + intent: { kind: "status-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + }), + classified: JSON.stringify({ + intent: { kind: "status-question" }, + confidence: 0.9, + source: "regex", + }), + validation: JSON.stringify({ + status: "pass", + text: "사진 인증만 남았어요.", + reasons: [], + }), + llmJudge: JSON.stringify({ + skipped: false, + model: "judge", + endpoint: "local", + provider: "test", + scores: { + friendliness: 4, + accuracy: 5, + concision: 4, + scope: 5, + overall: 4.5, + }, + rationale: "ok", + raw: "{}", + }), + tier: "0", + latencyMs: "123", + evaluatedAt: "2026-05-01T00:00:00.000Z", + }, + }; +} + +describe("archiveEvalResults", () => { + it("archives stream entries idempotently and advances the cursor", async () => { + const garnet = new MemoryGarnet([streamEntry("1-0")]); + const connection = new MemoryConnection(); + + const first = await archiveEvalResults(garnet, connection, { + sinceId: "0-0", + nowMs: Date.parse("2026-05-01T00:00:00.000Z"), + }); + const second = await archiveEvalResults(garnet, connection, { + sinceId: "0-0", + nowMs: Date.parse("2026-05-01T00:00:00.000Z"), + }); + + expect(first).toEqual({ lastArchivedId: "1-0", archivedCount: 1 }); + expect(second).toEqual({ lastArchivedId: "1-0", archivedCount: 0 }); + expect(garnet.strings.get(EVAL_ARCHIVE_CURSOR_KEY)).toBe("1-0"); + expect(connection.collectionInstance.docs.get("1-0")).toMatchObject({ + streamId: "1-0", + latencyMs: 123, + tier: 0, + }); + expect(connection.collectionInstance.indexes).toContainEqual({ streamId: 1 }); + }); +}); diff --git a/src/eval/archive.ts b/src/eval/archive.ts new file mode 100644 index 0000000..a07f166 --- /dev/null +++ b/src/eval/archive.ts @@ -0,0 +1,404 @@ +import type { AgentRequest, AgentResponse } from "@agent/types"; +import type { GarnetClient, StreamEntry } from "@common/garnet"; +import type { + ClassifiedIntent, + DerivedState, +} from "@skills/onboarding/state/onboarding-state"; +import type { ValidationResult } from "@skills/onboarding/runtime/validator"; +import { + getEvalArchiveBatchSize, + getEvalArchiveTtlDays, +} from "../config.js"; +import type { LlmJudgeResult } from "./types.js"; +import type { ProcessJudgeResult } from "./types.js"; +import { + EVAL_RESULTS_STREAM_KEY, + type SliceKey, + type SliceReportRow, +} from "./aggregator.js"; + +export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor"; +export const EVAL_ARCHIVE_COLLECTION = "eval_archive"; + +export interface EvalArchiveEntry { + streamId: string; + evaluatedAt: Date; + request: AgentRequest; + response: AgentResponse; + derived: DerivedState; + classified: ClassifiedIntent; + validation: ValidationResult; + llmJudge: LlmJudgeResult; + processJudge?: ProcessJudgeResult | undefined; + process?: unknown; + latencyMs: number | null; + tier: 0 | 1 | 2; +} + +interface BulkWriteResultLike { + upsertedCount?: number | undefined; +} + +interface FindCursorLike { + limit(value: number): FindCursorLike; + sort(value: Record): FindCursorLike; + toArray(): Promise; +} + +interface EvalArchiveCollectionLike { + bulkWrite( + operations: unknown[], + options?: { ordered?: boolean | undefined }, + ): Promise; + createIndex( + keys: Record, + options?: Record, + ): Promise; + find(filter: Record): FindCursorLike; +} + +export interface MongoConnectionLike { + collection(name: string): unknown; +} + +type ArchiveGarnetClient = Pick< + GarnetClient, + "readStreamRange" | "setString" +>; + +function parseJsonField(value: string | undefined, field: string): T { + if (!value?.trim()) { + throw new Error(`Missing ${field} field`); + } + + return JSON.parse(value) as T; +} + +function parseOptionalJsonField(value: string | undefined): T | undefined { + if (!value?.trim()) { + return undefined; + } + + return JSON.parse(value) as T; +} + +function parseFiniteNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value !== "string" || !value.trim()) { + return null; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function parseTier(value: unknown): 0 | 1 | 2 | null { + const parsed = parseFiniteNumber(value); + if (parsed === 0 || parsed === 1 || parsed === 2) { + return parsed; + } + return null; +} + +function parseEvaluatedAt(value: string | undefined, nowMs: number): Date { + if (!value?.trim()) { + return new Date(nowMs); + } + + const parsed = new Date(value); + return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs); +} + +function streamRangeStart(sinceId: string): string { + const trimmed = sinceId.trim(); + if (!trimmed || trimmed === "0-0" || trimmed === "-") { + return "-"; + } + return `(${trimmed}`; +} + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +function calculatePercentile(values: number[], percentile: number): number | null { + if (values.length === 0) { + return null; + } + + const sorted = [...values].sort((left, right) => left - right); + const index = Math.min( + sorted.length - 1, + Math.max(0, Math.ceil(sorted.length * percentile) - 1), + ); + const selected = sorted[index]; + return selected === undefined ? null : round(selected); +} + +function getSlice(entry: EvalArchiveEntry): SliceKey { + return { + tier: entry.tier, + intent: entry.derived.intent.kind, + phase: entry.derived.phase.kind, + }; +} + +function getLlmOverall(entry: EvalArchiveEntry): number | null { + if (entry.llmJudge.skipped) { + return null; + } + + return entry.llmJudge.scores.overall; +} + +function sliceKeyToMapKey(slice: SliceKey): string { + return `${slice.tier}:${slice.intent}:${slice.phase}`; +} + +function parseEvalArchiveEntry( + entry: StreamEntry, + nowMs: number, +): EvalArchiveEntry { + const input = parseJsonField<{ request: AgentRequest }>( + entry.message["input"], + "input", + ); + const derived = parseJsonField( + entry.message["derived"], + "derived", + ); + const processJudge = parseOptionalJsonField( + entry.message["processJudge"], + ); + const process = parseOptionalJsonField(entry.message["process"]); + const tier = parseTier(entry.message["tier"]) ?? parseTier(derived.tier); + + if (tier === null) { + throw new Error("Missing valid tier"); + } + + const archiveEntry: EvalArchiveEntry = { + streamId: entry.id, + evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs), + request: input.request, + response: parseJsonField(entry.message["output"], "output"), + derived, + classified: parseJsonField( + entry.message["classified"], + "classified", + ), + validation: parseJsonField( + entry.message["validation"], + "validation", + ), + llmJudge: parseJsonField( + entry.message["llmJudge"], + "llmJudge", + ), + latencyMs: parseFiniteNumber(entry.message["latencyMs"]), + tier, + }; + + if (processJudge !== undefined) { + archiveEntry.processJudge = processJudge; + } + if (process !== undefined) { + archiveEntry.process = process; + } + + return archiveEntry; +} + +async function ensureEvalArchiveIndexes( + collection: EvalArchiveCollectionLike, +): Promise { + await Promise.all([ + collection.createIndex({ streamId: 1 }, { unique: true }), + collection.createIndex( + { evaluatedAt: 1 }, + { expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 }, + ), + collection.createIndex({ + tier: 1, + "derived.intent.kind": 1, + "derived.phase.kind": 1, + evaluatedAt: -1, + }), + ]); +} + +export function getEvalArchiveCollection( + mongoConnection: MongoConnectionLike, +): EvalArchiveCollectionLike { + return mongoConnection.collection(EVAL_ARCHIVE_COLLECTION) as EvalArchiveCollectionLike; +} + +export async function archiveEvalResults( + garnetClient: ArchiveGarnetClient, + mongoConnection: MongoConnectionLike, + options: { + batchSize?: number | undefined; + nowMs: number; + sinceId: string; + }, +): Promise<{ lastArchivedId: string; archivedCount: number }> { + const collection = getEvalArchiveCollection(mongoConnection); + await ensureEvalArchiveIndexes(collection); + + const entries = await garnetClient.readStreamRange( + EVAL_RESULTS_STREAM_KEY, + streamRangeStart(options.sinceId), + "+", + { + count: options.batchSize ?? getEvalArchiveBatchSize(), + }, + ); + + if (entries.length === 0) { + return { + lastArchivedId: options.sinceId, + archivedCount: 0, + }; + } + + const lastArchivedId = entries.at(-1)?.id ?? options.sinceId; + const archiveEntries = entries.flatMap((entry) => { + try { + return [parseEvalArchiveEntry(entry, options.nowMs)]; + } catch { + return []; + } + }); + + let archivedCount = 0; + if (archiveEntries.length > 0) { + const result = await collection.bulkWrite( + archiveEntries.map((entry) => ({ + updateOne: { + filter: { streamId: entry.streamId }, + update: { $setOnInsert: entry }, + upsert: true, + }, + })), + { ordered: false }, + ); + archivedCount = result.upsertedCount ?? 0; + } + + await garnetClient.setString(EVAL_ARCHIVE_CURSOR_KEY, lastArchivedId); + + return { + lastArchivedId, + archivedCount, + }; +} + +export function archiveEntryToSliceReportRows( + entries: EvalArchiveEntry[], +): SliceReportRow[] { + const grouped = new Map< + string, + { + count: number; + latencySamples: number[]; + llmOverallCount: number; + llmOverallSum: number; + slice: SliceKey; + validatorRejectCount: number; + validatorSoftWarnCount: number; + updatedAt: Date; + } + >(); + + for (const entry of entries) { + const slice = getSlice(entry); + const key = sliceKeyToMapKey(slice); + const current = + grouped.get(key) ?? + { + count: 0, + latencySamples: [], + llmOverallCount: 0, + llmOverallSum: 0, + slice, + validatorRejectCount: 0, + validatorSoftWarnCount: 0, + updatedAt: new Date(0), + }; + + current.count += 1; + if (entry.latencyMs !== null) { + current.latencySamples.push(entry.latencyMs); + } + const llmOverall = getLlmOverall(entry); + if (llmOverall !== null) { + current.llmOverallCount += 1; + current.llmOverallSum += llmOverall; + } + if (entry.validation.status === "reject") { + current.validatorRejectCount += 1; + } + if (entry.validation.status === "soft-warn") { + current.validatorSoftWarnCount += 1; + } + if (entry.evaluatedAt > current.updatedAt) { + current.updatedAt = entry.evaluatedAt; + } + + grouped.set(key, current); + } + + return [...grouped.values()].map((group) => ({ + slice: group.slice, + count: group.count, + llmOverallAvg: + group.llmOverallCount === 0 + ? null + : round(group.llmOverallSum / group.llmOverallCount), + processJudgeAvg: null, + validatorRejectRate: + group.count === 0 ? 0 : round(group.validatorRejectCount / group.count), + validatorSoftWarnRate: + group.count === 0 + ? 0 + : round(group.validatorSoftWarnCount / group.count), + p50LatencyMs: calculatePercentile(group.latencySamples, 0.5), + p95LatencyMs: calculatePercentile(group.latencySamples, 0.95), + updatedAt: group.updatedAt.toISOString(), + })); +} + +export async function queryArchiveSliceReportRows( + mongoConnection: MongoConnectionLike, + options?: { + limit?: number | undefined; + sinceMs?: number | undefined; + sort?: "count-desc" | "score-asc" | "score-desc" | undefined; + }, +): Promise { + const sinceMs = options?.sinceMs ?? 7 * 24 * 60 * 60 * 1000; + const since = new Date(Date.now() - sinceMs); + const archiveEntries = await getEvalArchiveCollection(mongoConnection) + .find({ evaluatedAt: { $gte: since } }) + .sort({ evaluatedAt: -1 }) + .limit(10_000) + .toArray(); + const rows = archiveEntryToSliceReportRows(archiveEntries); + const sort = options?.sort ?? "count-desc"; + + rows.sort((left, right) => { + if (sort === "score-asc") { + return (left.llmOverallAvg ?? Number.POSITIVE_INFINITY) - + (right.llmOverallAvg ?? Number.POSITIVE_INFINITY); + } + if (sort === "score-desc") { + return (right.llmOverallAvg ?? Number.NEGATIVE_INFINITY) - + (left.llmOverallAvg ?? Number.NEGATIVE_INFINITY); + } + return right.count - left.count; + }); + + return rows.slice(0, options?.limit ?? 20); +} diff --git a/src/eval/capture.ts b/src/eval/capture.ts index f2854ff..06d7cf3 100644 --- a/src/eval/capture.ts +++ b/src/eval/capture.ts @@ -50,7 +50,9 @@ export async function captureGoldenSetScaffold( process.env["MONGODB_CONNECTION_STRING"]; if (!connectionString) { - throw new Error("Missing MONGODB_CONNECTION_STRING for eval capture"); + throw new Error( + "Missing MONGODB_CONNECTION_STRING for eval capture. Ensure IMDS exposes Infisical client credentials and Infisical contains MONGODB_CONNECTION_STRING, or set it directly in the environment.", + ); } await initMongoDB(connectionString); diff --git a/src/eval/cli.ts b/src/eval/cli.ts index 7d1cca3..e625971 100644 --- a/src/eval/cli.ts +++ b/src/eval/cli.ts @@ -1,3 +1,7 @@ +import type { MongoConnectionLike } from "./archive.js"; +import type { SliceKey, SliceReportRow } from "./aggregator.js"; +import type { GarnetClient } from "@common/garnet"; + function parsePositiveInteger(value: string | undefined): number | undefined { if (!value?.trim()) { return undefined; @@ -7,6 +11,78 @@ function parsePositiveInteger(value: string | undefined): number | undefined { return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; } +function parsePositiveNumber(value: string | undefined): number | undefined { + if (!value?.trim()) { + return undefined; + } + + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + +function parseDurationMs(value: string | undefined): number { + const raw = value?.trim().toLowerCase() || "7d"; + const match = raw.match(/^(\d+)(h|d)$/u); + if (!match) { + throw new Error("--since must use a duration like 24h, 7d, or 30d"); + } + + const amount = Number(match[1]); + const unit = match[2]; + if (!Number.isInteger(amount) || amount <= 0) { + throw new Error("--since must be a positive duration"); + } + + return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000; +} + +function parseCsv(value: string | undefined): string[] { + if (!value?.trim()) { + return []; + } + + return value + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); +} + +function parseSlice(value: string | undefined): SliceKey { + if (!value?.trim()) { + throw new Error( + "--slice must use tier=0,intent=status-question,phase=photo-pending", + ); + } + + const fields: Record = {}; + for (const pair of value.split(",")) { + const [key, ...rest] = pair.split("="); + const normalizedKey = key?.trim(); + if (normalizedKey) { + fields[normalizedKey] = rest.join("=").trim(); + } + } + const tier = Number(fields["tier"]); + const intent = fields["intent"]; + const phase = fields["phase"]; + + if ( + (tier !== 0 && tier !== 1 && tier !== 2) || + !intent?.trim() || + !phase?.trim() + ) { + throw new Error( + "--slice must include tier=0|1|2,intent=,phase=", + ); + } + + return { + tier, + intent, + phase, + }; +} + interface ParsedArgs { command: string; flags: Map; @@ -108,6 +184,122 @@ function printRunSummary(report: { console.info(`[eval] report=${reportPath}`); } +function formatNullable(value: number | null): string { + return value === null ? "n/a" : String(value); +} + +function printSliceRows(rows: SliceReportRow[]): void { + console.info( + [ + "tier", + "intent", + "phase", + "count", + "llmOverallAvg", + "validatorRejectRate", + "p50LatencyMs", + "p95LatencyMs", + "updatedAt", + ].join("\t"), + ); + + for (const row of rows) { + console.info( + [ + row.slice.tier, + row.slice.intent, + row.slice.phase, + row.count, + formatNullable(row.llmOverallAvg), + row.validatorRejectRate, + formatNullable(row.p50LatencyMs), + formatNullable(row.p95LatencyMs), + row.updatedAt, + ].join("\t"), + ); + } +} + +function printCalibrationReport(report: { + generatedAt: string; + meanGapOverall: number; + pairs: number; + pearsonByDim: { + accuracy: number; + concision: number; + friendliness: number; + scope: number; + }; + pearsonOverall: number; + worstDisagreements: Array<{ + evalStreamId: string; + humanScore: number; + llmOverall: number; + }>; +}): void { + console.info(`[eval] generatedAt=${report.generatedAt}`); + console.info(`[eval] pairs=${report.pairs}`); + console.info(`[eval] pearsonOverall=${report.pearsonOverall}`); + console.info(`[eval] meanGapOverall=${report.meanGapOverall}`); + console.info( + `[eval] pearsonByDim friendliness=${report.pearsonByDim.friendliness} accuracy=${report.pearsonByDim.accuracy} concision=${report.pearsonByDim.concision} scope=${report.pearsonByDim.scope}`, + ); + for (const item of report.worstDisagreements) { + console.info( + `[eval] disagreement evalStreamId=${item.evalStreamId} human=${item.humanScore} llm=${item.llmOverall}`, + ); + } +} + +function printTournamentResult(result: { + generatedAt: string; + matches: Array<{ judgeVerdict: string; variantA: string; variantB: string }>; + variants: Array<{ elo: number; id: string; matches: number; wins: number }>; +}): void { + console.info(`[eval] generatedAt=${result.generatedAt}`); + console.info(`[eval] matches=${result.matches.length}`); + for (const variant of result.variants) { + console.info( + `[eval] variant id=${variant.id} elo=${variant.elo} matches=${variant.matches} wins=${variant.wins}`, + ); + } +} + +async function connectMongoForEval(): Promise<{ + close: () => Promise; + connection: MongoConnectionLike; +}> { + const { initMongoDB, closeMongoDB } = await import("@common/mongodb"); + const { getSecret } = await import("@common/secret-manager"); + const connectionString = + getSecret("MONGODB_CONNECTION_STRING") ?? + process.env["MONGODB_CONNECTION_STRING"]; + + if (!connectionString) { + throw new Error( + "Missing MONGODB_CONNECTION_STRING for eval command. Set it in the environment or secrets.", + ); + } + + return { + connection: await initMongoDB(connectionString), + close: closeMongoDB, + }; +} + +async function connectGarnetForEval(): Promise<{ + close: () => Promise; + garnetClient: GarnetClient; +}> { + const { GarnetClient } = await import("@common/garnet"); + const garnetClient = new GarnetClient(); + await garnetClient.connect(); + return { + garnetClient, + close: () => garnetClient.close(), + }; +} + async function main(): Promise { const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader"); loadEnvironmentFromNodeEnv(); @@ -174,6 +366,211 @@ async function main(): Promise { return; } + if (command === "report") { + const { queryArchiveSliceReportRows } = await import("./archive.js"); + const limit = + parsePositiveInteger(getFlagString(flags, "worst")) ?? + parsePositiveInteger(getFlagString(flags, "top")) ?? + 20; + const sort = hasFlag(flags, "worst") ? "score-asc" : "count-desc"; + const mongo = await connectMongoForEval(); + try { + const rows = await queryArchiveSliceReportRows(mongo.connection, { + limit, + sinceMs: parseDurationMs(getFlagString(flags, "since")), + sort, + }); + printSliceRows(rows); + } finally { + await mongo.close(); + } + return; + } + + if (command === "promote") { + const { appendCandidatesToJsonl, selectPromotionCandidates } = await import( + "./promote.js" + ); + const mongo = await connectMongoForEval(); + try { + const candidates = await selectPromotionCandidates(mongo.connection, { + limit: parsePositiveInteger(getFlagString(flags, "limit")), + minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")), + }); + + if (hasFlag(flags, "dry-run")) { + for (const candidate of candidates) { + const overall = candidate.llmJudge.skipped + ? "n/a" + : candidate.llmJudge.scores.overall; + console.info( + `[eval] candidate id=${candidate.caseId} reason=${candidate.reason} overall=${overall} trigger=${candidate.request.trigger}`, + ); + } + console.info(`[eval] dryRunCandidates=${candidates.length}`); + return; + } + + const result = await appendCandidatesToJsonl( + candidates, + getFlagString(flags, "output"), + ); + console.info( + `[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`, + ); + } finally { + await mongo.close(); + } + return; + } + + if (command === "review") { + const { parseReviewReason, runReviewCli } = await import("./review/cli.js"); + const garnet = await connectGarnetForEval(); + try { + const result = await runReviewCli(garnet.garnetClient, { + exportLabeled: getFlagString(flags, "export-labeled"), + limit: parsePositiveInteger(getFlagString(flags, "limit")), + reason: parseReviewReason(getFlagString(flags, "reason")), + resetPending: hasFlag(flags, "reset-pending"), + }); + console.info( + `[eval] review completed labeled=${result.labeled} skipped=${result.skipped}`, + ); + } finally { + await garnet.close(); + } + return; + } + + if (command === "calibrate") { + const { getEvalCalibrationMinCorr } = await import("../config.js"); + const { computeCalibrationReport } = await import( + "./review/calibration.js" + ); + const mongo = await connectMongoForEval(); + const garnet = await connectGarnetForEval(); + try { + const report = await computeCalibrationReport( + mongo.connection, + garnet.garnetClient, + { + sinceMs: getFlagString(flags, "since") + ? parseDurationMs(getFlagString(flags, "since")) + : undefined, + }, + ); + printCalibrationReport(report); + if (report.pairs < 10) { + console.warn( + "[eval] calibration pairs<10; check gate is informational until at least 10 labels are available", + ); + } + if ( + hasFlag(flags, "check") && + report.pairs >= 10 && + report.pearsonOverall < getEvalCalibrationMinCorr() + ) { + process.exitCode = 1; + } + } finally { + await garnet.close(); + await mongo.close(); + } + return; + } + + if (command === "propose") { + const { createEvalLlmClient } = await import("./runner.js"); + const { + buildProposalInputFromArchive, + proposeVariants, + writePromptVariantProposal, + } = await import("./loop/propose.js"); + const mongo = await connectMongoForEval(); + try { + const input = await buildProposalInputFromArchive( + mongo.connection, + parseSlice(getFlagString(flags, "slice")), + { + force: hasFlag(flags, "force"), + limit: parsePositiveInteger(getFlagString(flags, "limit")), + }, + ); + const proposals = await proposeVariants(createEvalLlmClient(), input); + for (const proposal of proposals.prompts) { + const outputPath = await writePromptVariantProposal( + proposal, + getFlagString(flags, "output-dir"), + ); + console.info( + `[eval] proposal variant=${proposal.variantId} output=${outputPath}`, + ); + } + for (const tool of proposals.tools) { + console.info( + `[eval] toolProposal variant=${tool.variantId} tool=${tool.toolName} why=${tool.whyNeeded}`, + ); + } + } finally { + await mongo.close(); + } + return; + } + + if (command === "tournament") { + const { createEvalLlmClient } = await import("./runner.js"); + const { + loadTournamentCases, + runTournament, + saveTournamentResult, + } = await import("./loop/tournament.js"); + const variantIds = parseCsv( + getFlagString(flags, "variants") ?? positionals[0], + ); + if (variantIds.length < 2) { + throw new Error( + 'eval:tournament requires --variants=control,', + ); + } + + const mongo = await connectMongoForEval(); + try { + const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches")); + const cases = await loadTournamentCases(mongo.connection, { + caseIds: parseCsv(getFlagString(flags, "case-ids")), + limit: maxMatches, + }); + const result = await runTournament(createEvalLlmClient(), { + cases, + maxMatches, + variantIds, + }); + await saveTournamentResult(mongo.connection, result); + printTournamentResult(result); + } finally { + await mongo.close(); + } + return; + } + + if (command === "tournament-results") { + const { queryTournamentResults } = await import("./loop/tournament.js"); + const mongo = await connectMongoForEval(); + try { + const results = await queryTournamentResults(mongo.connection, { + limit: parsePositiveInteger(getFlagString(flags, "limit")), + variantId: getFlagString(flags, "variant"), + }); + for (const result of results) { + printTournamentResult(result); + } + } finally { + await mongo.close(); + } + return; + } + throw new Error(`Unknown eval command: ${command}`); } catch (error) { logger.error( diff --git a/src/eval/drift.test.ts b/src/eval/drift.test.ts new file mode 100644 index 0000000..5d1ab23 --- /dev/null +++ b/src/eval/drift.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; + +import type { EvalArchiveEntry } from "./archive.js"; +import { computeDriftReportsFromEntries } from "./drift.js"; + +function archiveEntry( + evaluatedAt: Date, + overall: number, +): EvalArchiveEntry { + return { + streamId: `${evaluatedAt.toISOString()}:${overall}`, + evaluatedAt, + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "닉", + trigger: "user-message", + }, + response: { + requestId: "req-1", + text: "응답", + confidence: 0.9, + source: "llm", + }, + derived: { + phase: { kind: "photo-pending" }, + intent: { kind: "status-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + }, + classified: { + intent: { kind: "status-question" }, + confidence: 0.9, + source: "regex", + }, + validation: { + status: "pass", + text: "응답", + reasons: [], + }, + llmJudge: { + skipped: false, + model: "judge", + endpoint: "local", + provider: "test", + scores: { + friendliness: overall, + accuracy: overall, + concision: overall, + scope: overall, + overall, + }, + rationale: "ok", + raw: "{}", + }, + latencyMs: 100, + tier: 0, + }; +} + +describe("computeDriftReportsFromEntries", () => { + it("marks a slice significant when the recent window drops enough", () => { + const nowMs = Date.parse("2026-05-01T00:00:00.000Z"); + const recent = Array.from({ length: 30 }, (_, index) => + archiveEntry(new Date(nowMs - index * 60_000), 2), + ); + const older = Array.from({ length: 40 }, (_, index) => + archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000 - index * 60_000), 4), + ); + + const reports = computeDriftReportsFromEntries([...recent, ...older], { + nowMs, + thresholdPct: 15, + }); + + expect(reports).toHaveLength(1); + expect(reports[0]).toMatchObject({ + recent24hAverage: 2, + recent24hCount: 30, + recent7dCount: 70, + significant: true, + }); + expect(reports[0]?.dropPct).toBeGreaterThan(15); + }); + + it("requires enough recent samples for significance", () => { + const nowMs = Date.parse("2026-05-01T00:00:00.000Z"); + const reports = computeDriftReportsFromEntries( + [ + archiveEntry(new Date(nowMs), 1), + archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000), 5), + ], + { + nowMs, + thresholdPct: 15, + }, + ); + + expect(reports[0]?.significant).toBe(false); + }); +}); diff --git a/src/eval/drift.ts b/src/eval/drift.ts new file mode 100644 index 0000000..e32f2bb --- /dev/null +++ b/src/eval/drift.ts @@ -0,0 +1,158 @@ +import { getAlertThresholdDriftPct } from "../config.js"; +import type { + EvalArchiveEntry, + MongoConnectionLike, +} from "./archive.js"; +import { getEvalArchiveCollection } from "./archive.js"; +import type { SliceKey } from "./aggregator.js"; + +const DAY_MS = 24 * 60 * 60 * 1000; +const RECENT_WINDOW_MS = DAY_MS; +const BASELINE_WINDOW_MS = 7 * DAY_MS; +const MIN_SIGNIFICANT_RECENT_COUNT = 30; + +export interface DriftReport { + slice: SliceKey; + recent24hAverage: number; + recent7dAverage: number; + dropPct: number; + significant: boolean; + recent24hCount: number; + recent7dCount: number; +} + +interface DriftGroup { + baselineValues: number[]; + recentValues: number[]; + slice: SliceKey; +} + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +function average(values: number[]): number | null { + if (values.length === 0) { + return null; + } + + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function sliceKeyToString(slice: SliceKey): string { + return `${slice.tier}:${slice.intent}:${slice.phase}`; +} + +function getSlice(entry: EvalArchiveEntry): SliceKey { + return { + tier: entry.tier, + intent: entry.derived.intent.kind, + phase: entry.derived.phase.kind, + }; +} + +function getMetricValue( + entry: EvalArchiveEntry, + metricKey: "llmOverall" | "validatorPassRate", +): number | null { + if (metricKey === "validatorPassRate") { + return entry.validation.status === "pass" ? 1 : 0; + } + + if (entry.llmJudge.skipped) { + return null; + } + + return entry.llmJudge.scores.overall; +} + +export function computeDriftReportsFromEntries( + entries: EvalArchiveEntry[], + options?: { + metricKey?: "llmOverall" | "validatorPassRate" | undefined; + nowMs?: number | undefined; + thresholdPct?: number | undefined; + }, +): DriftReport[] { + const metricKey = options?.metricKey ?? "llmOverall"; + const nowMs = options?.nowMs ?? Date.now(); + const recentCutoff = nowMs - RECENT_WINDOW_MS; + const thresholdPct = options?.thresholdPct ?? getAlertThresholdDriftPct(); + const groups = new Map(); + + for (const entry of entries) { + const value = getMetricValue(entry, metricKey); + if (value === null) { + continue; + } + + const evaluatedAtMs = entry.evaluatedAt.getTime(); + if (!Number.isFinite(evaluatedAtMs)) { + continue; + } + + const slice = getSlice(entry); + const key = sliceKeyToString(slice); + const group = + groups.get(key) ?? + { + baselineValues: [], + recentValues: [], + slice, + }; + + group.baselineValues.push(value); + if (evaluatedAtMs >= recentCutoff) { + group.recentValues.push(value); + } + groups.set(key, group); + } + + return [...groups.values()].flatMap((group) => { + const recentAverage = average(group.recentValues); + const baselineAverage = average(group.baselineValues); + if (recentAverage === null || baselineAverage === null || baselineAverage <= 0) { + return []; + } + + const dropPct = ((baselineAverage - recentAverage) / baselineAverage) * 100; + + return [ + { + slice: group.slice, + recent24hAverage: round(recentAverage), + recent7dAverage: round(baselineAverage), + dropPct: round(dropPct), + significant: + group.recentValues.length >= MIN_SIGNIFICANT_RECENT_COUNT && + dropPct >= thresholdPct, + recent24hCount: group.recentValues.length, + recent7dCount: group.baselineValues.length, + }, + ]; + }); +} + +export async function computeDriftReports( + mongoConnection: MongoConnectionLike, + options?: { + metricKey?: "llmOverall" | "validatorPassRate" | undefined; + nowMs?: number | undefined; + }, +): Promise { + const nowMs = options?.nowMs ?? Date.now(); + const entries = await getEvalArchiveCollection(mongoConnection) + .find({ + evaluatedAt: { + $gte: new Date(nowMs - BASELINE_WINDOW_MS), + }, + }) + .sort({ evaluatedAt: -1 }) + .limit(50_000) + .toArray(); + + return computeDriftReportsFromEntries(entries, { + metricKey: options?.metricKey, + nowMs, + }); +} diff --git a/src/eval/golden/onboarding.candidate.jsonl b/src/eval/golden/onboarding.candidate.jsonl new file mode 100644 index 0000000..7caa6ab --- /dev/null +++ b/src/eval/golden/onboarding.candidate.jsonl @@ -0,0 +1,122 @@ + +{"id":"case_119e66ab","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777693377499-0; llmOverall=1.5; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=카카오톡 닉네임은 이름/출생 연도 두 자리/성별/티어 형식이에요. 예를 들면 면정학/89/남/D처럼 쓰시면 돼요.","input":{"request":{"requestId":"req_7e7337de","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"닉네임 변경 및 하트 인증은 하였는데, 혹시 어느 부분에서 문제가 있는지 여쭈어 볼 수 있을까요?"}},"expected":{}} +{"id":"case_6bc9858f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777692194987-0; llmOverall=1.5; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=공지방은 따로 운영하지 않습니다.","input":{"request":{"requestId":"req_c91897cf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"죄송합니다. 제가 기억하기론 공지방이 따로 있던 것으로 기억하는데, 공지방은 따로 없는지 여쭈어 볼 수 있을까요?"}},"expected":{}} +{"id":"case_edbec2e7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777678836132-0; llmOverall=2; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=디스코드의 주민신청 채널에서 멘션 남기면 됩니다.","input":{"request":{"requestId":"req_e69bfe09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_2096a462","nickname":"","trigger":"user-message","userMessage":"디코 공지 확인이랑 닉변까지 다 했는데 어디에서 멘션하면 되나여"}},"expected":{}} +{"id":"case_756da19b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693260088-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=아, 그러셨군요. 지금은 카카오톡 닉네임 변경만 남아 있어서 그 과정에서 막히신 거죠. \n무슨 부분이 어려우신가요? 어떤 과정에서 막히셨는지 알려 주세요.","input":{"request":{"requestId":"req_f09cd933","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"하는데"}},"expected":{}} +{"id":"case_70b3db02","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693242995-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=네, 배치 보려면 먼저 카카오톡 1단계부터 끝내야 해요. 현재는 닉네임 변경만 남아 있고, 마감이 임박했으니 지금 바로 닉네임만 먼저 완료해 주세요.","input":{"request":{"requestId":"req_b68c3a89","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"얼른 배치를 봐야만"}},"expected":{}} +{"id":"case_d8efa98f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777692242389-0; llmOverall=3.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 맞습니다. 구인글은 구인방에서만 올리는 게 원칙이에요.","input":{"request":{"requestId":"req_027cab09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"감사합니다.\n구인방에서만 구인글을 게시하는지 마지막으로 여쭈어 볼 수 있을까요?"}},"expected":{}} +{"id":"case_b93a3eac","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777691863103-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=하트 인증은 이렇게만 다시 보내면 돼요.\n\n채팅방 오른쪽 상단 메뉴 버튼 → ‘커버 보기’ 들어가기 \n하트를 빨간색으로 바꾸기 \n그 상태를 편집 없이 원본 그대로 캡처해서 사진으로 보내기\n\n자르기나 화면 꾸미기 같은 편집은 하지 말고, 그냥 바로 스크린샷 그대로 보내주세요.","input":{"request":{"requestId":"req_1785e212","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"photo-failed"}},"expected":{}} +{"id":"case_4c4e71d6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655309842-0; llmOverall=4.75; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=무슨 말인지 모르겠어용 ㅎㅎ 한 번만 더 정확히 말해줄 수 있어임니당?","input":{"request":{"requestId":"req_1147b077","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"","trigger":"user-message","userMessage":"머징"}},"expected":{}} +{"id":"case_b3ba68ec","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655250210-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=오키, 점수 200점 확인했어용! 잘 적어둘게요.","input":{"request":{"requestId":"req_1f66afdc","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"","trigger":"user-message","userMessage":"200점수따리라"}},"expected":{}} +{"id":"case_475ee343","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777649519625-0; llmOverall=4.75; derivedIntent=acknowledgement; derivedPhase=completed; assistantResponse=네, 다 잘 끝났으니까 푹 쉬었다 다녀오세요~","input":{"request":{"requestId":"req_11add9b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57a7acdc","nickname":"","trigger":"user-message","userMessage":"가겠습니당~~"}},"expected":{}} +{"id":"case_3029ecb9","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777778911429-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 카카오톡 구인방 입장, 디스코드 서버 닉네임 설정, 디스코드 주민신청 채널 메시지 남기기예요.","input":{"request":{"requestId":"req_d1ec54d0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"","trigger":"user-message","userMessage":"ai인가요 저분"}},"expected":{}} +{"id":"case_65a019a2","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777786270802-0; llmOverall=1.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=27세 이상이면 지금은 2가지만 하면 돼요. \n카카오톡 닉네임 형식으로 변경하고, 방 하트 인증을 완료해 주세요. \n카카오톡 닉네임 형식은 이름/출생 연도 두 자리/성별/티어이고, 방 하트 인증은 채팅방 오른쪽 상단 메뉴의 '커버 보기'로 들어가 하트를 빨간색으로 바꾼 뒤 편집 없는 원본 스크린샷을 올려주시면 돼요.","input":{"request":{"requestId":"req_d5b6165a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_911011bd","nickname":"","trigger":"user-message","userMessage":"가입하려구하는데어케하나요?"}},"expected":{}} +{"id":"case_60504e1f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777711499377-0; llmOverall=1.5; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 디스코드 주민신청 채널에 메시지 남기기만 완료하시면 됩니다.","input":{"request":{"requestId":"req_a50a581c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ae8873b8","nickname":"","trigger":"user-message","userMessage":"안녕하세요 :)\n오늘 내전 있을까요?"}},"expected":{}} +{"id":"case_a912b1f1","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777708255924-0; llmOverall=1.5; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=지금은 카카오톡 1단계(닉네임/방 하트 인증) 조건이 먼저 필요해요. \n현재 남은 항목은 방 하트 인증이고 아직 미완료 상태라, 이게 끝나야 디스코드 안내를 이어드릴 수 있어요.","input":{"request":{"requestId":"req_7ed0470a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c6e63b5e","nickname":"","trigger":"user-message","userMessage":"/디코"}},"expected":{}} +{"id":"case_1742409e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777778244667-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임이 안 되는 이유는 티어가 빠졌기 때문이에요. \n형식은 닉네임/출생연도/성별/티어이고, 지금 망태/95/남 뒤에 /D 같은 한 글자 티어만 추가하면 됩니다.","input":{"request":{"requestId":"req_cb17dffc","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_1ae6d8a5","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777768915244-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 형식이 맞지 않아서 안 됩니다. \n3렙탑갱필킬/ 뒤에 출생 연도 두 자리, 성별, 티어를 이름/연도/성별/티어 순서로 채워야 해요. \n예를 들어 3렙탑갱필킬/89/남/D 형태로 수정하면 됩니다.","input":{"request":{"requestId":"req_ec4696f6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0b248a72","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_a47d12f7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777708249627-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 너뒤에내가있다/98/남은 형식에서 마지막 티어가 빠져서 안 되는 거예요. \n닉네임/출생연도/성별/티어로 맞추려면 예) 너뒤에내가있다/98/남/D처럼 티어를 하나 추가하면 됩니다. \n티어는 GM, I, B, S, G, P, E, M, C, U 중 하나로 넣으면 돼요.","input":{"request":{"requestId":"req_0497f34d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c6e63b5e","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_fe83f154","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777710710256-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_47316f87","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_aaa91459","nickname":"","trigger":"user-message","userMessage":"넵"}},"expected":{}} +{"id":"case_07b6910d","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777778857540-0; llmOverall=2.75; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=구인방은 구인/구해요 글을 올리거나 확인하는 방이야.","input":{"request":{"requestId":"req_1480e5f2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"","trigger":"user-message","userMessage":"/구인방"}},"expected":{}} +{"id":"case_2e1c9c7d","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777711775572-0; llmOverall=2.75; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 잘 마무리하셨어요. 천천히 잘 마쳤네요.","input":{"request":{"requestId":"req_d2a86b68","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_dfd55a6d","nickname":"","trigger":"user-message","userMessage":"이제야 다 끝냈넹요,,"}},"expected":{}} +{"id":"case_c3be2795","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777778838385-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=완료된 건 빼고 남은 건 3개예요. \n카카오톡 구인방 입장, 디스코드 서버 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기.","input":{"request":{"requestId":"req_af508d85","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"","trigger":"user-message","userMessage":"아이게아닌가"}},"expected":{}} +{"id":"case_14386176","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777778305118-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=방하트 인증은 새내기 등록 1단계에서 채팅방 오른쪽 상단의 하트를 빨간색으로 바꿔 두고, 그 화면이 보이게 편집 없이 원본 스크린샷으로 보내는 인증이에요. \n방 하트 인증은 등록 1단계의 일부에 해당해요.","input":{"request":{"requestId":"req_6a17948f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"","trigger":"user-message","userMessage":"방하트인증은뭔가요?"}},"expected":{}} +{"id":"case_56116efc","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777778262296-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요. 편하게 진행해 주세요.","input":{"request":{"requestId":"req_f8ff4569","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"","trigger":"user-message","userMessage":"넹"}},"expected":{}} +{"id":"case_d44b5677","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777769958583-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 카카오톡 닉네임 변경과 방 하트 인증입니다.","input":{"request":{"requestId":"req_ad0a173e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_78fa631c","nickname":"","trigger":"user-message","userMessage":"아이언인데 괜찮나요"}},"expected":{}} +{"id":"case_42167e0d","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777769920617-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=어서오세요!","input":{"request":{"requestId":"req_525ed39b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_78fa631c","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_09834f65","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777768954579-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요, 반갑습니다!","input":{"request":{"requestId":"req_5763eb31","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0b248a72","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_f5695075","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777863604872-0; llmOverall=1.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=안 되는 이유는 성별(남/여) 칸이 빠져서예요. \n낑꽁껑꿍#2509/99/D에서 99와 D 사이에 남 또는 여를 넣어 \n낑꽁껑꿍#2509/99/남/D처럼 고치면 됩니다.","input":{"request":{"requestId":"req_49fcd14f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40adc660","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_aaaaf8dc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777876463824-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 “긁적이는 춘식이”는 이름만 있어요. \n규칙은 '/'로 이름/출생연도 두 자리/성별/티어 4칸이 들어가야 하는데, 지금은 1칸뿐이라 형식 오류예요. \n\n이름 뒤에 출생연도 두 자리/성별/티어를 붙여서 \n예) 춘식이/95/남/D 처럼 바꿔주면 돼요.","input":{"request":{"requestId":"req_fdb2a403","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c61645af","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_64c23ac4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777876455103-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 이동윤이 안 되는 이유는 형식이 안 맞아서예요. \n지금은 이름/출생연도 두 자리/성별/티어 중에서 이름만 있고 연도·성별·티어가 빠짐. \n\n이동윤/89/남/D처럼 이름/출생연도 두 자리/성별/티어 순서로 바꿔주면 돼요.","input":{"request":{"requestId":"req_b104b41f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c61645af","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_ae2f0863","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777876377377-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=어서오세요!","input":{"request":{"requestId":"req_0f32d5e8","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c61645af","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_537bd6bd","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777863619306-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_1582e6b5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40adc660","nickname":"","trigger":"user-message","userMessage":"넵 잠시만요!"}},"expected":{}} +{"id":"case_e57f3c00","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777833126526-0; llmOverall=n/a; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=네, 지금은 디스코드 승인 진행이 가능합니다. \n최종 등록 완료는 디스코드 관련 항목(서버 프로필 닉네임 설정, 주민신청 채널 메시지 남기기)과 카카오톡 본인인증방/구인방 입장이 모두 끝났을 때 확정돼요.","input":{"request":{"requestId":"req_9887396c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"","trigger":"user-message","userMessage":"혹시 디스코드 승인 가능한가요?"}},"expected":{}} +{"id":"case_d736890c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777831792635-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 편하게 진행해 주세요.","input":{"request":{"requestId":"req_97da2b0d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"","trigger":"user-message","userMessage":"네"}},"expected":{}} +{"id":"case_86ed5b26","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777831774469-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=연화/99/남/D 형식은 규칙에 맞습니다. \n카카오톡 닉네임으로 사용하면 됩니다.","input":{"request":{"requestId":"req_c3783067","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"","trigger":"user-message","userMessage":"연화/99/남/D"}},"expected":{}} +{"id":"case_af091d04","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777831748947-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=연화/99/남/D3는 티어가 규칙과 맞지 않습니다. \n티어는 GM, I, B, S, G, P, E, D, M, C, U 하나만 사용해 주세요. 예: 연화/99/남/D","input":{"request":{"requestId":"req_d833b83a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"","trigger":"user-message","userMessage":"연화/99/남/D3"}},"expected":{}} +{"id":"case_e138ce19","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777958026869-0; llmOverall=1.25; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=지금은 카카오톡 1단계가 아직 끝나지 않아서(닉네임 형식 수정 미완료) 디스코드 안내는 진행할 수 없어요. \n닉네임을 먼저 바꿔서 1단계를 완료하면 디스코드 쪽 안내해드릴게요.","input":{"request":{"requestId":"req_92962005","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"","trigger":"user-message","userMessage":"디스코드"}},"expected":{}} +{"id":"case_361b1834","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906129199-0; llmOverall=1.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임에서 안 되는 이유는 마지막 티어가 소문자 e라서예요. \n티어 코드는 대문자만 허용되니 .../여/E처럼 마지막 부분만 대문자 E로 바꿔주시면 됩니다.","input":{"request":{"requestId":"req_9b857f60","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_729b540d","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777958095517-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=좋아요, 재롱잉/99/남/M은 카카오톡 닉네임 형식에 맞아요. 카카오톡 1단계(닉네임 변경)는 완료됐어요.","input":{"request":{"requestId":"req_9e72e7c1","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"","trigger":"user-message","userMessage":"재롱잉/99/남/M"}},"expected":{}} +{"id":"case_c205d59f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906018385-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_a52d560b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"","trigger":"user-message","userMessage":"롤닉적는건가요?"}},"expected":{}} +{"id":"case_cb61358e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777886150774-0; llmOverall=1.5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요! 요청하신 형식대로 JSON 객체만 다시 출력했어요","input":{"request":{"requestId":"req_23d248e5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_9a2e13c5","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_7db8c4dd","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777886215575-0; llmOverall=1.75; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코는 지금 바로 진행하면 됩니다. 카카오톡 1단계(닉네임 변경/하트 인증)는 이미 끝났고, 이제 디스코드로 넘어가시면 돼요. \n현재는 본인인증방/구인방 입장과 디스코드 서버 프로필 설정, 그리고 주민신청 채널에 메시지 남기기가 남아 있어요.","input":{"request":{"requestId":"req_5e290120","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_9a2e13c5","nickname":"","trigger":"user-message","userMessage":"디코는"}},"expected":{}} +{"id":"case_da5eb130","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906410144-0; llmOverall=2; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=그건 닉네임 뒤에 붙는 티어 코드야. \n티어 코드는 GM, I, B, S, G, P, E, D, M, C, U 중 하나만 대문자로 쓰면 돼. \n지금은 뒤의 e 대신 E처럼 대문자로 바꿔주면 해결돼.","input":{"request":{"requestId":"req_4e5c8c85","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"","trigger":"user-message","userMessage":"그게몬가여"}},"expected":{}} +{"id":"case_27613760","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906039837-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=닉네임 길이 자체는 괜찮아요. 다만 형식은 꼭 '이름/출생 연도 두 자리/성별/티어' 4칸으로 맞춰야 해요. \n이름이 길면 줄여서 입력해도 됩니다.","input":{"request":{"requestId":"req_ade6fcdf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"","trigger":"user-message","userMessage":"닉네임이 엄청긴데.."}},"expected":{}} +{"id":"case_9bf50f92","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906011838-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=@1111님 닉네임을 조금만 수정해 주세요!\n'/'로 구분된 4개 항목(이름/나이/성별/티어)이 필요해요. 현재 1개입니다.","input":{"request":{"requestId":"req_b87c7d1a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_3be52429","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777887243301-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코 신청해주신 것 확인했어요. \n지금은 남은 항목이 있어서 아직 전체 등록이 마무리되지 않았습니다. \n남은 항목: 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널 메시지 남기기","input":{"request":{"requestId":"req_b7c0c89d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_9a2e13c5","nickname":"","trigger":"user-message","userMessage":"디코 신청 했습니다"}},"expected":{}} +{"id":"case_ce59eef5","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777958037609-0; llmOverall=n/a; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 변경한 카카오톡 닉네임을 보내주시면 바로 확인할게요.","input":{"request":{"requestId":"req_e32dd182","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"","trigger":"user-message","userMessage":"넣어놨는데"}},"expected":{}} +{"id":"case_c0d8c07b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777957698916-0; llmOverall=3.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=재롱잉#0606은 카카오톡 닉네임 형식이 아니에요. \n#는 형식에 맞지 않고, 연도/성별/티어 항목이 비어 있어요. \n카카오톡 닉네임의 '재롱잉' 뒤에 연도(2자리)/성별/티어를 순서대로 써서 고치면 됩니다.","input":{"request":{"requestId":"req_9a0ce4a3","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_9a324a7b","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980725744-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=이제 카카오톡 본인인증방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 남아 있어요.","input":{"request":{"requestId":"req_7cb8c7b8","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"","trigger":"user-message","userMessage":"방금 들어가긴 했는데 된건가여"}},"expected":{}} +{"id":"case_ef5a5a39","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980147468-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=현재 남은 건 카카오톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 그리고 디스코드 주민신청 채널에 메시지 남기기야.","input":{"request":{"requestId":"req_15b6a97b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"","trigger":"user-message","userMessage":"정독 후 변경 완료 했는데 혹시 제가 놓친 부분이 있을까여"}},"expected":{}} +{"id":"case_49c7d986","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777976015637-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기예요.","input":{"request":{"requestId":"req_b22f9f8a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_35175df8","nickname":"","trigger":"user-message","userMessage":"안녕하세용 자랭 한 분 가능하신 사람 있으신가요?? 서폿이 비어서요!!"}},"expected":{}} +{"id":"case_0313331d","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777997633396-0; llmOverall=1.5; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=좋아요, 하트 인증 완료로 확인했어요.","input":{"request":{"requestId":"req_ea94198d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_8dfc83e0","nickname":"","trigger":"user-message","userMessage":"하트 했습니다!"}},"expected":{}} +{"id":"case_057e6219","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777976055251-0; llmOverall=1.5; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디스코드를 안 하면 새내기 등록이 아직 완료되지 않습니다. \n지금 남은 건 카카오톡 본인인증방/구인방 입장하고, 디스코드에서 서버 닉네임 설정과 주민신청 채널 메시지 남기기 두 개라서 필수로 처리해야 해요.","input":{"request":{"requestId":"req_201d5367","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_35175df8","nickname":"","trigger":"user-message","userMessage":"제가 디스코드를 안하는데,,"}},"expected":{}} +{"id":"case_e5b73b18","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778043112369-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=잘하셨어요. 디코 신청까지 진행한 거 확인했어요. 지금 남은 건 카카오톡 본인인증방 입장, 디스코드 서버 닉네임 설정, 주민신청 채널 메시지 남기기예요. 마감 전에 3가지만 빨리 마무리하면 돼요.","input":{"request":{"requestId":"req_ba7b68e0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ef6b0188","nickname":"","trigger":"user-message","userMessage":"디코 신청까지 했습미당🙏🏻"}},"expected":{}} +{"id":"case_f2bb8318","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777993150807-0; llmOverall=2; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 카카오톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기입니다.","input":{"request":{"requestId":"req_7ee35c98","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f2bef29","nickname":"","trigger":"user-message","userMessage":"이렇게 하면 되나용"}},"expected":{}} +{"id":"case_8bf55b0b","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980262565-0; llmOverall=2; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요ㅎㅎ 채팅방 상단 고정 공지에 있는 입장 버튼 누르면 바로 들어가져요.","input":{"request":{"requestId":"req_8ca31700","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"","trigger":"user-message","userMessage":"예?.."}},"expected":{}} +{"id":"case_3b3eeed7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980240472-0; llmOverall=2; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=채팅방 상단 고정 공지에 있는 카카오톡 본인인증방 입장 버튼 누르면 바로 들어갈 수 있어.","input":{"request":{"requestId":"req_957b3c03","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"","trigger":"user-message","userMessage":"카카오톡 본인인증은 어디서 하나여? 처음이라ㅠㅠ"}},"expected":{}} +{"id":"case_11779215","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777975920326-0; llmOverall=2.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임이 성별/티어 형식을 안 맞춰서 안 됩니다. \n어피치/00/M에서 M은 남 또는 여로 바꿔야 하고, 끝에 티어 한 글자(GM, I, B, S, G, P, E, D, M, C, U)를 추가해야 해요. \n예시: 어피치/00/남/D","input":{"request":{"requestId":"req_f3956f12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_35175df8","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_5e681f63","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777998375356-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 마저 진행해 주세요.","input":{"request":{"requestId":"req_2adf33e7","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_8dfc83e0","nickname":"","trigger":"user-message","userMessage":"넵 알겠습니다"}},"expected":{}} +{"id":"case_e714d976","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777997006108-0; llmOverall=2.75; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 형식이 한 칸 부족해요. \nAsolOX/97/M은 이름/출생연도/성별/티어 순서에서 성별 뒤에 티어가 빠져 있고, M도 성별 표기로는 안 됩니다.\n\n성별을 남 또는 여로 쓰고, 마지막에 티어 한 글자(예: D)를 붙여서 AsolOX/97/남/D처럼 바꾸면 돼요.","input":{"request":{"requestId":"req_9684219b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_8dfc83e0","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_6e1c52ff","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777998239663-0; llmOverall=3; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=네, 다시 시도할 때는 아래만 지켜주세요. \n1) 채팅방 오른쪽 상단 메뉴 → 커버 보기 \n2) 하트를 다시 빨간색으로 변경 \n3) 바로 그 화면을 편집 없이 원본으로 새로 캡처해서 다시 업로드\n\n이렇게 하면 다시 인증 요청으로 처리돼요.","input":{"request":{"requestId":"req_7b09ac12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"","trigger":"photo-failed"}},"expected":{}} +{"id":"case_745fddda","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778043731233-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 입장 잘 마무리하셨어요. 편하게 진행해 주세요.","input":{"request":{"requestId":"req_28d12b61","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ef6b0188","nickname":"","trigger":"user-message","userMessage":"입장절차 끝..!! 잘부탁드려용!!"}},"expected":{}} +{"id":"case_6ad937b9","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778018572746-0; llmOverall=5; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=네, 여기다 올리면 됩니다.","input":{"request":{"requestId":"req_847b39a1","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_b874c5cb","nickname":"","trigger":"user-message","userMessage":"안녕하세요 여기다 올리면 될까요?"}},"expected":{}} +{"id":"case_3d833f9f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998586336-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=건강/99/남/U님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_ea34474f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"","trigger":"user-message","userMessage":"첨보네요"}},"expected":{}} +{"id":"case_dffda8fa","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998570053-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=채팅방 프로필에서 하트를 눌러 빨간색으로 만든 다음, 그 화면이 보이게 원본 스크린샷을 보내주시면 인증돼요.","input":{"request":{"requestId":"req_7268a270","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"","trigger":"user-message","userMessage":"프사도 있는건"}},"expected":{}} +{"id":"case_ee4deca6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998289607-0; llmOverall=5; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=네, 닉네임 인증은 완료됐어요.","input":{"request":{"requestId":"req_42e6f27a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"","trigger":"user-message","userMessage":"저닉네임인증된거맞나요"}},"expected":{}} +{"id":"case_c7a3bcc7","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998231025-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 올릴 수 있어요. 하트 인증은 편집 없이 원본 스크린샷으로 업로드하면 돼요.","input":{"request":{"requestId":"req_de2331dd","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"","trigger":"user-message","userMessage":"사진올림되나요"}},"expected":{}} +{"id":"case_bf5f3496","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998213299-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse={\"message\":\"방 하트 인증은 채팅방 오른쪽 상단 메뉴 버튼에서 '커버 보기'로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.\"}","input":{"request":{"requestId":"req_b1312c5a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"","trigger":"user-message","userMessage":"하트인증은"}},"expected":{}} +{"id":"case_96b3e285","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778059340376-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_7b92ce89","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_38e0f76d","nickname":"","trigger":"user-message","userMessage":"감사합니다"}},"expected":{}} +{"id":"case_a2ec1268","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778118929517-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디스코드 서버프로필 닉네임은 디스코드 안에서 보여지는 이름 형식이에요. 형식은 닉네임#태그/출생연도 두 자리/티어입니다. 예: 할리#프렌치불독/88/D. 카카오톡 형식(성별 포함)과 다릅니다.","input":{"request":{"requestId":"req_fffe630c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"","trigger":"user-message","userMessage":"디코 서버프로필 닉네임이 뭘까요,,,,,,,,"}},"expected":{}} +{"id":"case_1bfe6ee3","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778056450407-0; llmOverall=2; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 방 하트 인증만 남았습니다.","input":{"request":{"requestId":"req_9d77d436","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_e89d3a0d","nickname":"","trigger":"user-message","userMessage":"여기서 듀오 구해두 될까요"}},"expected":{}} +{"id":"case_c16428d0","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778119100551-0; llmOverall=2.25; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요. 계속 천천히 마무리해 보세요.","input":{"request":{"requestId":"req_9ffcfc63","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"","trigger":"user-message","userMessage":"같아요"}},"expected":{}} +{"id":"case_b2997e82","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119474172-0; llmOverall=2.75; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 2개예요. 디스코드 서버 프로필 닉네임 설정하고, 주민신청 채널에 메시지 남기기만 하면 돼요.","input":{"request":{"requestId":"req_51c17a6d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"","trigger":"user-message","userMessage":"방 티어가 대체적으로"}},"expected":{}} +{"id":"case_49fe7b98","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119334116-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=네, 아직 남은 건 \n1) 디스코드 서버 프로필 닉네임 설정 \n2) 주민신청 채널에 메시지 남기기 \n둘 다예요.","input":{"request":{"requestId":"req_d068d781","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"","trigger":"user-message","userMessage":"절차 완료인가요!?"}},"expected":{}} +{"id":"case_d598f1d3","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119088788-0; llmOverall=n/a; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 잘 하셨어요.","input":{"request":{"requestId":"req_0b53acfa","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"","trigger":"user-message","userMessage":"성공한것"}},"expected":{}} +{"id":"case_f47d6cfb","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119000121-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=아니요. 계정정보 별명이 아니라 디스코드 서버에서 보이는 서버 프로필 닉네임을 바꿔야 해요.","input":{"request":{"requestId":"req_58ba3c26","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"","trigger":"user-message","userMessage":"계정정보에서 별명을 바꾸면 되는걸까요?"}},"expected":{}} +{"id":"case_bf6cf546","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778086326681-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=현재 남은 절차는 카카오톡 닉네임 변경과 방 하트 인증 두 가지만 완료하시면 됩니다.","input":{"request":{"requestId":"req_0c7527af","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f728e162","nickname":"","trigger":"user-message","userMessage":"카카오 채널 1초로그인 -> 마이쿠폰함 확인 -> 네이버페이 쿠폰 등록 -> 코드 입력\n\nhttps://place.cashnote.kr/login/referrals/h776U3JtDjT\n\n지금 받을 수 있는 쿠폰이 빠르게 소진되고 있어요. (*남은 수량 : 560장)"}},"expected":{}} +{"id":"case_6101b64c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778086319157-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=다시 찍을 때는 이렇게만 해주면 돼요.\n\n1) 채팅방 오른쪽 위 메뉴 버튼에서 \n2) “커버 보기” 들어가기 \n3) 방 하트를 빨간색으로 바꾸기 \n4) 방금 그 화면이 딱 보이게 원본 캡처본만 올리기 \n(편집, 크롭, 필터, 글자 추가 없이 그대로)","input":{"request":{"requestId":"req_4ad09644","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f728e162","nickname":"","trigger":"photo-failed"}},"expected":{}} +{"id":"case_33c66dba","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778075276814-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=지금 남은 건 딱 세 가지예요. 카카오톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기입니다.","input":{"request":{"requestId":"req_5494a948","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f08f2d3d","nickname":"","trigger":"user-message","userMessage":"주민신청했는데 기다리면 되는 건가요"}},"expected":{}} +{"id":"case_377bc44a","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778074538639-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse={\"message\":\"안녕하세요~\"}","input":{"request":{"requestId":"req_3f66d1f6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f08f2d3d","nickname":"","trigger":"user-message","userMessage":"안녕하세여"}},"expected":{}} +{"id":"case_cff93d4f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778202454309-0; llmOverall=1.75; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코까지 완료된 건 확인했어요. \n아직 카카오톡 닉네임 변경만 남아 있어서 1단계가 끝나지 않았어요. \n닉네임은 '공주야너랑살래/99/남/D'처럼 슬래시 형식으로 수정해 주세요.","input":{"request":{"requestId":"req_35185cf6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7bed5e81","nickname":"","trigger":"user-message","userMessage":"디코까지 완료했습니다~"}},"expected":{}} +{"id":"case_030c1b63","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778202545113-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=형식이 조금만 달라도 반영이 안 될 수 있어요. \n카톡 닉네임은 이름/출생연도 두 자리/성별/티어, 티어는 한 글자만 가능해서 형식 오차 시 미반영돼요.","input":{"request":{"requestId":"req_e288dc78","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7bed5e81","nickname":"","trigger":"user-message","userMessage":"닉변도 양식대로 변경했는데 아직 안바뀐거로 나오나요..?"}},"expected":{}} +{"id":"case_583e4bf2","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778154690883-0; llmOverall=2; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 3개예요. \n카톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기입니다.","input":{"request":{"requestId":"req_e2a2a3e2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_13eeeed8","nickname":"","trigger":"user-message","userMessage":"됐늘까요"}},"expected":{}} +{"id":"case_7fd8f41e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778154403955-0; llmOverall=2.75; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임이 안 되는 이유는 형식이 맞지 않아서예요. \n현재는 \"/\"가 없어 4개 항목이 한 번도 분리되지 않았거든요.\n\n고칠 부분은 닉네임 형식만 맞추면 됩니다:\n이름 / 출생 연도 두 자리 / 성별 / 티어 \n(성별: 남 또는 여 / 티어: GM, I, B, S, G, P, E, D, M, C, U 중 1개)","input":{"request":{"requestId":"req_9ef80785","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_13eeeed8","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_e29b7e22","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778301741564-0; llmOverall=1.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=문제는 티어 부분이 소문자 p라서 안 되는 거예요. 티어는 지정된 코드의 대문자여야 해서 p를 P로 고치면 됩니다.\n티어 | p를 P로 | akaps/92/남/P","input":{"request":{"requestId":"req_5370e799","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3150904d","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_8c1af121","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778260759554-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 그렇게 해주시면 돼요.","input":{"request":{"requestId":"req_0e99bbb6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"","trigger":"user-message","userMessage":"아아 넵 감사합니다"}},"expected":{}} +{"id":"case_a9637997","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778260901486-0; llmOverall=2; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=아니요, 별도 양식은 따로 없어요. 방 하트 인증은 하트 화면 원본 캡처만 올리면 됩니다.","input":{"request":{"requestId":"req_57e5ef12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"","trigger":"user-message","userMessage":"양식이 따로 있나요?"}},"expected":{}} +{"id":"case_b6f5a13e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778240845275-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=형식은 닉네임/출생 연도 두 자리/성별/티어(한 글자)예요.\n현재 닉네임은 두 번째 칸이 연도 자리에 '별동대'이고, 성별과 티어 위치가 맞지 않아요.\n두 번째 칸을 2자리 출생연도로 바꾸고 마지막에 티어를 한 글자 추가하면 됩니다.","input":{"request":{"requestId":"req_076e8a92","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_de12d27e","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_87b58a6a","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778236553292-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 맞아요. \n방 하트 인증은 채팅방 오른쪽 상단의 \"커버 보기\"에서 하트를 빨간색으로 바꾼 뒤, 편집 없는 원본 화면을 스크린샷으로 보내면 돼요.","input":{"request":{"requestId":"req_0a071eba","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_27efff54","nickname":"","trigger":"user-message","userMessage":"인증 이러케하면 되나용?"}},"expected":{}} +{"id":"case_a55d7708","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778260808728-0; llmOverall=2.25; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 천천히 진행해 주세요.","input":{"request":{"requestId":"req_9aa4d55f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"","trigger":"user-message","userMessage":"감사합니다 몰랐어요"}},"expected":{}} +{"id":"case_f3191b8e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778260885661-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_b99c1bfd","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"","trigger":"user-message","userMessage":"아아 넵 알겠습니다"}},"expected":{}} +{"id":"case_07b7087f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778309150052-0; llmOverall=2.5; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 방 하트 인증뿐입니다.","input":{"request":{"requestId":"req_81fc82d0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bda05adb","nickname":"","trigger":"user-message","userMessage":"될까요?"}},"expected":{}} +{"id":"case_20b46c53","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778309136760-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 맞아요. \n채팅방 오른쪽 상단 메뉴에서 커버 보기 들어가서 하트를 빨간색으로 바꾼 뒤, 그 화면을 편집 없이 원본으로 스크린샷 찍어 업로드해 주세요.","input":{"request":{"requestId":"req_4933bc72","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bda05adb","nickname":"","trigger":"user-message","userMessage":"방 하트 인증은 사진 보내면"}},"expected":{}} +{"id":"case_d334d6dd","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778260937318-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_ca9a5bd2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"","trigger":"user-message","userMessage":"넵"}},"expected":{}} +{"id":"case_a4345a1f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778255244619-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임에서 안 되는 부분은 성별과 티어 사이 구분자 /가 빠진 점이에요. \n스카피/98/남D → 스카피/98/남/D로 바꿔주시면 돼요.","input":{"request":{"requestId":"req_2507bb99","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_d8b4617f","nickname":"","trigger":"nickname-invalid"}},"expected":{}} +{"id":"case_401941a0","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778350051338-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=네, 지금이라도 마감 전이면 가능합니다. 마감 전까지만 가능하니 지금 바로 진행해 주세요.","input":{"request":{"requestId":"req_f59d7357","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"","trigger":"user-message","userMessage":"률중인데,,"}},"expected":{}} +{"id":"case_251723ce","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778322156175-0; llmOverall=1.5; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=좋아요, 지금 남은 건 카카오톡 본인인증방 입장뿐이에요.","input":{"request":{"requestId":"req_073f9f3b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"","trigger":"user-message","userMessage":"주민신청,단통방하트인증,구인방입장 완료햇습니다"}},"expected":{}} +{"id":"case_0382016f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778320965167-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 천천히 진행해 주세요.","input":{"request":{"requestId":"req_28d990e5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"","trigger":"user-message","userMessage":"감사합니다"}},"expected":{}} +{"id":"case_52c3e5d8","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778316434893-0; llmOverall=1.75; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=조아재/92/남/E님 이제 추가 절차만 남았어요. 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기을 확인해 주세요.","input":{"request":{"requestId":"req_b0351ff3","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_04deef9f","nickname":"","trigger":"user-message","userMessage":"디스코드는 조금 기다려야겠죠?"}},"expected":{}} +{"id":"case_a03ff5a9","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778322017346-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코 승인은 디스코드 주민신청 채널에 메시지를 남겨야 신청이 접수돼요. \n현재는 카카오톡 1단계(닉네임/하트)가 끝났으니 여기서 바로 진행하시면 됩니다.","input":{"request":{"requestId":"req_b76d0382","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"","trigger":"user-message","userMessage":"관리자님~ 디코 승인부탁드립니다"}},"expected":{}} +{"id":"case_33f24d6f","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778335070993-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 천천히 진행해 주세요.","input":{"request":{"requestId":"req_6948769b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_961ebb8c","nickname":"","trigger":"user-message","userMessage":"넵, 감사합니다."}},"expected":{}} +{"id":"case_4f8acb72","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778350065869-0; llmOverall=3; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 확인했어요.","input":{"request":{"requestId":"req_2adcf9cb","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"","trigger":"user-message","userMessage":"네ㅔㅔ"}},"expected":{}} +{"id":"case_84729b2b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778392723529-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_77df97b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_5822e361","nickname":"","trigger":"user-message","userMessage":"안녕하세요~"}},"expected":{}} +{"id":"case_b27986e1","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778350033557-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=죄송하지만 지금은 마감이 임박해 있어요. 조금이라도 빨리 처리해 주세요. 마감 전까지만 가능합니다.","input":{"request":{"requestId":"req_626ca48c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"","trigger":"user-message","userMessage":"죄송한데 조금만 이ㅛ다가 해도더ㅣ냐요?"}},"expected":{}} +{"id":"case_8270994b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778348438021-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_518fd7a5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"","trigger":"user-message","userMessage":"네넨"}},"expected":{}} +{"id":"case_32db0b90","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778348422100-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=아니요, 롤 닉네임이 아니라 카카오톡 닉네임을 이름/출생 연도 두 자리/성별/티어 형식으로 등록해야 해요.","input":{"request":{"requestId":"req_11cd0f34","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"","trigger":"user-message","userMessage":"닉네임은 롤 닉네임가여?"}},"expected":{}} +{"id":"case_79d82387","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778348415532-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_abd7b438","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"","trigger":"user-message","userMessage":"안녕하세영"}},"expected":{}} +{"id":"case_c810958e","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778334757877-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_6a9f1a7b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_961ebb8c","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_d368dec6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778322598506-0; llmOverall=3.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 확인했어요. 지금은 다 끝난 상태로 보여요.","input":{"request":{"requestId":"req_7f7d49de","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"","trigger":"user-message","userMessage":"주민신청,단통방하트인증,구인방입장,본인인증 완료햇습니다"}},"expected":{}} +{"id":"case_f06874cb","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778447250218-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_f2313e98","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"","trigger":"user-message","userMessage":"Okay Mr, it was pleasure to see you here"}},"expected":{}} +{"id":"case_df73a771","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778446712145-0; llmOverall=2.25; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요. 이곳은 롤림푸스 새내기 등록을 돕는 안내 채팅입니다.","input":{"request":{"requestId":"req_a269699a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"","trigger":"user-message","userMessage":"Yeah I guess"}},"expected":{}} +{"id":"case_41173969","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778482229470-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse={\"message\":\"안녕하세요~\"}","input":{"request":{"requestId":"req_0ca7be91","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3c700174","nickname":"","trigger":"user-message","userMessage":"안녕하세요~"}},"expected":{}} +{"id":"case_58a651e2","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778477393576-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_99ce06e3","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_16262d3a","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_6b6ad52a","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778477202451-0; llmOverall=5; derivedIntent=age-question; derivedPhase=deadline-near; assistantResponse=네, 딱 27살도 가능합니다. 롤림푸스는 27세 이상만 가입이 가능해요.","input":{"request":{"requestId":"req_ed27fe73","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_16262d3a","nickname":"","trigger":"user-message","userMessage":"안녕하세요 혹시 딱 27살도 가능할까요?"}},"expected":{}} +{"id":"case_87896362","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778464458764-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_c8b6b29c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_1f854d18","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} +{"id":"case_95b15072","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778447091058-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=아니에요, 괜찮아요.","input":{"request":{"requestId":"req_6f5881ac","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"","trigger":"user-message","userMessage":"Oh sorry sorry"}},"expected":{}} +{"id":"case_647fb0f0","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778446903996-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_54d027f4","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"","trigger":"user-message","userMessage":"Hi everyone"}},"expected":{}} +{"id":"case_2459a879","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778501754586-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=아직은 어렵습니다. 디스코드 참여는 카카오톡 1단계가 먼저 완료되어야 가능해요. \n현재는 방 하트 인증이 미완료라 지금은 디코방 이용이 어려워요.","input":{"request":{"requestId":"req_ad497829","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"","trigger":"user-message","userMessage":"혹시 디코방 참여해서 겜 같이해도되나요?"}},"expected":{}} +{"id":"case_f7818487","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778501905206-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_76d6f9da","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"","trigger":"user-message","userMessage":"넵"}},"expected":{}} +{"id":"case_fe74c37e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778501809468-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_65762501","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"","trigger":"user-message","userMessage":"넵"}},"expected":{}} +{"id":"case_3a30d20f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778502401994-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 완료 확인했습니다.","input":{"request":{"requestId":"req_b8dce22e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"","trigger":"user-message","userMessage":"완료요!"}},"expected":{}} +{"id":"case_b1e0dff1","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778501699301-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_d42492fe","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}} diff --git a/src/eval/golden/onboarding.capture.jsonl b/src/eval/golden/onboarding.capture.jsonl new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/eval/golden/onboarding.capture.jsonl @@ -0,0 +1 @@ + diff --git a/src/eval/golden/onboarding.jsonl b/src/eval/golden/onboarding.jsonl index 368a86c..3f21ab7 100644 --- a/src/eval/golden/onboarding.jsonl +++ b/src/eval/golden/onboarding.jsonl @@ -1,3 +1,11 @@ -{"id":"greeting-short","name":"Greeting should stay short and welcoming","input":{"request":{"requestId":"eval_greeting_1","skill":"onboarding","chatroomId":"room_eval_1","userId":"user_eval_1","nickname":"","trigger":"user-message","userMessage":"안녕하세요"},"session":null,"conversationHistory":[]},"expected":{"intent":"greeting","responseMode":"llm","mustIncludeAny":["환영","반갑"],"mustNotInclude":["디스코드"]}} -{"id":"status-photo-remaining","name":"Status question should answer only the remaining step","input":{"request":{"requestId":"eval_status_1","skill":"onboarding","chatroomId":"room_eval_2","userId":"user_eval_2","nickname":"프리리/95/여/D","trigger":"user-message","userMessage":"이제 뭐 남았지"},"session":{"chatroomId":"room_eval_2","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","kakaotalkNickname":"프리리/95/여/D","kakaotalkNicknameVerified":true,"lastEventAt":"2026-04-16T00:10:00.000Z","photoVerified":false,"status":"pending","userId":"user_eval_2"},"conversationHistory":[{"role":"user","content":"안녕하세요","timestamp":"2026-04-16T00:00:00.000Z"},{"role":"assistant","content":"환영합니다! 반갑습니다.","timestamp":"2026-04-16T00:00:01.000Z"}]},"expected":{"intent":"status-question","responseMode":"llm","mustIncludeAny":["남았","하트"],"mustNotInclude":["디스코드"]}} -{"id":"nickname-invalid-tier","name":"Nickname invalid flow should focus on nickname correction","input":{"request":{"requestId":"eval_nick_1","skill":"onboarding","chatroomId":"room_eval_3","userId":"user_eval_3","nickname":"프리리/95/여/다이아","trigger":"nickname-invalid"},"session":{"chatroomId":"room_eval_3","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","kakaotalkNickname":"프리리/95/여/다이아","kakaotalkNicknameVerified":false,"lastEventAt":"2026-04-16T00:10:00.000Z","photoVerified":false,"status":"pending","userId":"user_eval_3"},"conversationHistory":[]},"expected":{"intent":"nickname-question","responseMode":"llm-with-tools","mustIncludeAny":["닉네임","티어","형식"],"mustNotInclude":["디스코드"]}} + +{"id":"case_119e66ab","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777693377499-0; llmOverall=1.5; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=카카오톡 닉네임은 이름/출생 연도 두 자리/성별/티어 형식이에요. 예를 들면 면정학/89/남/D처럼 쓰시면 돼요.","input":{"request":{"requestId":"req_7e7337de","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"닉네임 변경 및 하트 인증은 하였는데, 혹시 어느 부분에서 문제가 있는지 여쭈어 볼 수 있을까요?"}},"expected":{}} +{"id":"case_6bc9858f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777692194987-0; llmOverall=1.5; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=공지방은 따로 운영하지 않습니다.","input":{"request":{"requestId":"req_c91897cf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"죄송합니다. 제가 기억하기론 공지방이 따로 있던 것으로 기억하는데, 공지방은 따로 없는지 여쭈어 볼 수 있을까요?"}},"expected":{}} +{"id":"case_edbec2e7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777678836132-0; llmOverall=2; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=디스코드의 주민신청 채널에서 멘션 남기면 됩니다.","input":{"request":{"requestId":"req_e69bfe09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_2096a462","nickname":"","trigger":"user-message","userMessage":"디코 공지 확인이랑 닉변까지 다 했는데 어디에서 멘션하면 되나여"}},"expected":{}} +{"id":"case_756da19b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693260088-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=아, 그러셨군요. 지금은 카카오톡 닉네임 변경만 남아 있어서 그 과정에서 막히신 거죠. \n무슨 부분이 어려우신가요? 어떤 과정에서 막히셨는지 알려 주세요.","input":{"request":{"requestId":"req_f09cd933","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"하는데"}},"expected":{}} +{"id":"case_70b3db02","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693242995-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=네, 배치 보려면 먼저 카카오톡 1단계부터 끝내야 해요. 현재는 닉네임 변경만 남아 있고, 마감이 임박했으니 지금 바로 닉네임만 먼저 완료해 주세요.","input":{"request":{"requestId":"req_b68c3a89","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"얼른 배치를 봐야만"}},"expected":{}} +{"id":"case_d8efa98f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777692242389-0; llmOverall=3.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 맞습니다. 구인글은 구인방에서만 올리는 게 원칙이에요.","input":{"request":{"requestId":"req_027cab09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"user-message","userMessage":"감사합니다.\n구인방에서만 구인글을 게시하는지 마지막으로 여쭈어 볼 수 있을까요?"}},"expected":{}} +{"id":"case_b93a3eac","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777691863103-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=하트 인증은 이렇게만 다시 보내면 돼요.\n\n채팅방 오른쪽 상단 메뉴 버튼 → ‘커버 보기’ 들어가기 \n하트를 빨간색으로 바꾸기 \n그 상태를 편집 없이 원본 그대로 캡처해서 사진으로 보내기\n\n자르기나 화면 꾸미기 같은 편집은 하지 말고, 그냥 바로 스크린샷 그대로 보내주세요.","input":{"request":{"requestId":"req_1785e212","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"","trigger":"photo-failed"}},"expected":{}} +{"id":"case_4c4e71d6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655309842-0; llmOverall=4.75; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=무슨 말인지 모르겠어용 ㅎㅎ 한 번만 더 정확히 말해줄 수 있어임니당?","input":{"request":{"requestId":"req_1147b077","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"","trigger":"user-message","userMessage":"머징"}},"expected":{}} +{"id":"case_b3ba68ec","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655250210-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=오키, 점수 200점 확인했어용! 잘 적어둘게요.","input":{"request":{"requestId":"req_1f66afdc","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"","trigger":"user-message","userMessage":"200점수따리라"}},"expected":{}} +{"id":"case_475ee343","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777649519625-0; llmOverall=4.75; derivedIntent=acknowledgement; derivedPhase=completed; assistantResponse=네, 다 잘 끝났으니까 푹 쉬었다 다녀오세요~","input":{"request":{"requestId":"req_11add9b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57a7acdc","nickname":"","trigger":"user-message","userMessage":"가겠습니당~~"}},"expected":{}} diff --git a/src/eval/judges/process-judge.test.ts b/src/eval/judges/process-judge.test.ts new file mode 100644 index 0000000..39d17a1 --- /dev/null +++ b/src/eval/judges/process-judge.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it } from "vitest"; + +import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; +import { judgeProcessRule } from "./process-judge.js"; + +function makeTrace( + overrides?: Partial, +): ProcessTraceSnapshot { + return { + steps: { + classify: { + name: "classify", + startedAtMs: 1, + durationMs: 1, + output: { + intent: "status-question", + confidence: 0.9, + source: "regex", + }, + }, + derive: { + name: "derive", + startedAtMs: 2, + durationMs: 1, + output: { + phase: "photo-pending", + responseMode: "llm-with-tools", + shouldRespond: true, + tier: 1, + }, + }, + plan: { + name: "plan", + startedAtMs: 3, + durationMs: 1, + output: { + approach: "validate-then-explain", + mustAvoid: [], + mustInclude: ["남은 절차"], + suggestedTools: ["validate-nickname"], + targetLengthChars: 120, + }, + }, + toolCalls: [ + { + name: "tool-call", + startedAtMs: 4, + durationMs: 1, + output: { + name: "validate-nickname", + args: { + nickname: "프리리/95/여/D", + }, + ok: true, + resultSummary: '{"valid":true}', + }, + }, + ], + respond: { + name: "respond", + startedAtMs: 5, + durationMs: 1, + output: { + draftText: "방 하트 인증만 남았어요.", + tokensIn: 0, + tokensOut: 0, + }, + }, + validate: { + name: "validate", + startedAtMs: 6, + durationMs: 1, + output: { + status: "pass", + reasons: [], + }, + }, + ...overrides, + }, + }; +} + +describe("judgeProcessRule", () => { + it("passes active process expectations", () => { + const result = judgeProcessRule( + { + intent: "status-question", + processExpected: { + expectedApproach: "validate-then-explain", + expectedTools: ["validate-nickname"], + minSteps: 5, + }, + }, + makeTrace(), + ); + + expect(result.score).toBe(1); + expect(result.totalCount).toBeGreaterThan(0); + expect(result.checks.every((check) => check.skipped || check.passed)).toBe( + true, + ); + }); + + it("fails intent, tool, and approach mismatches", () => { + const result = judgeProcessRule( + { + intent: "photo-question", + processExpected: { + expectedApproach: "answer-direct", + expectedTools: ["check-onboarding-progress"], + toolMode: "exact", + }, + }, + makeTrace(), + ); + + expect( + result.checks.find((check) => check.rule === "intent-match")?.passed, + ).toBe(false); + expect( + result.checks.find((check) => check.rule === "planner-approach") + ?.passed, + ).toBe(false); + expect( + result.checks.find((check) => check.rule === "tools-selected-correctly") + ?.passed, + ).toBe(false); + }); + + it("fails invalid tool arguments", () => { + const result = judgeProcessRule( + {}, + makeTrace({ + toolCalls: [ + { + name: "tool-call", + startedAtMs: 4, + durationMs: 1, + output: { + name: "validate-nickname", + args: {}, + ok: false, + resultSummary: "invalid", + }, + }, + ], + }), + ); + + expect( + result.checks.find((check) => check.rule === "tool-args-valid")?.passed, + ).toBe(false); + }); + + it("fails tier and validator process regressions", () => { + const result = judgeProcessRule( + {}, + makeTrace({ + derive: { + name: "derive", + startedAtMs: 2, + durationMs: 1, + output: { + phase: "photo-pending", + responseMode: "llm-with-tools", + shouldRespond: true, + tier: 0, + }, + }, + validate: { + name: "validate", + startedAtMs: 6, + durationMs: 1, + output: { + status: "reject", + reasons: ["stateConsistency"], + }, + }, + }), + ); + + expect( + result.checks.find((check) => check.rule === "tier-appropriate")?.passed, + ).toBe(false); + expect( + result.checks.find((check) => check.rule === "validate-no-reject") + ?.passed, + ).toBe(false); + }); + + it("skips optional expectations when a golden case has not opted in", () => { + const result = judgeProcessRule({}, makeTrace()); + + expect( + result.checks.find((check) => check.rule === "intent-match")?.skipped, + ).toBe(true); + expect( + result.checks.find((check) => check.rule === "planner-approach") + ?.skipped, + ).toBe(true); + }); +}); diff --git a/src/eval/judges/process-judge.ts b/src/eval/judges/process-judge.ts new file mode 100644 index 0000000..1cdafcc --- /dev/null +++ b/src/eval/judges/process-judge.ts @@ -0,0 +1,353 @@ +import { z } from "zod"; + +import type { AgentRequest } from "@agent/types"; +import type { LlmClient } from "@llm/llm-client"; +import { createBuiltInToolRegistry } from "@tools/builtins/index"; +import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; +import { + getEvalProcessLlmJudgeMaxTokens, + getLlmJudgeModel, +} from "../../config.js"; +import type { + EvalCaseExpected, + ProcessJudgeCheck, + ProcessJudgeResult, +} from "../types.js"; + +const processLlmJudgeSchema = z.object({ + rationale: z.string().trim().min(1).max(300), + score: z.number().min(0).max(1), +}); + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +function makeCheck( + step: string, + rule: ProcessJudgeCheck["rule"], + passed: boolean, + detail?: string, +): ProcessJudgeCheck { + return { + step, + rule, + passed, + skipped: false, + detail, + }; +} + +function makeSkippedCheck( + step: string, + rule: ProcessJudgeCheck["rule"], + detail: string, +): ProcessJudgeCheck { + return { + step, + rule, + passed: true, + skipped: true, + detail, + }; +} + +function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudgeResult { + const activeChecks = checks.filter((check) => !check.skipped); + const passedCount = activeChecks.filter((check) => check.passed).length; + const totalCount = activeChecks.length; + const result: ProcessJudgeResult = { + checks, + passedCount, + score: totalCount === 0 ? 1 : round(passedCount / totalCount), + totalCount, + }; + + if (rationale) { + result.rationale = rationale; + } + + return result; +} + +function hasAllTools( + actualTools: string[], + expectedTools: string[], + mode: "exact" | "subset", +): boolean { + const actual = new Set(actualTools); + const expected = new Set(expectedTools); + const subset = expectedTools.every((tool) => actual.has(tool)); + + if (mode === "subset") { + return subset; + } + + return subset && actual.size === expected.size; +} + +function countRecordedSteps(trace: ProcessTraceSnapshot): number { + const scalarSteps = [ + trace.steps.classify, + trace.steps.derive, + trace.steps.plan, + trace.steps.respond, + trace.steps.critic, + trace.steps.validate, + ].filter(Boolean).length; + + return scalarSteps + (trace.steps.toolCalls?.length ?? 0); +} + +function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck { + const toolCalls = trace.steps.toolCalls ?? []; + if (toolCalls.length === 0) { + return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls"); + } + + const registry = createBuiltInToolRegistry(); + const failures: string[] = []; + + for (const call of toolCalls) { + try { + registry.get(call.output.name).argsSchema.parse(call.output.args); + } catch { + failures.push(call.output.name); + } + } + + return makeCheck( + "tool-call", + "tool-args-valid", + failures.length === 0, + failures.length === 0 + ? undefined + : `invalid args for: ${failures.join(", ")}`, + ); +} + +export function judgeProcessRule( + expected: EvalCaseExpected, + trace: ProcessTraceSnapshot, +): ProcessJudgeResult { + const checks: ProcessJudgeCheck[] = []; + + if (expected.intent) { + const actualIntent = trace.steps.classify?.output.intent; + checks.push( + makeCheck( + "classify", + "intent-match", + actualIntent === expected.intent, + `expected=${expected.intent}, actual=${actualIntent ?? "missing"}`, + ), + ); + } else { + checks.push(makeSkippedCheck("classify", "intent-match", "no expected intent")); + } + + const derived = trace.steps.derive?.output; + if (derived?.responseMode === "llm-with-tools") { + checks.push( + makeCheck( + "derive", + "tier-appropriate", + derived.tier >= 1, + `responseMode=${derived.responseMode}, tier=${derived.tier}`, + ), + ); + } else { + checks.push( + makeSkippedCheck( + "derive", + "tier-appropriate", + "responseMode is not llm-with-tools", + ), + ); + } + + const processExpected = expected.processExpected; + if (processExpected?.expectedApproach) { + const actualApproach = trace.steps.plan?.output.approach; + checks.push( + makeCheck( + "plan", + "planner-approach", + actualApproach === processExpected.expectedApproach, + `expected=${processExpected.expectedApproach}, actual=${actualApproach ?? "missing"}`, + ), + ); + } else { + checks.push( + makeSkippedCheck("plan", "planner-approach", "no expected approach"), + ); + } + + if (processExpected?.expectedTools?.length) { + const actualTools = + trace.steps.toolCalls?.map((call) => call.output.name) ?? []; + const mode = processExpected.toolMode ?? "subset"; + checks.push( + makeCheck( + "tool-call", + "tools-selected-correctly", + hasAllTools(actualTools, processExpected.expectedTools, mode), + `expected=${processExpected.expectedTools.join(",")}, actual=${actualTools.join(",")}, mode=${mode}`, + ), + ); + } else { + checks.push( + makeSkippedCheck( + "tool-call", + "tools-selected-correctly", + "no expected tools", + ), + ); + } + + checks.push(validateToolArgs(trace)); + + if (processExpected?.minSteps !== undefined) { + const actualSteps = countRecordedSteps(trace); + checks.push( + makeCheck( + "trace", + "min-steps", + actualSteps >= processExpected.minSteps, + `expected>=${processExpected.minSteps}, actual=${actualSteps}`, + ), + ); + } else { + checks.push(makeSkippedCheck("trace", "min-steps", "no minSteps")); + } + + const critic = trace.steps.critic?.output; + if (critic && derived?.tier === 2) { + const validationStatus = trace.steps.validate?.output.status; + checks.push( + makeCheck( + "critic", + "critic-decision-justified", + critic.decision !== "keep" || validationStatus !== "reject", + `decision=${critic.decision}, validate=${validationStatus ?? "missing"}`, + ), + ); + } else { + checks.push( + makeSkippedCheck( + "critic", + "critic-decision-justified", + "no tier 2 critic decision", + ), + ); + } + + const validate = trace.steps.validate?.output; + if (validate) { + checks.push( + makeCheck( + "validate", + "validate-no-reject", + validate.status !== "reject", + `status=${validate.status}`, + ), + ); + } else { + checks.push( + makeCheck("validate", "validate-no-reject", false, "missing validate step"), + ); + } + + return finalize(checks); +} + +function buildProcessJudgePrompt( + trace: ProcessTraceSnapshot, + options: { finalText: string; request: AgentRequest }, +): string { + return [ + "온보딩 에이전트의 단계별 실행 trace를 평가해 주세요.", + "planner 선택, tool 선택/인자, critic 판단, 최종 검증 흐름이 합리적인지 보세요.", + "score는 0부터 1 사이 숫자입니다.", + "", + "[request]", + JSON.stringify(options.request), + "", + "[trace]", + JSON.stringify(trace), + "", + "[finalText]", + options.finalText, + "", + 'JSON object only: {"score":0.8,"rationale":"..."}', + ].join("\n"); +} + +export async function judgeProcessWithLlm( + llmClient: LlmClient, + trace: ProcessTraceSnapshot, + options: { finalText: string; request: AgentRequest }, +): Promise { + if (!llmClient.hasEndpoints()) { + return finalize( + [ + makeSkippedCheck( + "trace", + "validate-no-reject", + "no LLM endpoint for process judge", + ), + ], + "skipped: no LLM endpoint", + ); + } + + const response = await llmClient.chatCompletion( + [ + { + role: "system", + content: + "당신은 온보딩 에이전트의 과정 품질 평가자입니다. 반드시 JSON object만 출력하세요.", + }, + { + role: "user", + content: buildProcessJudgePrompt(trace, options), + }, + ], + { + jsonMode: true, + maxTokens: getEvalProcessLlmJudgeMaxTokens(), + modelOverride: getLlmJudgeModel(), + role: "critic", + temperature: 0, + tier: 2, + }, + ); + + if (!response) { + return finalize( + [makeCheck("trace", "validate-no-reject", false, "judge call failed")], + "process judge call failed", + ); + } + + try { + const parsed = processLlmJudgeSchema.parse(JSON.parse(response.content)); + const passed = parsed.score >= 0.7; + return finalize( + [ + makeCheck( + "trace", + "validate-no-reject", + passed, + `llmScore=${round(parsed.score)}`, + ), + ], + parsed.rationale, + ); + } catch { + return finalize( + [makeCheck("trace", "validate-no-reject", false, "invalid judge json")], + "invalid process judge json", + ); + } +} diff --git a/src/eval/loop/elo.test.ts b/src/eval/loop/elo.test.ts new file mode 100644 index 0000000..62ea448 --- /dev/null +++ b/src/eval/loop/elo.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; + +import { createEloRating, scoreForVerdict, updateEloPair } from "./elo.js"; + +describe("elo", () => { + it("updates equal ratings by the standard K=32 margin", () => { + const control = createEloRating("control"); + const variant = createEloRating("variant"); + + const updated = updateEloPair(control, variant, 1); + + expect(updated.ratingA.elo).toBe(1516); + expect(updated.ratingB.elo).toBe(1484); + expect(updated.ratingA.matches).toBe(1); + expect(updated.ratingA.wins).toBe(1); + }); + + it("maps side-by-side verdicts to scores", () => { + expect(scoreForVerdict("A")).toBe(1); + expect(scoreForVerdict("B")).toBe(0); + expect(scoreForVerdict("tie")).toBe(0.5); + }); +}); diff --git a/src/eval/loop/elo.ts b/src/eval/loop/elo.ts new file mode 100644 index 0000000..c47e7de --- /dev/null +++ b/src/eval/loop/elo.ts @@ -0,0 +1,69 @@ +export interface EloRating { + id: string; + elo: number; + matches: number; + wins: number; +} + +export interface EloUpdate { + ratingA: EloRating; + ratingB: EloRating; +} + +const DEFAULT_ELO = 1_500; +const DEFAULT_K_FACTOR = 32; + +function expectedScore(left: number, right: number): number { + return 1 / (1 + 10 ** ((right - left) / 400)); +} + +function roundElo(value: number): number { + return Number(value.toFixed(3)); +} + +export function createEloRating(id: string): EloRating { + return { + id, + elo: DEFAULT_ELO, + matches: 0, + wins: 0, + }; +} + +export function updateEloPair( + ratingA: EloRating, + ratingB: EloRating, + scoreA: 0 | 0.5 | 1, + kFactor = DEFAULT_K_FACTOR, +): EloUpdate { + const expectedA = expectedScore(ratingA.elo, ratingB.elo); + const scoreB = (1 - scoreA) as 0 | 0.5 | 1; + const expectedB = 1 - expectedA; + + return { + ratingA: { + ...ratingA, + elo: roundElo(ratingA.elo + kFactor * (scoreA - expectedA)), + matches: ratingA.matches + 1, + wins: ratingA.wins + (scoreA === 1 ? 1 : 0), + }, + ratingB: { + ...ratingB, + elo: roundElo(ratingB.elo + kFactor * (scoreB - expectedB)), + matches: ratingB.matches + 1, + wins: ratingB.wins + (scoreB === 1 ? 1 : 0), + }, + }; +} + +export function scoreForVerdict( + verdict: "A" | "B" | "tie", +): 0 | 0.5 | 1 { + if (verdict === "A") { + return 1; + } + if (verdict === "B") { + return 0; + } + return 0.5; +} diff --git a/src/eval/loop/propose.test.ts b/src/eval/loop/propose.test.ts new file mode 100644 index 0000000..d808695 --- /dev/null +++ b/src/eval/loop/propose.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; + +import type { LlmClient } from "@llm/llm-client"; +import type { EvalArchiveEntry } from "../archive.js"; +import { buildCurrentPromptSnapshot, proposeVariants } from "./propose.js"; + +function archiveEntry(): EvalArchiveEntry { + return { + streamId: "1-0", + evaluatedAt: new Date("2026-05-01T00:00:00.000Z"), + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "닉", + trigger: "user-message", + userMessage: "사진 어디로 보내요?", + }, + response: { + requestId: "req-1", + text: "디스코드에 보내주세요.", + confidence: 0.9, + source: "llm", + }, + derived: { + phase: { kind: "photo-pending" }, + intent: { kind: "photo-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + }, + classified: { + intent: { kind: "photo-question" }, + confidence: 0.9, + source: "regex", + }, + validation: { + status: "pass", + text: "디스코드에 보내주세요.", + reasons: [], + }, + llmJudge: { + skipped: false, + model: "judge", + endpoint: "local", + provider: "test", + scores: { + friendliness: 4, + accuracy: 1, + concision: 4, + scope: 2, + overall: 2.75, + }, + rationale: "wrong channel", + raw: "{}", + }, + latencyMs: 100, + tier: 0, + }; +} + +describe("proposeVariants", () => { + it("parses LLM JSON into stable prompt and tool proposals", async () => { + const llmClient = { + hasEndpoints: () => true, + chatCompletion: () => + Promise.resolve({ + content: JSON.stringify({ + prompts: [ + { + description: "Clarify photo location", + diff: [ + { + layer: "policy", + op: "append", + text: "사진 인증 위치를 디스코드와 혼동하지 마.", + }, + ], + rationale: "low-score cases confuse photo location", + expectedImprovement: "fewer wrong-channel answers", + }, + ], + tools: [ + { + description: "Photo routing helper", + toolName: "lookup-photo-route", + schema: { type: "object" }, + whyNeeded: "Repeated hallucination on photo channel", + }, + ], + }), + endpoint: "mock", + message: { role: "assistant", content: "{}" }, + provider: "cliproxy", + raw: {}, + }), + } as unknown as LlmClient; + + const proposals = await proposeVariants(llmClient, { + slice: { + tier: 0, + intent: "photo-question", + phase: "photo-pending", + }, + recentLabels: [], + worstCases: [archiveEntry()], + currentPromptSnapshot: buildCurrentPromptSnapshot( + { + tier: 0, + intent: "photo-question", + phase: "photo-pending", + }, + archiveEntry(), + ), + }); + + expect(proposals.prompts).toHaveLength(1); + expect(proposals.prompts[0]?.variantId).toMatch(/^[a-f0-9]{8}$/u); + expect(proposals.prompts[0]?.diff[0]).toMatchObject({ + layer: "policy", + op: "append", + }); + expect(proposals.tools[0]).toMatchObject({ + toolName: "lookup-photo-route", + }); + }); +}); diff --git a/src/eval/loop/propose.ts b/src/eval/loop/propose.ts new file mode 100644 index 0000000..a7c1c61 --- /dev/null +++ b/src/eval/loop/propose.ts @@ -0,0 +1,300 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +import { z } from "zod"; +import type { LlmClient } from "@llm/llm-client"; +import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state"; +import type { Blackboard, PromptContext } from "@skills/onboarding/types"; +import { + PromptBuilder, + type PromptLayerDiff, + type PromptLayerName, + type PromptVariantManifest, +} from "@skills/onboarding/prompts/builder"; +import { + getEvalTournamentMinCases, + getLlmProposeMaxTokens, + getLlmProposeModel, +} from "../../config.js"; +import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js"; +import { getEvalArchiveCollection } from "../archive.js"; +import type { SliceKey } from "../aggregator.js"; +import { queryArchivedReviewLabels } from "../review/label-archive.js"; +import type { ReviewLabel } from "../review/queue.js"; + +export interface ProposalInput { + slice: SliceKey; + recentLabels: ReviewLabel[]; + worstCases: EvalArchiveEntry[]; + currentPromptSnapshot: string; +} + +export interface PromptVariantProposal extends PromptVariantManifest { + description: string; + expectedImprovement: string; + rationale: string; +} + +export interface ToolProposal { + description: string; + schema: unknown; + toolName: string; + variantId: string; + whyNeeded: string; +} + +const promptDiffSchema = z.object({ + layer: z.enum([ + "context", + "examples", + "persona", + "policy", + "state", + "tool-policy", + ]), + op: z.enum(["append", "remove", "replace"]), + reason: z.string().trim().max(500).optional(), + text: z.string().trim().max(4_000).optional(), +}); + +const proposalSchema = z.object({ + prompts: z + .array( + z.object({ + description: z.string().trim().min(1).max(300), + diff: z.array(promptDiffSchema).min(1).max(10), + expectedImprovement: z.string().trim().min(1).max(500), + rationale: z.string().trim().min(1).max(500), + }), + ) + .max(5), + tools: z + .array( + z.object({ + description: z.string().trim().min(1).max(300), + schema: z.unknown(), + toolName: z.string().trim().min(1).max(80), + whyNeeded: z.string().trim().min(1).max(500), + }), + ) + .max(5) + .default([]), +}); + +function stableHash(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(value)) + .digest("hex") + .slice(0, 8); +} + +function normalizePromptDiff(diff: z.infer): PromptLayerDiff { + const normalized: PromptLayerDiff = { + layer: diff.layer as PromptLayerName, + op: diff.op, + }; + if (diff.reason !== undefined) { + normalized.reason = diff.reason; + } + if (diff.text !== undefined) { + normalized.text = diff.text; + } + return normalized; +} + +function getLlmOverall(entry: EvalArchiveEntry): number | null { + return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall; +} + +function emptyBlackboard(): Blackboard { + return { + flaggedIssues: [], + openQuestions: [], + recentNicknameAttempts: [], + toneMemory: {}, + }; +} + +function derivedFromSlice(slice: SliceKey): DerivedState { + return { + allowShortResponse: false, + intent: { kind: slice.intent } as Intent, + phase: { kind: slice.phase } as OnboardingPhase, + responseMode: "llm", + shouldRespond: true, + suggestedMaxTokens: 384, + tier: slice.tier, + }; +} + +function defaultPromptContext(): PromptContext { + return { + blackboard: emptyBlackboard(), + conversationHistory: [], + userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.", + }; +} + +export function buildCurrentPromptSnapshot( + slice: SliceKey, + archiveEntry?: EvalArchiveEntry, +): string { + return PromptBuilder.forDerived( + archiveEntry?.derived ?? derivedFromSlice(slice), + null, + defaultPromptContext(), + ).buildSystemPrompt(); +} + +function summarizeArchiveEntry(entry: EvalArchiveEntry): Record { + return { + streamId: entry.streamId, + request: entry.request, + response: entry.response.text, + validation: entry.validation, + llmJudge: entry.llmJudge, + processJudge: entry.processJudge, + process: entry.process, + evaluatedAt: entry.evaluatedAt.toISOString(), + }; +} + +function buildProposalPrompt(input: ProposalInput): string { + return [ + "온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.", + "반드시 JSON object만 출력하세요.", + "prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.", + "과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.", + "", + "[slice]", + JSON.stringify(input.slice), + "", + "[recent_human_labels]", + JSON.stringify(input.recentLabels.slice(0, 20)), + "", + "[worst_cases]", + JSON.stringify(input.worstCases.slice(0, 5).map(summarizeArchiveEntry)), + "", + "[current_prompt_snapshot]", + input.currentPromptSnapshot, + "", + 'JSON schema: {"prompts":[{"description":"...","diff":[{"layer":"policy","op":"append","text":"...","reason":"..."}],"rationale":"...","expectedImprovement":"..."}],"tools":[{"description":"...","toolName":"...","schema":{},"whyNeeded":"..."}]}', + ].join("\n"); +} + +export async function proposeVariants( + llmClient: LlmClient, + input: ProposalInput, +): Promise<{ prompts: PromptVariantProposal[]; tools: ToolProposal[] }> { + if (!llmClient.hasEndpoints()) { + throw new Error("Cannot propose variants without an LLM endpoint"); + } + + const response = await llmClient.chatCompletion( + [ + { + role: "system", + content: + "당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.", + }, + { + role: "user", + content: buildProposalPrompt(input), + }, + ], + { + jsonMode: true, + maxTokens: getLlmProposeMaxTokens(), + modelOverride: getLlmProposeModel(), + temperature: 0.2, + }, + ); + + if (!response) { + throw new Error("Proposal LLM call failed"); + } + + const parsed = proposalSchema.parse(JSON.parse(response.content)); + const prompts = parsed.prompts.map((proposal) => { + const diff = proposal.diff.map(normalizePromptDiff); + return { + ...proposal, + diff, + variantId: stableHash({ + description: proposal.description, + diff, + rationale: proposal.rationale, + }), + }; + }); + const tools = parsed.tools.map((proposal) => ({ + ...proposal, + variantId: stableHash(proposal), + })); + + return { + prompts, + tools, + }; +} + +export async function buildProposalInputFromArchive( + mongoConnection: MongoConnectionLike, + slice: SliceKey, + options?: { + force?: boolean | undefined; + limit?: number | undefined; + minCases?: number | undefined; + nowMs?: number | undefined; + }, +): Promise { + const limit = options?.limit ?? 50; + const entries = await getEvalArchiveCollection(mongoConnection) + .find({ + tier: slice.tier, + "derived.intent.kind": slice.intent, + "derived.phase.kind": slice.phase, + }) + .sort({ evaluatedAt: -1 }) + .limit(limit) + .toArray(); + const minCases = options?.minCases ?? getEvalTournamentMinCases(); + + if (!options?.force && entries.length < minCases) { + throw new Error( + `Not enough archived cases for propose: ${entries.length}/${minCases}. Use --force to override.`, + ); + } + + const ids = new Set(entries.map((entry) => entry.streamId)); + const recentLabels = (await queryArchivedReviewLabels(mongoConnection, { + limit: 1_000, + nowMs: options?.nowMs, + })).filter((label) => ids.has(label.evalStreamId)); + const worstCases = [...entries] + .filter((entry) => getLlmOverall(entry) !== null) + .sort( + (left, right) => + (getLlmOverall(left) ?? Number.POSITIVE_INFINITY) - + (getLlmOverall(right) ?? Number.POSITIVE_INFINITY), + ) + .slice(0, 5); + + return { + slice, + recentLabels, + worstCases: worstCases.length > 0 ? worstCases : entries.slice(0, 5), + currentPromptSnapshot: buildCurrentPromptSnapshot(slice, entries[0]), + }; +} + +export async function writePromptVariantProposal( + proposal: PromptVariantProposal, + outputDir = "src/eval/proposals", +): Promise { + const outputPath = resolve(process.cwd(), outputDir, `${proposal.variantId}.json`); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(`${outputPath}`, `${JSON.stringify(proposal, null, 2)}\n`, "utf8"); + return outputPath; +} diff --git a/src/eval/loop/tournament.test.ts b/src/eval/loop/tournament.test.ts new file mode 100644 index 0000000..dea8418 --- /dev/null +++ b/src/eval/loop/tournament.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from "vitest"; + +import type { LlmClient } from "@llm/llm-client"; +import type { EvalArchiveEntry } from "../archive.js"; +import { runTournament } from "./tournament.js"; + +function archiveEntry(streamId: string): EvalArchiveEntry { + return { + streamId, + evaluatedAt: new Date("2026-05-01T00:00:00.000Z"), + request: { + requestId: streamId, + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "닉", + trigger: "user-message", + userMessage: "사진 어디로 보내요?", + }, + response: { + requestId: streamId, + text: "이 채팅방에 올려주세요.", + confidence: 0.9, + source: "llm", + }, + derived: { + phase: { kind: "photo-pending" }, + intent: { kind: "photo-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + }, + classified: { + intent: { kind: "photo-question" }, + confidence: 0.9, + source: "regex", + }, + validation: { + status: "pass", + text: "이 채팅방에 올려주세요.", + reasons: [], + }, + llmJudge: { + skipped: false, + model: "judge", + endpoint: "local", + provider: "test", + scores: { + friendliness: 4, + accuracy: 4, + concision: 4, + scope: 4, + overall: 4, + }, + rationale: "ok", + raw: "{}", + }, + latencyMs: 100, + tier: 0, + }; +} + +describe("runTournament", () => { + it("runs swapped A/B matches and updates ELO toward winning variants", async () => { + const llmClient = { + hasEndpoints: () => true, + chatCompletion: ( + messages: Array<{ content: string; role: string }>, + options?: { jsonMode?: boolean | undefined }, + ) => { + const content = messages.map((message) => message.content).join("\n"); + if (options?.jsonMode) { + const verdict = content.includes("[response_A]\nvariant answer") + ? "A" + : "B"; + return Promise.resolve({ + content: JSON.stringify({ + verdict, + rationale: "variant is more accurate", + }), + endpoint: "mock", + message: { role: "assistant", content: "{}" }, + provider: "cliproxy", + raw: {}, + }); + } + + return Promise.resolve({ + content: content.includes("사진 인증 위치를 명시해") + ? "variant answer" + : "control answer", + endpoint: "mock", + message: { role: "assistant", content: "ok" }, + provider: "cliproxy", + raw: {}, + }); + }, + } as unknown as LlmClient; + + const result = await runTournament(llmClient, { + cases: [archiveEntry("1-0"), archiveEntry("2-0")], + maxMatches: 4, + nowMs: Date.parse("2026-05-01T02:00:00.000Z"), + variantIds: ["control", "abc12345"], + variantManifests: { + abc12345: { + variantId: "abc12345", + description: "photo fix", + diff: [ + { + layer: "policy", + op: "append", + text: "사진 인증 위치를 명시해.", + }, + ], + }, + }, + }); + + expect(result.generatedAt).toBe("2026-05-01T02:00:00.000Z"); + expect(result.matches).toHaveLength(4); + expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([ + "control", + "abc12345", + ]); + expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([ + "abc12345", + "control", + ]); + expect(result.variants[0]).toMatchObject({ + id: "abc12345", + matches: 4, + wins: 4, + }); + expect(result.variants[0]?.elo).toBeGreaterThan(1_500); + }); +}); diff --git a/src/eval/loop/tournament.ts b/src/eval/loop/tournament.ts new file mode 100644 index 0000000..acfb87d --- /dev/null +++ b/src/eval/loop/tournament.ts @@ -0,0 +1,363 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; +import type { LlmClient } from "@llm/llm-client"; +import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder"; +import type { Blackboard, PromptContext } from "@skills/onboarding/types"; +import { getEvalTournamentMaxMatches } from "../../config.js"; +import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js"; +import { getEvalArchiveCollection } from "../archive.js"; +import { + createEloRating, + scoreForVerdict, + updateEloPair, + type EloRating, +} from "./elo.js"; + +export const EVAL_TOURNAMENT_COLLECTION = "eval_tournaments"; + +export interface TournamentMatch { + caseId: string; + judgeRationale: string; + judgeVerdict: "A" | "B" | "tie"; + matchId: string; + responseA: string; + responseB: string; + variantA: string; + variantB: string; +} + +export interface TournamentResult { + generatedAt: string; + matches: TournamentMatch[]; + variants: EloRating[]; +} + +interface TournamentCollectionLike { + createIndex( + keys: Record, + options?: Record, + ): Promise; + find(filter: Record): { + limit(value: number): { + sort(value: Record): { + toArray(): Promise; + }; + }; + sort(value: Record): { + limit(value: number): { + toArray(): Promise; + }; + }; + }; + insertOne(document: TournamentResult): Promise; +} + +const sideBySideJudgeSchema = z.object({ + rationale: z.string().trim().min(1).max(300), + verdict: z.enum(["A", "B", "tie"]), +}); + +function stableHash(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(value)) + .digest("hex") + .slice(0, 12); +} + +function emptyBlackboard(): Blackboard { + return { + flaggedIssues: [], + openQuestions: [], + recentNicknameAttempts: [], + toneMemory: {}, + }; +} + +function promptContextForCase(entry: EvalArchiveEntry): PromptContext { + return { + blackboard: emptyBlackboard(), + conversationHistory: [], + userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.", + }; +} + +function getVariantManifest( + variantId: string, + manifests?: Record, +): PromptVariantManifest | string | null { + if (variantId === "control") { + return null; + } + return manifests?.[variantId] ?? variantId; +} + +async function generateTournamentResponse( + llmClient: LlmClient, + entry: EvalArchiveEntry, + variantId: string, + manifests?: Record, +): Promise { + let builder = PromptBuilder.forDerived( + entry.derived, + null, + promptContextForCase(entry), + ); + const manifest = getVariantManifest(variantId, manifests); + if (manifest) { + builder = builder.applyVariant(manifest); + } + + const response = await llmClient.chatCompletion(builder.build(), { + maxTokens: entry.derived.suggestedMaxTokens, + temperature: 0, + tier: entry.derived.tier, + }); + + return response?.content.trim() ?? ""; +} + +function buildJudgePrompt( + entry: EvalArchiveEntry, + responseA: string, + responseB: string, +): string { + return [ + "아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.", + "평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.", + "A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.", + "", + "[request]", + JSON.stringify(entry.request), + "", + "[derived]", + JSON.stringify(entry.derived), + "", + "[archived_reference_response]", + entry.response.text, + "", + "[response_A]", + responseA, + "", + "[response_B]", + responseB, + "", + 'JSON object only: {"verdict":"A","rationale":"..."}', + ].join("\n"); +} + +async function judgeTournamentMatch( + llmClient: LlmClient, + entry: EvalArchiveEntry, + responseA: string, + responseB: string, +): Promise<{ rationale: string; verdict: "A" | "B" | "tie" }> { + const response = await llmClient.chatCompletion( + [ + { + role: "system", + content: + "당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.", + }, + { + role: "user", + content: buildJudgePrompt(entry, responseA, responseB), + }, + ], + { + jsonMode: true, + maxTokens: 256, + temperature: 0, + }, + ); + + if (!response) { + return { + rationale: "judge-call-failed", + verdict: "tie", + }; + } + + try { + return sideBySideJudgeSchema.parse(JSON.parse(response.content)); + } catch { + return { + rationale: "invalid-judge-json", + verdict: "tie", + }; + } +} + +function getTournamentCollection( + mongoConnection: MongoConnectionLike, +): TournamentCollectionLike { + return mongoConnection.collection( + EVAL_TOURNAMENT_COLLECTION, + ) as TournamentCollectionLike; +} + +async function ensureTournamentIndexes( + collection: TournamentCollectionLike, +): Promise { + await Promise.all([ + collection.createIndex({ generatedAt: -1 }), + collection.createIndex({ "variants.id": 1, generatedAt: -1 }), + ]); +} + +function buildVariantPairs(variantIds: string[]): Array<[string, string]> { + const pairs: Array<[string, string]> = []; + for (let left = 0; left < variantIds.length; left++) { + for (let right = left + 1; right < variantIds.length; right++) { + const variantA = variantIds[left]; + const variantB = variantIds[right]; + if (variantA && variantB) { + pairs.push([variantA, variantB], [variantB, variantA]); + } + } + } + return pairs; +} + +export async function runTournament( + llmClient: LlmClient, + options: { + cases: EvalArchiveEntry[]; + maxMatches?: number | undefined; + nowMs?: number | undefined; + variantIds: string[]; + variantManifests?: Record | undefined; + }, +): Promise { + if (!llmClient.hasEndpoints()) { + throw new Error("Cannot run tournament without an LLM endpoint"); + } + if (options.variantIds.length < 2) { + throw new Error("Tournament requires at least two variants"); + } + + const maxMatches = options.maxMatches ?? getEvalTournamentMaxMatches(); + const ratings = new Map( + options.variantIds.map((variantId) => [variantId, createEloRating(variantId)]), + ); + const matches: TournamentMatch[] = []; + const pairs = buildVariantPairs(options.variantIds); + + for (const entry of options.cases) { + for (const [variantA, variantB] of pairs) { + if (matches.length >= maxMatches) { + break; + } + + const [responseA, responseB] = await Promise.all([ + generateTournamentResponse( + llmClient, + entry, + variantA, + options.variantManifests, + ), + generateTournamentResponse( + llmClient, + entry, + variantB, + options.variantManifests, + ), + ]); + const judge = await judgeTournamentMatch( + llmClient, + entry, + responseA, + responseB, + ); + const ratingA = ratings.get(variantA); + const ratingB = ratings.get(variantB); + if (!ratingA || !ratingB) { + throw new Error("Missing tournament rating"); + } + const updated = updateEloPair( + ratingA, + ratingB, + scoreForVerdict(judge.verdict), + ); + ratings.set(variantA, updated.ratingA); + ratings.set(variantB, updated.ratingB); + + matches.push({ + caseId: entry.streamId, + judgeRationale: judge.rationale, + judgeVerdict: judge.verdict, + matchId: stableHash({ + caseId: entry.streamId, + index: matches.length, + variantA, + variantB, + }), + responseA, + responseB, + variantA, + variantB, + }); + } + if (matches.length >= maxMatches) { + break; + } + } + + return { + generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(), + matches, + variants: [...ratings.values()].sort((left, right) => right.elo - left.elo), + }; +} + +export async function loadTournamentCases( + mongoConnection: MongoConnectionLike, + options: { + caseIds?: string[] | undefined; + limit?: number | undefined; + }, +): Promise { + const filter = + options.caseIds && options.caseIds.length > 0 + ? { + streamId: { + $in: options.caseIds, + }, + } + : {}; + + return getEvalArchiveCollection(mongoConnection) + .find(filter) + .sort({ evaluatedAt: -1 }) + .limit(options.limit ?? getEvalTournamentMaxMatches()) + .toArray(); +} + +export async function saveTournamentResult( + mongoConnection: MongoConnectionLike, + result: TournamentResult, +): Promise { + const collection = getTournamentCollection(mongoConnection); + await ensureTournamentIndexes(collection); + await collection.insertOne(result); +} + +export async function queryTournamentResults( + mongoConnection: MongoConnectionLike, + options?: { + limit?: number | undefined; + variantId?: string | undefined; + }, +): Promise { + const filter = options?.variantId + ? { + "variants.id": options.variantId, + } + : {}; + + return getTournamentCollection(mongoConnection) + .find(filter) + .sort({ generatedAt: -1 }) + .limit(options?.limit ?? 20) + .toArray(); +} diff --git a/src/eval/promote.test.ts b/src/eval/promote.test.ts new file mode 100644 index 0000000..137811d --- /dev/null +++ b/src/eval/promote.test.ts @@ -0,0 +1,171 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { describe, expect, it } from "vitest"; + +import type { EvalArchiveEntry } from "./archive.js"; +import { + appendCandidatesToJsonl, + selectPromotionCandidatesFromArchiveEntries, +} from "./promote.js"; + +function archiveEntry( + streamId: string, + options: { + intent?: "photo-question" | "status-question" | undefined; + overall: number; + phase?: "fresh" | "photo-pending" | undefined; + status: "pass" | "soft-warn" | "reject"; + }, +): EvalArchiveEntry { + return { + streamId, + evaluatedAt: new Date("2026-05-01T00:00:00.000Z"), + request: { + requestId: streamId, + skill: "onboarding", + chatroomId: "raw-room", + userId: "raw-user", + nickname: "홍길동", + trigger: "user-message", + userMessage: "전화번호는 010-1234-5678이고 인증 어디서 해요?", + }, + response: { + requestId: streamId, + text: "010-1234-5678로 안내했어요.", + confidence: 0.9, + source: "llm", + }, + derived: { + phase: { kind: options.phase ?? "photo-pending" }, + intent: { kind: options.intent ?? "status-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + }, + classified: { + intent: { kind: options.intent ?? "status-question" }, + confidence: 0.9, + source: "regex", + }, + validation: { + status: options.status, + text: "응답", + reasons: [], + }, + llmJudge: { + skipped: false, + model: "judge", + endpoint: "local", + provider: "test", + scores: { + friendliness: options.overall, + accuracy: options.overall, + concision: options.overall, + scope: options.overall, + overall: options.overall, + }, + rationale: "ok", + raw: "{}", + }, + latencyMs: 100, + tier: 0, + }; +} + +describe("selectPromotionCandidatesFromArchiveEntries", () => { + it("selects low-score, disagreement, and rare-slice candidates", () => { + const candidates = selectPromotionCandidatesFromArchiveEntries( + [ + archiveEntry("1-0", { overall: 1, status: "pass" }), + archiveEntry("2-0", { overall: 2.5, status: "pass" }), + archiveEntry("3-0", { + intent: "photo-question", + overall: 5, + phase: "fresh", + status: "pass", + }), + ], + { + limit: 10, + minLowScore: 2, + rareThreshold: 1, + }, + ); + + expect(candidates.map((candidate) => candidate.reason)).toEqual([ + "low-score", + "rule-llm-disagree", + "rare-slice", + ]); + expect(candidates[0]?.request.userId).not.toBe("raw-user"); + expect(candidates[0]?.request.userMessage).toContain("[REDACTED:phone]"); + expect(candidates[0]?.response.text).toContain("[REDACTED:phone]"); + }); + + it("prioritizes human labeled failures and carries better responses into notes", () => { + const candidates = selectPromotionCandidatesFromArchiveEntries( + [archiveEntry("1-0", { overall: 4, status: "pass" })], + { + limit: 10, + minLowScore: 2, + rareThreshold: 0, + }, + [ + { + evalStreamId: "1-0", + verdict: "fail", + humanScore: 1, + rationale: "전화번호 010-1234-5678 노출", + reviewedBy: "tester", + reviewedAt: "2026-05-01T01:00:00.000Z", + betterResponse: "개인정보는 이 채팅방에 남기지 말고 운영진에게 문의해 주세요.", + }, + ], + ); + + expect(candidates[0]).toMatchObject({ + reason: "labeled-fail", + humanScore: 1, + }); + expect(candidates[0]?.labelRationale).toContain("[REDACTED:phone]"); + expect(candidates[0]?.betterResponse).toContain("운영진"); + }); +}); + +describe("appendCandidatesToJsonl", () => { + it("writes candidates in eval case format and skips duplicate case ids", async () => { + const directory = await mkdtemp(join(tmpdir(), "lolympus-promote-")); + const outputPath = join(directory, "candidate.jsonl"); + const candidates = selectPromotionCandidatesFromArchiveEntries( + [archiveEntry("1-0", { overall: 1, status: "pass" })], + { + limit: 10, + minLowScore: 2, + rareThreshold: 1, + }, + ); + + try { + const first = await appendCandidatesToJsonl(candidates, outputPath); + const second = await appendCandidatesToJsonl(candidates, outputPath); + const raw = await readFile(outputPath, "utf8"); + + expect(first.appendedCount).toBe(1); + expect(second.appendedCount).toBe(0); + expect(raw.trim().split(/\r?\n/u)).toHaveLength(1); + expect(JSON.parse(raw) as unknown).toMatchObject({ + input: { + request: { + skill: "onboarding", + }, + }, + expected: {}, + }); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/src/eval/promote.ts b/src/eval/promote.ts new file mode 100644 index 0000000..296514a --- /dev/null +++ b/src/eval/promote.ts @@ -0,0 +1,355 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +import type { AgentRequest, AgentResponse } from "@agent/types"; +import { + getEvalPromoteLowScore, + getEvalPromotePerCategoryLimit, + getEvalPromoteRareThreshold, +} from "../config.js"; +import { + anonymizeIdentifier, + anonymizeNickname, + anonymizeText, +} from "./anonymize.js"; +import type { EvalCaseDefinition } from "./types.js"; +import type { EvalArchiveEntry, MongoConnectionLike } from "./archive.js"; +import { getEvalArchiveCollection } from "./archive.js"; +import { queryArchivedReviewLabels } from "./review/label-archive.js"; +import type { ReviewLabel } from "./review/queue.js"; + +const DEFAULT_CANDIDATE_PATH = "src/eval/golden/onboarding.candidate.jsonl"; +const DAY_MS = 24 * 60 * 60 * 1000; + +export interface PromotionCandidate { + betterResponse?: string | undefined; + caseId: string; + humanScore?: number | undefined; + labelRationale?: string | undefined; + request: AgentRequest; + response: AgentResponse; + derived: EvalArchiveEntry["derived"]; + llmJudge: EvalArchiveEntry["llmJudge"]; + reason: "labeled-fail" | "low-score" | "rule-llm-disagree" | "rare-slice"; + sourceStreamId: string; +} + +interface SelectionOptions { + limit?: number | undefined; + minLowScore?: number | undefined; + nowMs?: number | undefined; + rareThreshold?: number | undefined; +} + +function stableShortHash(value: string): string { + return createHash("sha256").update(value).digest("hex").slice(0, 8); +} + +function getLlmOverall(entry: EvalArchiveEntry): number | null { + if (entry.llmJudge.skipped) { + return null; + } + + return entry.llmJudge.scores.overall; +} + +function sliceKey(entry: EvalArchiveEntry): string { + return `${entry.tier}:${entry.derived.intent.kind}:${entry.derived.phase.kind}`; +} + +function sortByLowestScore( + left: EvalArchiveEntry, + right: EvalArchiveEntry, +): number { + return (getLlmOverall(left) ?? Number.POSITIVE_INFINITY) - + (getLlmOverall(right) ?? Number.POSITIVE_INFINITY); +} + +function anonymizeUnknown(value: unknown): unknown { + if (typeof value === "string") { + return anonymizeText(value); + } + if (Array.isArray(value)) { + return value.map((item) => anonymizeUnknown(item)); + } + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value).map(([key, nested]) => [ + key, + anonymizeUnknown(nested), + ]), + ); + } + return value; +} + +function anonymizeRequest(request: AgentRequest): AgentRequest { + const anonymized: AgentRequest = { + requestId: + anonymizeIdentifier("req", request.requestId) ?? + `req_${stableShortHash(request.requestId)}`, + skill: request.skill, + chatroomId: + anonymizeIdentifier("room", request.chatroomId) ?? + `room_${stableShortHash(request.chatroomId)}`, + userId: + anonymizeIdentifier("user", request.userId) ?? + `user_${stableShortHash(request.userId)}`, + nickname: anonymizeNickname(request.nickname) ?? "", + trigger: request.trigger, + }; + + if (request.userMessage !== undefined) { + anonymized.userMessage = anonymizeText(request.userMessage); + } + if (request.context !== undefined) { + anonymized.context = anonymizeUnknown(request.context) as Record< + string, + unknown + >; + } + + return anonymized; +} + +function anonymizeResponse(response: AgentResponse): AgentResponse { + const anonymized: AgentResponse = { + requestId: + anonymizeIdentifier("req", response.requestId) ?? + `req_${stableShortHash(response.requestId)}`, + text: anonymizeText(response.text), + confidence: response.confidence, + source: response.source, + }; + + if (response.metadata !== undefined) { + anonymized.metadata = anonymizeUnknown(response.metadata) as Record< + string, + unknown + >; + } + + return anonymized; +} + +function toPromotionCandidate( + entry: EvalArchiveEntry, + reason: PromotionCandidate["reason"], + label?: ReviewLabel, +): PromotionCandidate { + const candidate: PromotionCandidate = { + caseId: + anonymizeIdentifier("case", entry.streamId) ?? + `case_${stableShortHash(entry.streamId)}`, + request: anonymizeRequest(entry.request), + response: anonymizeResponse(entry.response), + derived: entry.derived, + llmJudge: entry.llmJudge, + reason, + sourceStreamId: entry.streamId, + }; + if (label?.betterResponse !== undefined) { + candidate.betterResponse = anonymizeText(label.betterResponse); + } + if (label?.humanScore !== undefined) { + candidate.humanScore = label.humanScore; + } + if (label?.rationale !== undefined) { + candidate.labelRationale = anonymizeText(label.rationale); + } + return candidate; +} + +function addUniqueCandidate( + candidates: PromotionCandidate[], + seen: Set, + entry: EvalArchiveEntry, + reason: PromotionCandidate["reason"], + label?: ReviewLabel, +): void { + if (seen.has(entry.streamId)) { + return; + } + + seen.add(entry.streamId); + candidates.push(toPromotionCandidate(entry, reason, label)); +} + +export function selectPromotionCandidatesFromArchiveEntries( + entries: EvalArchiveEntry[], + options?: SelectionOptions, + labels: ReviewLabel[] = [], +): PromotionCandidate[] { + const perCategoryLimit = options?.limit ?? getEvalPromotePerCategoryLimit(); + const minLowScore = options?.minLowScore ?? getEvalPromoteLowScore(); + const rareThreshold = options?.rareThreshold ?? getEvalPromoteRareThreshold(); + const candidates: PromotionCandidate[] = []; + const seen = new Set(); + const entryByStreamId = new Map( + entries.map((entry) => [entry.streamId, entry]), + ); + + const labeledFail = labels + .filter((label) => label.humanScore <= 2) + .slice(0, perCategoryLimit); + for (const label of labeledFail) { + const entry = entryByStreamId.get(label.evalStreamId); + if (entry) { + addUniqueCandidate(candidates, seen, entry, "labeled-fail", label); + } + } + + const lowScore = entries + .filter((entry) => { + const overall = getLlmOverall(entry); + return overall !== null && overall <= minLowScore; + }) + .sort(sortByLowestScore) + .slice(0, perCategoryLimit); + for (const entry of lowScore) { + addUniqueCandidate(candidates, seen, entry, "low-score"); + } + + const disagree = entries + .filter((entry) => { + const overall = getLlmOverall(entry); + if (overall === null) { + return false; + } + return ( + (entry.validation.status === "pass" && overall <= 3) || + (entry.validation.status === "reject" && overall >= 4) + ); + }) + .sort(sortByLowestScore) + .slice(0, perCategoryLimit); + for (const entry of disagree) { + addUniqueCandidate(candidates, seen, entry, "rule-llm-disagree"); + } + + const sliceCounts = new Map(); + for (const entry of entries) { + const key = sliceKey(entry); + sliceCounts.set(key, (sliceCounts.get(key) ?? 0) + 1); + } + + const rare = entries + .filter((entry) => (sliceCounts.get(sliceKey(entry)) ?? 0) <= rareThreshold) + .slice(0, perCategoryLimit); + for (const entry of rare) { + addUniqueCandidate(candidates, seen, entry, "rare-slice"); + } + + return candidates; +} + +export async function selectPromotionCandidates( + mongoConnection: MongoConnectionLike, + options?: SelectionOptions, +): Promise { + const nowMs = options?.nowMs ?? Date.now(); + const [entries, labels] = await Promise.all([ + getEvalArchiveCollection(mongoConnection) + .find({ + evaluatedAt: { + $gte: new Date(nowMs - DAY_MS), + }, + }) + .sort({ evaluatedAt: -1 }) + .limit(10_000) + .toArray(), + queryArchivedReviewLabels(mongoConnection, { + nowMs, + sinceMs: DAY_MS, + }), + ]); + + return selectPromotionCandidatesFromArchiveEntries(entries, options, labels); +} + +function candidateToEvalCase(candidate: PromotionCandidate): EvalCaseDefinition { + const llmOverall = candidate.llmJudge.skipped + ? "n/a" + : String(candidate.llmJudge.scores.overall); + const notes = [ + `promotionReason=${candidate.reason}`, + `sourceStreamId=${candidate.sourceStreamId}`, + `llmOverall=${llmOverall}`, + candidate.humanScore === undefined + ? null + : `humanScore=${candidate.humanScore}`, + candidate.labelRationale === undefined + ? null + : `humanRationale=${candidate.labelRationale}`, + `derivedIntent=${candidate.derived.intent.kind}`, + `derivedPhase=${candidate.derived.phase.kind}`, + `assistantResponse=${candidate.response.text}`, + candidate.betterResponse === undefined + ? null + : `betterResponse=${candidate.betterResponse}`, + ] + .filter((note): note is string => note !== null) + .join("; "); + + return { + id: candidate.caseId, + lineNumber: 0, + name: `promotion:${candidate.reason}`, + notes, + input: { + request: candidate.request, + }, + expected: {}, + }; +} + +function extractExistingCaseIds(raw: string): Set { + const ids = new Set(); + for (const line of raw.split(/\r?\n/u)) { + if (!line.trim()) { + continue; + } + + try { + const parsed = JSON.parse(line) as { id?: unknown }; + if (typeof parsed.id === "string") { + ids.add(parsed.id); + } + } catch { + continue; + } + } + return ids; +} + +export async function appendCandidatesToJsonl( + candidates: PromotionCandidate[], + outputPath = DEFAULT_CANDIDATE_PATH, +): Promise<{ appendedCount: number; outputPath: string }> { + const resolved = resolve(process.cwd(), outputPath); + await mkdir(dirname(resolved), { recursive: true }); + const existingRaw = existsSync(resolved) + ? await readFile(resolved, "utf8") + : ""; + const existingIds = extractExistingCaseIds(existingRaw); + const lines: string[] = []; + + for (const candidate of candidates) { + if (existingIds.has(candidate.caseId)) { + continue; + } + lines.push(JSON.stringify(candidateToEvalCase(candidate))); + existingIds.add(candidate.caseId); + } + + if (lines.length > 0) { + await appendFile(resolved, `${lines.join("\n")}\n`, "utf8"); + } + + return { + appendedCount: lines.length, + outputPath: resolved, + }; +} diff --git a/src/eval/review/calibration.test.ts b/src/eval/review/calibration.test.ts new file mode 100644 index 0000000..df1264a --- /dev/null +++ b/src/eval/review/calibration.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; + +import type { EvalArchiveEntry } from "../archive.js"; +import type { ReviewLabel } from "./queue.js"; +import { computeCalibrationReportFromData } from "./calibration.js"; + +function archiveEntry(streamId: string, overall: number): EvalArchiveEntry { + return { + streamId, + evaluatedAt: new Date("2026-05-01T00:00:00.000Z"), + request: { + requestId: streamId, + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "닉", + trigger: "user-message", + userMessage: "상태 알려줘", + }, + response: { + requestId: streamId, + text: "사진 인증만 남았어요.", + confidence: 0.9, + source: "llm", + }, + derived: { + phase: { kind: "photo-pending" }, + intent: { kind: "status-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + }, + classified: { + intent: { kind: "status-question" }, + confidence: 0.9, + source: "regex", + }, + validation: { + status: "pass", + text: "사진 인증만 남았어요.", + reasons: [], + }, + llmJudge: { + skipped: false, + model: "judge", + endpoint: "local", + provider: "test", + scores: { + friendliness: overall, + accuracy: overall, + concision: overall, + scope: overall, + overall, + }, + rationale: "ok", + raw: "{}", + }, + latencyMs: 100, + tier: 0, + }; +} + +function label(evalStreamId: string, humanScore: 1 | 2 | 3 | 4 | 5): ReviewLabel { + return { + evalStreamId, + verdict: humanScore >= 4 ? "pass" : "fail", + humanScore, + rationale: "reviewed", + reviewedBy: "tester", + reviewedAt: "2026-05-01T01:00:00.000Z", + }; +} + +describe("computeCalibrationReportFromData", () => { + it("computes correlation and gaps from human/LLM score pairs", () => { + const report = computeCalibrationReportFromData( + [label("1-0", 1), label("2-0", 3), label("3-0", 5)], + [ + archiveEntry("1-0", 1), + archiveEntry("2-0", 3), + archiveEntry("3-0", 5), + ], + Date.parse("2026-05-01T02:00:00.000Z"), + ); + + expect(report).toMatchObject({ + generatedAt: "2026-05-01T02:00:00.000Z", + pairs: 3, + pearsonOverall: 1, + meanGapOverall: 0, + }); + expect(report.pearsonByDim.accuracy).toBe(1); + }); + + it("ignores labels without usable archived LLM judge data", () => { + const skipped = archiveEntry("2-0", 3); + skipped.llmJudge = { + skipped: true, + reason: "disabled", + }; + + const report = computeCalibrationReportFromData( + [label("1-0", 1), label("2-0", 3), label("missing", 5)], + [archiveEntry("1-0", 2), skipped], + Date.parse("2026-05-01T02:00:00.000Z"), + ); + + expect(report.pairs).toBe(1); + expect(report.meanGapOverall).toBe(-1); + expect(report.worstDisagreements).toEqual([ + { + evalStreamId: "1-0", + humanScore: 1, + llmOverall: 2, + }, + ]); + }); +}); diff --git a/src/eval/review/calibration.ts b/src/eval/review/calibration.ts new file mode 100644 index 0000000..6888994 --- /dev/null +++ b/src/eval/review/calibration.ts @@ -0,0 +1,204 @@ +import { + type EvalArchiveEntry, + getEvalArchiveCollection, + type MongoConnectionLike, +} from "../archive.js"; +import { queryArchivedReviewLabels } from "./label-archive.js"; +import type { ReviewLabel } from "./queue.js"; +import { readReviewLabels } from "./queue.js"; +import type { GarnetClient } from "@common/garnet"; + +export interface CalibrationReport { + generatedAt: string; + meanGapOverall: number; + pairs: number; + pearsonByDim: { + accuracy: number; + concision: number; + friendliness: number; + scope: number; + }; + pearsonOverall: number; + worstDisagreements: Array<{ + evalStreamId: string; + humanScore: number; + llmOverall: number; + }>; +} + +interface CalibrationPair { + evalStreamId: string; + humanScore: number; + llmAccuracy: number; + llmConcision: number; + llmFriendliness: number; + llmOverall: number; + llmScope: number; +} + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +function pearson(left: number[], right: number[]): number { + if (left.length < 2 || right.length < 2 || left.length !== right.length) { + return 0; + } + + const leftMean = left.reduce((sum, value) => sum + value, 0) / left.length; + const rightMean = right.reduce((sum, value) => sum + value, 0) / right.length; + let numerator = 0; + let leftVariance = 0; + let rightVariance = 0; + + for (let index = 0; index < left.length; index++) { + const leftDelta = (left[index] ?? 0) - leftMean; + const rightDelta = (right[index] ?? 0) - rightMean; + numerator += leftDelta * rightDelta; + leftVariance += leftDelta * leftDelta; + rightVariance += rightDelta * rightDelta; + } + + const denominator = Math.sqrt(leftVariance * rightVariance); + return denominator === 0 ? 0 : round(numerator / denominator); +} + +function toCalibrationPairs( + labels: ReviewLabel[], + archives: EvalArchiveEntry[], +): CalibrationPair[] { + const archiveByStreamId = new Map( + archives.map((entry) => [entry.streamId, entry]), + ); + const pairs: CalibrationPair[] = []; + + for (const label of labels) { + const archive = archiveByStreamId.get(label.evalStreamId); + if (!archive || archive.llmJudge.skipped) { + continue; + } + + pairs.push({ + evalStreamId: label.evalStreamId, + humanScore: label.humanScore, + llmAccuracy: archive.llmJudge.scores.accuracy, + llmConcision: archive.llmJudge.scores.concision, + llmFriendliness: archive.llmJudge.scores.friendliness, + llmOverall: archive.llmJudge.scores.overall, + llmScope: archive.llmJudge.scores.scope, + }); + } + + return pairs; +} + +function mergeLabels( + streamLabels: ReviewLabel[], + archivedLabels: ReviewLabel[], +): ReviewLabel[] { + const merged: ReviewLabel[] = []; + const seen = new Set(); + + for (const label of [...streamLabels, ...archivedLabels]) { + const key = [ + label.evalStreamId, + label.reviewedAt, + label.reviewedBy, + label.humanScore, + ].join(":"); + if (seen.has(key)) { + continue; + } + seen.add(key); + merged.push(label); + } + + return merged; +} + +export function computeCalibrationReportFromData( + labels: ReviewLabel[], + archives: EvalArchiveEntry[], + nowMs = Date.now(), +): CalibrationReport { + const pairs = toCalibrationPairs(labels, archives); + const humanScores = pairs.map((pair) => pair.humanScore); + const overallScores = pairs.map((pair) => pair.llmOverall); + const meanGapOverall = + pairs.length === 0 + ? 0 + : round( + pairs.reduce( + (sum, pair) => sum + (pair.humanScore - pair.llmOverall), + 0, + ) / pairs.length, + ); + + return { + generatedAt: new Date(nowMs).toISOString(), + meanGapOverall, + pairs: pairs.length, + pearsonByDim: { + accuracy: pearson(humanScores, pairs.map((pair) => pair.llmAccuracy)), + concision: pearson(humanScores, pairs.map((pair) => pair.llmConcision)), + friendliness: pearson( + humanScores, + pairs.map((pair) => pair.llmFriendliness), + ), + scope: pearson(humanScores, pairs.map((pair) => pair.llmScope)), + }, + pearsonOverall: pearson(humanScores, overallScores), + worstDisagreements: pairs + .map((pair) => ({ + evalStreamId: pair.evalStreamId, + humanScore: pair.humanScore, + llmOverall: pair.llmOverall, + })) + .sort( + (left, right) => + Math.abs(right.humanScore - right.llmOverall) - + Math.abs(left.humanScore - left.llmOverall), + ) + .slice(0, 10), + }; +} + +export async function computeCalibrationReport( + mongoConnection: MongoConnectionLike, + garnetClient: Pick, + options?: { sinceMs?: number | undefined }, +): Promise { + const nowMs = Date.now(); + const [streamLabels, archivedLabels] = await Promise.all([ + readReviewLabels(garnetClient), + queryArchivedReviewLabels(mongoConnection, { + sinceMs: options?.sinceMs, + nowMs, + }), + ]); + const labels = mergeLabels(streamLabels, archivedLabels); + const cutoffMs = + options?.sinceMs === undefined ? null : nowMs - options.sinceMs; + const filteredLabels = + cutoffMs === null + ? labels + : labels.filter((label) => { + const reviewedAt = Date.parse(label.reviewedAt); + return Number.isFinite(reviewedAt) && reviewedAt >= cutoffMs; + }); + const ids = [...new Set(filteredLabels.map((label) => label.evalStreamId))]; + const archives = + ids.length === 0 + ? [] + : await getEvalArchiveCollection(mongoConnection) + .find({ + streamId: { + $in: ids, + }, + }) + .sort({ evaluatedAt: -1 }) + .limit(ids.length) + .toArray(); + + return computeCalibrationReportFromData(filteredLabels, archives); +} diff --git a/src/eval/review/cli.ts b/src/eval/review/cli.ts new file mode 100644 index 0000000..cec7091 --- /dev/null +++ b/src/eval/review/cli.ts @@ -0,0 +1,269 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { createInterface } from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; + +import type { GarnetClient } from "@common/garnet"; +import { + acknowledgeReviewItem, + enqueueReviewCandidate, + fetchNextReviewItem, + recordReviewLabel, + resetPendingReviewItems, + type ReviewLabel, + type ReviewQueueEntry, + type ReviewReason, + type ReviewConsumerGarnetClient, +} from "./queue.js"; + +export interface ReviewCliOptions { + consumerName?: string | undefined; + exportLabeled?: string | undefined; + limit?: number | undefined; + reason?: ReviewReason | undefined; + resetPending?: boolean | undefined; +} + +function getReviewerId(): string { + const configured = process.env["REVIEWER_ID"]?.trim(); + if (configured) { + return configured; + } + + try { + return execFileSync("git", ["config", "user.email"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim() || "unknown"; + } catch { + return "unknown"; + } +} + +function formatLlmJudge(entry: ReviewQueueEntry): string { + if (entry.llmJudge.skipped) { + return `skipped=${entry.llmJudge.reason}`; + } + + return [ + `overall=${entry.llmJudge.scores.overall}`, + `friendliness=${entry.llmJudge.scores.friendliness}`, + `accuracy=${entry.llmJudge.scores.accuracy}`, + `concision=${entry.llmJudge.scores.concision}`, + `scope=${entry.llmJudge.scores.scope}`, + `rationale=${entry.llmJudge.rationale}`, + ].join(" "); +} + +function formatProcess(entry: ReviewQueueEntry): string { + const steps = entry.process?.steps; + if (!steps) { + return "n/a"; + } + + return [ + steps.classify ? "classify" : null, + steps.derive ? "derive" : null, + steps.plan ? "plan" : null, + ...(steps.toolCalls?.map((call) => `tool:${call.output.name}`) ?? []), + steps.respond ? "respond" : null, + steps.critic ? "critic" : null, + steps.validate ? "validate" : null, + ] + .filter((value): value is string => Boolean(value)) + .join(" -> "); +} + +function printReviewItem( + entry: ReviewQueueEntry, + index: number, + limit: number, +): void { + console.info(""); + console.info(`[${index}/${limit}] reason: ${entry.reason}`); + console.info( + `chatroom: ${entry.request.chatroomId} / user: ${entry.request.userId} / trigger: ${entry.request.trigger}`, + ); + console.info( + `intent: ${entry.classified.intent.kind} (confidence ${entry.classified.confidence}, ${entry.classified.source})`, + ); + console.info( + `tier: ${entry.derived.tier} / phase: ${entry.derived.phase.kind} / responseMode: ${entry.derived.responseMode}`, + ); + console.info(""); + console.info("[user message]"); + console.info(entry.request.userMessage ?? "(no user message)"); + console.info(""); + console.info("[assistant response]"); + console.info(entry.response.text); + console.info(""); + console.info(`[llm judge] ${formatLlmJudge(entry)}`); + console.info(`[validator] status=${entry.validation.status} reasons=${entry.validation.reasons.join(",") || "-"}`); + console.info(`[process] ${formatProcess(entry)}`); + console.info(""); +} + +async function askVerdict( + rl: ReturnType, +): Promise<"fail" | "pass" | "quit" | "skip"> { + while (true) { + const answer = (await rl.question("verdict (p=pass / f=fail / s=skip / q=quit) > ")) + .trim() + .toLowerCase(); + if (answer === "p") return "pass"; + if (answer === "f") return "fail"; + if (answer === "s") return "skip"; + if (answer === "q") return "quit"; + } +} + +async function askHumanScore( + rl: ReturnType, +): Promise<1 | 2 | 3 | 4 | 5> { + while (true) { + const answer = Number((await rl.question("humanScore (1-5) > ")).trim()); + if (answer === 1 || answer === 2 || answer === 3 || answer === 4 || answer === 5) { + return answer; + } + } +} + +async function askNonEmpty( + rl: ReturnType, + prompt: string, +): Promise { + while (true) { + const answer = (await rl.question(prompt)).trim(); + if (answer) { + return answer; + } + } +} + +async function exportLabels( + labels: ReviewLabel[], + outputPath: string, +): Promise { + const resolved = resolve(process.cwd(), outputPath); + await mkdir(dirname(resolved), { recursive: true }); + await writeFile( + resolved, + labels.map((label) => JSON.stringify(label)).join("\n") + "\n", + "utf8", + ); + return resolved; +} + +function isReviewReason(value: string | undefined): value is ReviewReason { + return ( + value === "low-score" || + value === "rare-slice" || + value === "rule-llm-disagree" || + value === "validator-soft-warn" + ); +} + +export function parseReviewReason(value: string | undefined): ReviewReason | undefined { + return isReviewReason(value) ? value : undefined; +} + +export async function runReviewCli( + garnetClient: Pick< + GarnetClient, + | "acknowledgeStream" + | "appendStream" + | "claimPendingStream" + | "createConsumerGroup" + | "getPendingStreamRange" + | "readGroup" + >, + options?: ReviewCliOptions, +): Promise<{ labeled: number; skipped: number }> { + const rl = createInterface({ input, output }); + const limit = options?.limit ?? 20; + const consumerName = options?.consumerName ?? `review-cli-${process.pid}`; + const reviewedBy = getReviewerId(); + const labels: ReviewLabel[] = []; + let skipped = 0; + let mismatchAttempts = 0; + + try { + if (options?.resetPending) { + const result = await resetPendingReviewItems(garnetClient, { + consumerName, + nowMs: Date.now(), + }); + console.info(`[eval] resetPending=${result.resetCount}`); + } + + for (let index = 1; index <= limit; index++) { + const item = await fetchNextReviewItem(garnetClient as ReviewConsumerGarnetClient, { + consumerName, + blockMs: 5_000, + }); + if (!item) { + break; + } + + if (options?.reason && item.entry.reason !== options.reason) { + await enqueueReviewCandidate(garnetClient, { + ...item.entry, + enqueuedAt: new Date().toISOString(), + }); + await acknowledgeReviewItem(garnetClient, item.streamId); + mismatchAttempts += 1; + if (mismatchAttempts >= limit * 5) { + break; + } + index -= 1; + continue; + } + + printReviewItem(item.entry, index, limit); + const verdict = await askVerdict(rl); + if (verdict === "quit") { + break; + } + if (verdict === "skip") { + await acknowledgeReviewItem(garnetClient, item.streamId); + skipped += 1; + continue; + } + + const humanScore = await askHumanScore(rl); + const rationale = await askNonEmpty(rl, "rationale > "); + const betterResponse = (await rl.question("better (optional) > ")).trim(); + const label: ReviewLabel = { + evalStreamId: item.entry.evalStreamId, + verdict, + humanScore, + rationale, + reviewedBy, + reviewedAt: new Date().toISOString(), + }; + if (betterResponse) { + label.betterResponse = betterResponse; + } + + await recordReviewLabel(garnetClient, { + consumerName, + label, + streamId: item.streamId, + }); + labels.push(label); + } + } finally { + rl.close(); + } + + if (options?.exportLabeled && labels.length > 0) { + const outputPath = await exportLabels(labels, options.exportLabeled); + console.info(`[eval] exported labeled=${labels.length} output=${outputPath}`); + } + + return { + labeled: labels.length, + skipped, + }; +} diff --git a/src/eval/review/label-archive.test.ts b/src/eval/review/label-archive.test.ts new file mode 100644 index 0000000..87057b6 --- /dev/null +++ b/src/eval/review/label-archive.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from "vitest"; + +import type { StreamEntry } from "@common/garnet"; +import { + archiveReviewLabels, + EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION, + EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, + queryArchivedReviewLabels, + type ReviewLabelArchiveEntry, +} from "./label-archive.js"; +import { EVAL_REVIEW_LABELED_STREAM_KEY, type ReviewLabel } from "./queue.js"; + +class MemoryGarnet { + readonly strings = new Map(); + + constructor(private readonly entries: StreamEntry[]) {} + + readStreamRange( + key: string, + start: string, + _end: string, + options?: { count?: number | undefined }, + ): Promise { + expect(key).toBe(EVAL_REVIEW_LABELED_STREAM_KEY); + const filtered = start.startsWith("(") + ? this.entries.filter((entry) => entry.id > start.slice(1)) + : this.entries; + return Promise.resolve(filtered.slice(0, options?.count)); + } + + setString(key: string, value: string): Promise { + this.strings.set(key, value); + return Promise.resolve(); + } +} + +class MemoryCursor { + constructor(private readonly entries: ReviewLabelArchiveEntry[]) {} + + sort(): MemoryCursor { + this.entries.sort( + (left, right) => right.reviewedAt.getTime() - left.reviewedAt.getTime(), + ); + return this; + } + + limit(value: number): MemoryCursor { + this.entries.splice(value); + return this; + } + + toArray(): Promise { + return Promise.resolve(this.entries); + } +} + +class MemoryCollection { + readonly docs = new Map(); + readonly indexes: Array> = []; + + bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> { + let upsertedCount = 0; + for (const operation of operations) { + const updateOne = (operation as { + updateOne: { + filter: { labelStreamId: string }; + update: { $setOnInsert: ReviewLabelArchiveEntry }; + }; + }).updateOne; + const streamId = updateOne.filter.labelStreamId; + if (!this.docs.has(streamId)) { + this.docs.set(streamId, updateOne.update.$setOnInsert); + upsertedCount += 1; + } + } + return Promise.resolve({ upsertedCount }); + } + + createIndex(keys: Record): Promise { + this.indexes.push(keys); + return Promise.resolve("idx"); + } + + find(filter: Record): MemoryCursor { + const reviewedAt = filter["reviewedAt"] as + | { $gte?: Date | undefined } + | undefined; + const entries = [...this.docs.values()].filter((entry) => { + if (!reviewedAt?.$gte) { + return true; + } + return entry.reviewedAt >= reviewedAt.$gte; + }); + return new MemoryCursor(entries); + } +} + +class MemoryConnection { + readonly collectionInstance = new MemoryCollection(); + + collection(name: string): MemoryCollection { + expect(name).toBe(EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION); + return this.collectionInstance; + } +} + +function streamEntry(id: string, label: ReviewLabel): StreamEntry { + return { + id, + message: { + type: "eval-review-label", + evalStreamId: label.evalStreamId, + reviewedAt: label.reviewedAt, + reviewedBy: label.reviewedBy, + payload: JSON.stringify(label), + }, + }; +} + +describe("archiveReviewLabels", () => { + it("archives label stream entries idempotently and queries them back", async () => { + const label: ReviewLabel = { + evalStreamId: "1-0", + verdict: "fail", + humanScore: 2, + rationale: "wrong channel", + reviewedBy: "tester", + reviewedAt: "2026-05-01T00:01:00.000Z", + }; + const garnet = new MemoryGarnet([streamEntry("1-0", label)]); + const connection = new MemoryConnection(); + + const first = await archiveReviewLabels(garnet, connection, { + sinceId: "0-0", + nowMs: Date.parse("2026-05-01T00:02:00.000Z"), + }); + const second = await archiveReviewLabels(garnet, connection, { + sinceId: "0-0", + nowMs: Date.parse("2026-05-01T00:02:00.000Z"), + }); + const labels = await queryArchivedReviewLabels(connection); + + expect(first).toEqual({ archivedCount: 1, lastArchivedId: "1-0" }); + expect(second).toEqual({ archivedCount: 0, lastArchivedId: "1-0" }); + expect(garnet.strings.get(EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY)).toBe("1-0"); + expect(connection.collectionInstance.indexes).toContainEqual({ + labelStreamId: 1, + }); + expect(labels).toEqual([label]); + }); +}); diff --git a/src/eval/review/label-archive.ts b/src/eval/review/label-archive.ts new file mode 100644 index 0000000..4a6abc3 --- /dev/null +++ b/src/eval/review/label-archive.ts @@ -0,0 +1,188 @@ +import type { GarnetClient, StreamEntry } from "@common/garnet"; +import { getEvalArchiveBatchSize } from "../../config.js"; +import type { MongoConnectionLike } from "../archive.js"; +import { + EVAL_REVIEW_LABELED_STREAM_KEY, + parseReviewLabelEntry, + type ReviewLabel, +} from "./queue.js"; + +export const EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY = + "lolympus-agent:eval:review:label-archive:cursor"; +export const EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION = "eval_review_labels"; + +export interface ReviewLabelArchiveEntry + extends Omit { + labelStreamId: string; + reviewedAt: Date; +} + +interface BulkWriteResultLike { + upsertedCount?: number | undefined; +} + +interface FindCursorLike { + limit(value: number): FindCursorLike; + sort(value: Record): FindCursorLike; + toArray(): Promise; +} + +interface ReviewLabelArchiveCollectionLike { + bulkWrite( + operations: unknown[], + options?: { ordered?: boolean | undefined }, + ): Promise; + createIndex( + keys: Record, + options?: Record, + ): Promise; + find(filter: Record): FindCursorLike; +} + +type LabelArchiveGarnetClient = Pick< + GarnetClient, + "readStreamRange" | "setString" +>; + +function streamRangeStart(sinceId: string): string { + const trimmed = sinceId.trim(); + if (!trimmed || trimmed === "0-0" || trimmed === "-") { + return "-"; + } + return `(${trimmed}`; +} + +function parseReviewedAt(value: string, nowMs: number): Date { + const parsed = new Date(value); + return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs); +} + +function toArchiveEntry( + streamEntry: StreamEntry, + nowMs: number, +): ReviewLabelArchiveEntry { + const label = parseReviewLabelEntry(streamEntry); + return { + ...label, + labelStreamId: streamEntry.id, + reviewedAt: parseReviewedAt(label.reviewedAt, nowMs), + }; +} + +async function ensureReviewLabelArchiveIndexes( + collection: ReviewLabelArchiveCollectionLike, +): Promise { + await Promise.all([ + collection.createIndex({ labelStreamId: 1 }, { unique: true }), + collection.createIndex({ evalStreamId: 1 }), + collection.createIndex({ reviewedAt: -1 }), + ]); +} + +export function getReviewLabelArchiveCollection( + mongoConnection: MongoConnectionLike, +): ReviewLabelArchiveCollectionLike { + return mongoConnection.collection( + EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION, + ) as ReviewLabelArchiveCollectionLike; +} + +export async function archiveReviewLabels( + garnetClient: LabelArchiveGarnetClient, + mongoConnection: MongoConnectionLike, + options: { + batchSize?: number | undefined; + nowMs: number; + sinceId: string; + }, +): Promise<{ archivedCount: number; lastArchivedId: string }> { + const collection = getReviewLabelArchiveCollection(mongoConnection); + await ensureReviewLabelArchiveIndexes(collection); + + const entries = await garnetClient.readStreamRange( + EVAL_REVIEW_LABELED_STREAM_KEY, + streamRangeStart(options.sinceId), + "+", + { + count: options.batchSize ?? getEvalArchiveBatchSize(), + }, + ); + + if (entries.length === 0) { + return { + archivedCount: 0, + lastArchivedId: options.sinceId, + }; + } + + const lastArchivedId = entries.at(-1)?.id ?? options.sinceId; + const archiveEntries = entries.flatMap((entry) => { + try { + return [toArchiveEntry(entry, options.nowMs)]; + } catch { + return []; + } + }); + + let archivedCount = 0; + if (archiveEntries.length > 0) { + const result = await collection.bulkWrite( + archiveEntries.map((entry) => ({ + updateOne: { + filter: { labelStreamId: entry.labelStreamId }, + update: { $setOnInsert: entry }, + upsert: true, + }, + })), + { ordered: false }, + ); + archivedCount = result.upsertedCount ?? 0; + } + + await garnetClient.setString( + EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, + lastArchivedId, + ); + + return { + archivedCount, + lastArchivedId, + }; +} + +export async function queryArchivedReviewLabels( + mongoConnection: MongoConnectionLike, + options?: { + limit?: number | undefined; + nowMs?: number | undefined; + sinceMs?: number | undefined; + }, +): Promise { + const filter: Record = {}; + if (options?.sinceMs !== undefined) { + filter["reviewedAt"] = { + $gte: new Date((options.nowMs ?? Date.now()) - options.sinceMs), + }; + } + + const entries = await getReviewLabelArchiveCollection(mongoConnection) + .find(filter) + .sort({ reviewedAt: -1 }) + .limit(options?.limit ?? 10_000) + .toArray(); + + return entries.map((entry) => { + const label: ReviewLabel = { + evalStreamId: entry.evalStreamId, + verdict: entry.verdict, + humanScore: entry.humanScore, + rationale: entry.rationale, + reviewedBy: entry.reviewedBy, + reviewedAt: entry.reviewedAt.toISOString(), + }; + if (entry.betterResponse !== undefined) { + label.betterResponse = entry.betterResponse; + } + return label; + }); +} diff --git a/src/eval/review/queue.test.ts b/src/eval/review/queue.test.ts new file mode 100644 index 0000000..2a47164 --- /dev/null +++ b/src/eval/review/queue.test.ts @@ -0,0 +1,301 @@ +import { describe, expect, it } from "vitest"; + +import type { StreamEntry, StreamReadResult } from "@common/garnet"; +import { + EVAL_REVIEW_CURSOR_KEY, + EVAL_REVIEW_GROUP, + EVAL_REVIEW_LABELED_STREAM_KEY, + EVAL_REVIEW_QUEUE_STREAM_KEY, + enqueueReviewCandidate, + fetchNextReviewItem, + recordReviewLabel, + resetPendingReviewItems, + routeReviewCandidates, + type ReviewQueueEntry, +} from "./queue.js"; +import { EVAL_RESULTS_STREAM_KEY, getSliceAggregateKey } from "../aggregator.js"; + +class MemoryGarnet { + readonly acknowledged: Array<{ group: string; id: string; key: string }> = []; + readonly dedupe = new Set(); + readonly groups = new Set(); + readonly hashes = new Map>(); + readonly streams = new Map(); + readonly strings = new Map(); + + acknowledgeStream(key: string, group: string, ids: string | string[]): Promise { + for (const id of Array.isArray(ids) ? ids : [ids]) { + this.acknowledged.push({ key, group, id }); + } + return Promise.resolve(Array.isArray(ids) ? ids.length : 1); + } + + appendStream(key: string, fields: Record): Promise { + const entries = this.streams.get(key) ?? []; + const id = `${entries.length + 1}-0`; + entries.push({ id, message: fields }); + this.streams.set(key, entries); + return Promise.resolve(id); + } + + createConsumerGroup(key: string, group: string): Promise { + this.groups.add(`${key}:${group}`); + return Promise.resolve(); + } + + claimPendingStream( + key: string, + _group: string, + _consumer: string, + _minIdleTimeMs: number, + ids: string | string[], + ): Promise { + const idSet = new Set(Array.isArray(ids) ? ids : [ids]); + return Promise.resolve( + (this.streams.get(key) ?? []).filter((entry) => idSet.has(entry.id)), + ); + } + + getHash(key: string): Promise> { + return Promise.resolve({ ...(this.hashes.get(key) ?? {}) }); + } + + getPendingStreamRange( + key: string, + group: string, + count: number, + ): Promise< + Array<{ + consumer: string; + deliveriesCounter: number; + id: string; + millisecondsSinceLastDelivery: number; + }> + > { + const acked = new Set( + this.acknowledged + .filter((entry) => entry.key === key && entry.group === group) + .map((entry) => entry.id), + ); + return Promise.resolve( + (this.streams.get(key) ?? []) + .filter((entry) => !acked.has(entry.id)) + .slice(0, count) + .map((entry) => ({ + consumer: "old-reviewer", + deliveriesCounter: 1, + id: entry.id, + millisecondsSinceLastDelivery: 3_600_000, + })), + ); + } + + readGroup( + key: string, + group: string, + _consumer: string, + ): Promise { + const entries = this.streams.get(key) ?? []; + const acked = new Set( + this.acknowledged + .filter((entry) => entry.key === key && entry.group === group) + .map((entry) => entry.id), + ); + const next = entries.find((entry) => !acked.has(entry.id)); + return Promise.resolve( + next + ? [ + { + name: key, + messages: [next], + }, + ] + : null, + ); + } + + readStreamRange(key: string, start: string): Promise { + const entries = this.streams.get(key) ?? []; + if (!start.startsWith("(")) { + return Promise.resolve(entries); + } + const sinceId = start.slice(1); + return Promise.resolve(entries.filter((entry) => entry.id > sinceId)); + } + + setString(key: string, value: string): Promise { + this.strings.set(key, value); + return Promise.resolve(); + } + + setStringIfAbsent(key: string): Promise { + if (this.dedupe.has(key)) { + return Promise.resolve(false); + } + this.dedupe.add(key); + return Promise.resolve(true); + } +} + +function reviewEntry(): ReviewQueueEntry { + return { + evalStreamId: "1-0", + reason: "low-score", + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "닉", + trigger: "user-message", + userMessage: "사진 어디로 보내요?", + }, + response: { + requestId: "req-1", + text: "디스코드에 보내주세요.", + confidence: 0.9, + source: "llm", + }, + derived: { + phase: { kind: "photo-pending" }, + intent: { kind: "photo-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + }, + classified: { + intent: { kind: "photo-question" }, + confidence: 0.9, + source: "regex", + }, + validation: { + status: "pass", + text: "디스코드에 보내주세요.", + reasons: [], + }, + llmJudge: { + skipped: false, + model: "judge", + endpoint: "local", + provider: "cliproxy", + scores: { + friendliness: 4, + accuracy: 1, + concision: 4, + scope: 2, + overall: 2.75, + }, + rationale: "wrong channel", + raw: "{}", + }, + enqueuedAt: "2026-05-01T00:00:00.000Z", + }; +} + +function evalResultEntry(id: string, overall: number): StreamEntry { + const entry = reviewEntry(); + const llmJudge = entry.llmJudge.skipped + ? entry.llmJudge + : { + ...entry.llmJudge, + scores: { + ...entry.llmJudge.scores, + overall, + }, + }; + + return { + id, + message: { + input: JSON.stringify({ request: entry.request }), + output: JSON.stringify(entry.response), + derived: JSON.stringify(entry.derived), + classified: JSON.stringify(entry.classified), + validation: JSON.stringify(entry.validation), + llmJudge: JSON.stringify(llmJudge), + }, + }; +} + +describe("review queue", () => { + it("enqueues, fetches, labels, and acknowledges review items", async () => { + const garnet = new MemoryGarnet(); + const streamId = await enqueueReviewCandidate(garnet, reviewEntry()); + + const item = await fetchNextReviewItem(garnet, { + consumerName: "reviewer-1", + blockMs: 1, + }); + expect(streamId).toBe("1-0"); + expect(item?.entry.reason).toBe("low-score"); + expect(garnet.groups.has(`${EVAL_REVIEW_QUEUE_STREAM_KEY}:${EVAL_REVIEW_GROUP}`)).toBe(true); + + await recordReviewLabel(garnet, { + consumerName: "reviewer-1", + streamId, + label: { + evalStreamId: "1-0", + verdict: "fail", + humanScore: 1, + rationale: "wrong channel", + reviewedBy: "tester", + reviewedAt: "2026-05-01T00:01:00.000Z", + }, + }); + + expect(garnet.streams.get(EVAL_REVIEW_LABELED_STREAM_KEY)).toHaveLength(1); + expect(garnet.acknowledged).toContainEqual({ + key: EVAL_REVIEW_QUEUE_STREAM_KEY, + group: EVAL_REVIEW_GROUP, + id: streamId, + }); + }); + + it("routes low-score eval results once with dedupe and cursor tracking", async () => { + const garnet = new MemoryGarnet(); + const sliceKey = getSliceAggregateKey({ + tier: 0, + intent: "photo-question", + phase: "photo-pending", + }); + garnet.hashes.set(sliceKey, { count: "10" }); + garnet.streams.set(EVAL_RESULTS_STREAM_KEY, [ + evalResultEntry("1-0", 2), + evalResultEntry("2-0", 4), + ]); + + const first = await routeReviewCandidates(garnet, { + sinceId: "0-0", + nowMs: Date.parse("2026-05-01T00:00:00.000Z"), + }); + const second = await routeReviewCandidates(garnet, { + sinceId: "0-0", + nowMs: Date.parse("2026-05-01T00:00:00.000Z"), + }); + + expect(first.enqueuedCount).toBe(1); + expect(second.enqueuedCount).toBe(0); + expect(garnet.strings.get(EVAL_REVIEW_CURSOR_KEY)).toBe("2-0"); + expect(garnet.streams.get(EVAL_REVIEW_QUEUE_STREAM_KEY)).toHaveLength(1); + }); + + it("resets stale pending review items by re-enqueueing and acknowledging originals", async () => { + const garnet = new MemoryGarnet(); + const streamId = await enqueueReviewCandidate(garnet, reviewEntry()); + + const result = await resetPendingReviewItems(garnet, { + consumerName: "reviewer-1", + nowMs: Date.parse("2026-05-01T00:02:00.000Z"), + }); + + expect(result.resetCount).toBe(1); + expect(garnet.acknowledged).toContainEqual({ + key: EVAL_REVIEW_QUEUE_STREAM_KEY, + group: EVAL_REVIEW_GROUP, + id: streamId, + }); + expect(garnet.streams.get(EVAL_REVIEW_QUEUE_STREAM_KEY)).toHaveLength(2); + }); +}); diff --git a/src/eval/review/queue.ts b/src/eval/review/queue.ts new file mode 100644 index 0000000..e80d7fc --- /dev/null +++ b/src/eval/review/queue.ts @@ -0,0 +1,506 @@ +import type { AgentRequest, AgentResponse } from "@agent/types"; +import type { GarnetClient, StreamEntry } from "@common/garnet"; +import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; +import type { ValidationResult } from "@skills/onboarding/runtime/validator"; +import type { + ClassifiedIntent, + DerivedState, +} from "@skills/onboarding/state/onboarding-state"; +import { + getEvalReviewDedupeTtlSec, + getEvalReviewLabelMaxLen, + getEvalReviewPerCategoryLimit, + getEvalReviewQueueMaxLen, +} from "../../config.js"; +import { + EVAL_RESULTS_STREAM_KEY, + getSliceAggregateKey, + type SliceKey, +} from "../aggregator.js"; +import type { LlmJudgeResult, ProcessJudgeResult } from "../types.js"; + +export const EVAL_REVIEW_QUEUE_STREAM_KEY = + "lolympus-agent:eval:review:queue"; +export const EVAL_REVIEW_LABELED_STREAM_KEY = + "lolympus-agent:eval:review:labeled"; +export const EVAL_REVIEW_CURSOR_KEY = "lolympus-agent:eval:review:cursor"; +export const EVAL_REVIEW_GROUP = "reviewers"; + +export type ReviewReason = + | "low-score" + | "rare-slice" + | "rule-llm-disagree" + | "validator-soft-warn"; + +export interface ReviewQueueEntry { + evalStreamId: string; + reason: ReviewReason; + request: AgentRequest; + response: AgentResponse; + derived: DerivedState; + classified: ClassifiedIntent; + validation: ValidationResult; + llmJudge: LlmJudgeResult; + processJudge?: ProcessJudgeResult | undefined; + process?: ProcessTraceSnapshot | undefined; + enqueuedAt: string; +} + +export interface ReviewLabel { + betterResponse?: string | undefined; + evalStreamId: string; + humanScore: 1 | 2 | 3 | 4 | 5; + rationale: string; + reviewedAt: string; + reviewedBy: string; + verdict: "fail" | "pass"; +} + +type ReviewRouteGarnetClient = Pick< + GarnetClient, + | "acknowledgeStream" + | "appendStream" + | "createConsumerGroup" + | "getHash" + | "readGroup" + | "readStreamRange" + | "setString" + | "setStringIfAbsent" +>; + +export type ReviewConsumerGarnetClient = Pick< + GarnetClient, + "createConsumerGroup" | "readGroup" +>; + +type ReviewPendingGarnetClient = Pick< + GarnetClient, + | "acknowledgeStream" + | "appendStream" + | "claimPendingStream" + | "createConsumerGroup" + | "getPendingStreamRange" +>; + +interface ParsedEvalResult { + classified: ClassifiedIntent; + derived: DerivedState; + evalStreamId: string; + llmJudge: LlmJudgeResult; + process?: ProcessTraceSnapshot | undefined; + processJudge?: ProcessJudgeResult | undefined; + request: AgentRequest; + response: AgentResponse; + validation: ValidationResult; +} + +function parseJsonField( + value: string | undefined, + fieldName: string, +): T { + if (!value?.trim()) { + throw new Error(`Missing ${fieldName} field`); + } + + return JSON.parse(value) as T; +} + +function parseOptionalJsonField(value: string | undefined): T | undefined { + if (!value?.trim()) { + return undefined; + } + + return JSON.parse(value) as T; +} + +function parseFiniteNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value !== "string" || !value.trim()) { + return null; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function streamRangeStart(sinceId: string): string { + const trimmed = sinceId.trim(); + if (!trimmed || trimmed === "0-0" || trimmed === "-") { + return "-"; + } + return `(${trimmed}`; +} + +function getLlmOverall(llmJudge: LlmJudgeResult): number | null { + return llmJudge.skipped ? null : llmJudge.scores.overall; +} + +function sliceFromDerived(derived: DerivedState): SliceKey { + return { + tier: derived.tier, + intent: derived.intent.kind, + phase: derived.phase.kind, + }; +} + +function parseEvalResult(entry: StreamEntry): ParsedEvalResult | null { + try { + const input = parseJsonField<{ request: AgentRequest }>( + entry.message["input"], + "input", + ); + const parsed: ParsedEvalResult = { + classified: parseJsonField( + entry.message["classified"], + "classified", + ), + derived: parseJsonField( + entry.message["derived"], + "derived", + ), + evalStreamId: entry.id, + llmJudge: parseJsonField( + entry.message["llmJudge"], + "llmJudge", + ), + request: input.request, + response: parseJsonField( + entry.message["output"], + "output", + ), + validation: parseJsonField( + entry.message["validation"], + "validation", + ), + }; + const processJudge = parseOptionalJsonField( + entry.message["processJudge"], + ); + const process = parseOptionalJsonField( + entry.message["process"], + ); + if (processJudge) { + parsed.processJudge = processJudge; + } + if (process) { + parsed.process = process; + } + return parsed; + } catch { + return null; + } +} + +function toQueueEntry( + parsed: ParsedEvalResult, + reason: ReviewReason, + nowMs: number, +): ReviewQueueEntry { + const entry: ReviewQueueEntry = { + evalStreamId: parsed.evalStreamId, + reason, + request: parsed.request, + response: parsed.response, + derived: parsed.derived, + classified: parsed.classified, + validation: parsed.validation, + llmJudge: parsed.llmJudge, + enqueuedAt: new Date(nowMs).toISOString(), + }; + if (parsed.processJudge) { + entry.processJudge = parsed.processJudge; + } + if (parsed.process) { + entry.process = parsed.process; + } + return entry; +} + +function getCandidateReasons(parsed: ParsedEvalResult): ReviewReason[] { + const reasons: ReviewReason[] = []; + const overall = getLlmOverall(parsed.llmJudge); + + if (overall !== null && overall <= 2) { + reasons.push("low-score"); + } + if ( + overall !== null && + ((parsed.validation.status === "pass" && overall <= 3) || + (parsed.validation.status === "reject" && overall >= 4)) + ) { + reasons.push("rule-llm-disagree"); + } + if (parsed.validation.status === "soft-warn") { + reasons.push("validator-soft-warn"); + } + + return reasons; +} + +function reasonLimitKey(reason: ReviewReason): ReviewReason { + return reason; +} + +async function isRareSlice( + garnetClient: Pick, + parsed: ParsedEvalResult, +): Promise { + const aggregate = await garnetClient.getHash( + getSliceAggregateKey(sliceFromDerived(parsed.derived)), + ); + const count = parseFiniteNumber(aggregate["count"]) ?? 0; + return count > 0 && count <= 5; +} + +async function ensureReviewConsumerGroup( + garnetClient: Pick, +): Promise { + try { + await garnetClient.createConsumerGroup( + EVAL_REVIEW_QUEUE_STREAM_KEY, + EVAL_REVIEW_GROUP, + "0", + ); + } catch (error) { + if (!String(error).includes("BUSYGROUP")) { + throw error; + } + } +} + +export async function enqueueReviewCandidate( + garnetClient: Pick, + entry: ReviewQueueEntry, +): Promise { + return garnetClient.appendStream( + EVAL_REVIEW_QUEUE_STREAM_KEY, + { + type: "eval-review", + evalStreamId: entry.evalStreamId, + reason: entry.reason, + payload: JSON.stringify(entry), + enqueuedAt: entry.enqueuedAt, + }, + { + maxLen: getEvalReviewQueueMaxLen(), + }, + ); +} + +export function parseReviewQueueEntry( + streamEntry: StreamEntry, +): ReviewQueueEntry { + return parseJsonField( + streamEntry.message["payload"], + "payload", + ); +} + +export function parseReviewLabelEntry(streamEntry: StreamEntry): ReviewLabel { + return parseJsonField(streamEntry.message["payload"], "payload"); +} + +export async function fetchNextReviewItem( + garnetClient: ReviewConsumerGarnetClient, + options: { + blockMs?: number | undefined; + consumerName: string; + }, +): Promise<{ entry: ReviewQueueEntry; streamId: string } | null> { + await ensureReviewConsumerGroup(garnetClient); + const streams = await garnetClient.readGroup( + EVAL_REVIEW_QUEUE_STREAM_KEY, + EVAL_REVIEW_GROUP, + options.consumerName, + ">", + { + blockMs: options.blockMs ?? 5_000, + count: 1, + }, + ); + const message = streams?.[0]?.messages[0]; + if (!message) { + return null; + } + + return { + streamId: message.id, + entry: parseReviewQueueEntry(message), + }; +} + +export async function acknowledgeReviewItem( + garnetClient: Pick, + streamId: string, +): Promise { + await garnetClient.acknowledgeStream( + EVAL_REVIEW_QUEUE_STREAM_KEY, + EVAL_REVIEW_GROUP, + streamId, + ); +} + +export async function recordReviewLabel( + garnetClient: Pick, + options: { + consumerName: string; + label: ReviewLabel; + streamId: string; + }, +): Promise { + await garnetClient.appendStream( + EVAL_REVIEW_LABELED_STREAM_KEY, + { + type: "eval-review-label", + evalStreamId: options.label.evalStreamId, + reviewedAt: options.label.reviewedAt, + reviewedBy: options.label.reviewedBy, + payload: JSON.stringify(options.label), + }, + { + maxLen: getEvalReviewLabelMaxLen(), + }, + ); + await acknowledgeReviewItem(garnetClient, options.streamId); +} + +export async function resetPendingReviewItems( + garnetClient: ReviewPendingGarnetClient, + options: { + consumerName: string; + count?: number | undefined; + idleMs?: number | undefined; + nowMs: number; + }, +): Promise<{ resetCount: number }> { + await ensureReviewConsumerGroup(garnetClient); + const pending = await garnetClient.getPendingStreamRange( + EVAL_REVIEW_QUEUE_STREAM_KEY, + EVAL_REVIEW_GROUP, + options.count ?? 100, + { + idleMs: options.idleMs ?? 60 * 60_000, + }, + ); + let resetCount = 0; + + for (const entry of pending) { + const claimed = await garnetClient.claimPendingStream( + EVAL_REVIEW_QUEUE_STREAM_KEY, + EVAL_REVIEW_GROUP, + options.consumerName, + options.idleMs ?? 60 * 60_000, + entry.id, + ); + const message = claimed[0]; + if (!message) { + continue; + } + + try { + const reviewEntry = parseReviewQueueEntry(message); + await enqueueReviewCandidate(garnetClient, { + ...reviewEntry, + enqueuedAt: new Date(options.nowMs).toISOString(), + }); + await acknowledgeReviewItem(garnetClient, message.id); + resetCount += 1; + } catch { + await acknowledgeReviewItem(garnetClient, message.id); + } + } + + return { + resetCount, + }; +} + +export async function readReviewLabels( + garnetClient: Pick, +): Promise { + const entries = await garnetClient.readStreamRange( + EVAL_REVIEW_LABELED_STREAM_KEY, + "-", + "+", + ); + return entries.flatMap((entry) => { + try { + return [parseReviewLabelEntry(entry)]; + } catch { + return []; + } + }); +} + +export async function routeReviewCandidates( + garnetClient: ReviewRouteGarnetClient, + options: { + nowMs: number; + sinceId: string; + }, +): Promise<{ enqueuedCount: number; lastSeenId: string }> { + const entries = await garnetClient.readStreamRange( + EVAL_RESULTS_STREAM_KEY, + streamRangeStart(options.sinceId), + "+", + ); + + if (entries.length === 0) { + return { + enqueuedCount: 0, + lastSeenId: options.sinceId, + }; + } + + const perCategoryLimit = getEvalReviewPerCategoryLimit(); + const counts = new Map(); + let enqueuedCount = 0; + let lastSeenId = options.sinceId; + + for (const entry of entries) { + lastSeenId = entry.id; + const parsed = parseEvalResult(entry); + if (!parsed) { + continue; + } + + const reasons = getCandidateReasons(parsed); + if (await isRareSlice(garnetClient, parsed)) { + reasons.push("rare-slice"); + } + + for (const reason of reasons) { + const key = reasonLimitKey(reason); + if ((counts.get(key) ?? 0) >= perCategoryLimit) { + continue; + } + + const dedupeKey = `lolympus-agent:eval:review:dedupe:${entry.id}`; + const claimed = await garnetClient.setStringIfAbsent( + dedupeKey, + reason, + getEvalReviewDedupeTtlSec(), + ); + if (!claimed) { + continue; + } + + await enqueueReviewCandidate( + garnetClient, + toQueueEntry(parsed, reason, options.nowMs), + ); + counts.set(key, (counts.get(key) ?? 0) + 1); + enqueuedCount += 1; + break; + } + } + + await garnetClient.setString(EVAL_REVIEW_CURSOR_KEY, lastSeenId); + + return { + enqueuedCount, + lastSeenId, + }; +} diff --git a/src/eval/runner.ts b/src/eval/runner.ts index b80cf06..7cedaf6 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -14,6 +14,10 @@ import { getLogger } from "@obs/logger"; import type { RoomMessage } from "@room/types"; import { RoomContextService } from "@room/room-context-service"; import { OnboardingSkill } from "@skills/onboarding/index"; +import { + ProcessTraceCollector, + runWithProcessTrace, +} from "@skills/onboarding/runtime/trace"; import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; import { OnboardingIntentClassifier } from "@skills/onboarding/state/classifier"; import { deriveState } from "@skills/onboarding/state/derive"; @@ -24,6 +28,7 @@ import type { import { noopShadowEvalPublisher } from "./shadow-bus.js"; import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js"; import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; +import { judgeProcessRule } from "./judges/process-judge.js"; import { judgeOnboardingRuleCase } from "./judges/rule-judge.js"; import type { EvalBaselineFile, @@ -182,7 +187,7 @@ function toProviderEndpoints( return urls.map((url) => ({ url, provider })); } -function createEvalLlmClient(): LlmClient { +export function createEvalLlmClient(): LlmClient { const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]); const llamaEndpoints = parseEndpointList( process.env["AGENT_LLM_API_URLS"], @@ -336,6 +341,41 @@ function parseExpected(raw: Record | undefined): EvalCaseExpect expected.mustNotInclude = mustNotInclude; } + const processExpectedRaw = raw["processExpected"]; + if (isRecord(processExpectedRaw)) { + const expectedTools = getStringArrayField( + processExpectedRaw, + "expectedTools", + ); + const expectedApproach = getStringField( + processExpectedRaw, + "expectedApproach", + ); + const toolModeRaw = getStringField(processExpectedRaw, "toolMode"); + const minStepsRaw = processExpectedRaw["minSteps"]; + const processExpected: NonNullable = {}; + + if (expectedTools) { + processExpected.expectedTools = expectedTools; + } + if (expectedApproach) { + processExpected.expectedApproach = expectedApproach; + } + if (toolModeRaw === "exact" || toolModeRaw === "subset") { + processExpected.toolMode = toolModeRaw; + } + if ( + typeof minStepsRaw === "number" && + Number.isInteger(minStepsRaw) && + minStepsRaw > 0 + ) { + processExpected.minSteps = minStepsRaw; + } + if (Object.keys(processExpected).length > 0) { + expected.processExpected = processExpected; + } + } + return expected; } @@ -605,6 +645,7 @@ async function evaluateCase( noopShadowEvalPublisher, ); const classifier = new OnboardingIntentClassifier(llmClient); + const processTrace = new ProcessTraceCollector(); return runWithContext( { @@ -616,7 +657,7 @@ async function evaluateCase( trigger: input.request.trigger, userId: input.request.userId, }, - async () => { + () => runWithProcessTrace(processTrace, async () => { const classified = await classifier.classify( input.request.userMessage, input.request.trigger, @@ -651,6 +692,11 @@ async function evaluateCase( skipped: true as const, reason: "disabled", }; + const processTraceSnapshot = processTrace.snapshot(); + const processJudge = judgeProcessRule( + evalCase.expected, + processTraceSnapshot, + ); return { id: evalCase.id, @@ -663,11 +709,14 @@ async function evaluateCase( response, ruleJudge, llmJudge, + processJudge, + processTrace: processTraceSnapshot, }; } catch (error) { const latencyMs = Date.now() - startedAt; const errorMessage = error instanceof Error ? error.message : String(error); + const processTraceSnapshot = processTrace.snapshot(); return { id: evalCase.id, @@ -691,10 +740,15 @@ async function evaluateCase( skipped: true, reason: "skill-error", }, + processJudge: judgeProcessRule( + evalCase.expected, + processTraceSnapshot, + ), + processTrace: processTraceSnapshot, error: errorMessage, }; } - }, + }), ); } diff --git a/src/eval/shadow-bus.ts b/src/eval/shadow-bus.ts index 971589a..3f67e83 100644 --- a/src/eval/shadow-bus.ts +++ b/src/eval/shadow-bus.ts @@ -5,6 +5,7 @@ import type { ClassifiedIntent, DerivedState, } from "@skills/onboarding/state/onboarding-state"; +import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import { getBackgroundStreamMaxLen, getShadowEvalSamplingRate, @@ -17,6 +18,7 @@ export interface ShadowEvalEnvelope { createdAt: string; derived: DerivedState; latencyMs?: number | undefined; + process?: ProcessTraceSnapshot | undefined; request: AgentRequest; response: AgentResponse; } @@ -52,27 +54,31 @@ export class ShadowEvalBus implements ShadowEvalPublisher { const logger = getRequestLogger(); + const fields: Record = { + type: "shadow-eval", + requestId: envelope.request.requestId, + chatroomId: envelope.request.chatroomId, + userId: envelope.request.userId, + tier: String(envelope.derived.tier), + latencyMs: + envelope.latencyMs !== undefined ? String(envelope.latencyMs) : "", + input: JSON.stringify({ + request: envelope.request, + }), + output: JSON.stringify(envelope.response), + derived: JSON.stringify(envelope.derived), + classified: JSON.stringify(envelope.classified), + createdAt: envelope.createdAt, + }; + + if (envelope.process) { + fields.process = JSON.stringify(envelope.process); + } + void this.garnetClient .appendStream( BACKGROUND_STREAM_KEY, - { - type: "shadow-eval", - requestId: envelope.request.requestId, - chatroomId: envelope.request.chatroomId, - userId: envelope.request.userId, - tier: String(envelope.derived.tier), - latencyMs: - envelope.latencyMs !== undefined - ? String(envelope.latencyMs) - : "", - input: JSON.stringify({ - request: envelope.request, - }), - output: JSON.stringify(envelope.response), - derived: JSON.stringify(envelope.derived), - classified: JSON.stringify(envelope.classified), - createdAt: envelope.createdAt, - }, + fields, { maxLen: getBackgroundStreamMaxLen(), }, diff --git a/src/eval/shadow-worker.ts b/src/eval/shadow-worker.ts index a5cdad4..e6eb77d 100644 --- a/src/eval/shadow-worker.ts +++ b/src/eval/shadow-worker.ts @@ -13,22 +13,59 @@ import type { LlmClient } from "@llm/llm-client"; import { getLogger } from "@obs/logger"; import type { AlertSender } from "@obs/alert"; import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator"; +import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import type { ClassifiedIntent, DerivedState, } from "@skills/onboarding/state/onboarding-state"; import type { ConversationStore } from "@conv/conversation-store"; import { + getAlertThresholdDriftPct, getAlertThresholdEvalScoreDropPct, getAlertThresholdLlmErrorRate, getAlertThresholdP95LatencyMs, + getEvalAggTickMs, + getEvalArchiveTickMs, + getEvalDriftTickMs, + getEvalProcessLlmJudgeEnabled, + getEvalProcessLlmJudgeSamplingRate, + getEvalPromoteTickMs, + getEvalReviewTickMs, getEvalStreamMaxLen, } from "../config.js"; +import { + aggregateRecentEvalResults, + EVAL_AGG_CURSOR_KEY, + sliceKeyToString, +} from "./aggregator.js"; +import { + archiveEvalResults, + EVAL_ARCHIVE_CURSOR_KEY, + type MongoConnectionLike, +} from "./archive.js"; +import { computeDriftReports } from "./drift.js"; import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; +import { + judgeProcessRule, + judgeProcessWithLlm, +} from "./judges/process-judge.js"; +import { + appendCandidatesToJsonl, + selectPromotionCandidates, +} from "./promote.js"; +import { + EVAL_REVIEW_CURSOR_KEY, + routeReviewCandidates, +} from "./review/queue.js"; +import { + archiveReviewLabels, + EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, +} from "./review/label-archive.js"; import type { EvalBaselineFile, EvalCaseDefinition, EvalDerivedSnapshot, + ProcessJudgeResult, } from "./types.js"; const BACKGROUND_STREAM_KEY = "lolympus-agent:background"; @@ -57,6 +94,7 @@ interface ShadowEvalJob { createdAt: string; derived: DerivedState; latencyMs?: number | undefined; + process?: ProcessTraceSnapshot | undefined; request: AgentRequest; response: AgentResponse; } @@ -112,6 +150,14 @@ function parseJsonField( return JSON.parse(value) as T; } +function parseOptionalJsonField(value: string | undefined): T | undefined { + if (!value?.trim()) { + return undefined; + } + + return JSON.parse(value) as T; +} + function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot { return { allowShortResponse: derived.allowShortResponse, @@ -140,8 +186,11 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob { message.message["input"], "input", ); + const process = parseOptionalJsonField( + message.message["process"], + ); - return { + const job: ShadowEvalJob = { classified: parseJsonField( message.message["classified"], "classified", @@ -158,6 +207,12 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob { "output", ), }; + + if (process) { + job.process = process; + } + + return job; } export function calculatePercentile( @@ -206,11 +261,18 @@ export function calculateLlmErrorRate( } export class ShadowWorker { - private lastAggregateAt = 0; + private archiveFailureCount = 0; + private lastAlertCheckAt = 0; + private lastArchiveAt = 0; + private lastDriftAt = 0; + private lastEvalAggregateAt = 0; + private lastPromoteAt = 0; + private lastReviewAt = 0; private loopPromise: Promise | null = null; private readonly logger = getLogger(); private readonly recentEvalObservations: EvalObservation[] = []; private stopRequested = false; + private warnedMissingMongo = false; constructor( private readonly garnetClient: GarnetClient, @@ -218,6 +280,7 @@ export class ShadowWorker { private readonly conversationStore: ConversationStore, private readonly alertSender?: AlertSender, private readonly consumerName = `bg-worker-${process.pid}`, + private readonly mongoConnection?: MongoConnectionLike | undefined, ) {} async start(): Promise { @@ -432,6 +495,9 @@ export class ShadowWorker { llmJudge.skipped && llmJudge.reason !== "empty-response" && llmJudge.reason !== "disabled"; + const processJudge = job.process + ? await this.judgeProcess(job, message.id) + : undefined; this.recordEvalObservation({ llmJudgeErrored, @@ -440,37 +506,81 @@ export class ShadowWorker { responseLatencyMs: job.latencyMs ?? null, }); + const resultFields: Record = { + type: "shadow-eval-result", + requestId: job.request.requestId, + chatroomId: job.request.chatroomId, + userId: job.request.userId, + tier: String(job.derived.tier), + intent: job.derived.intent.kind, + phase: job.derived.phase.kind, + responseMode: job.derived.responseMode, + validatorStatus: validation.status, + latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "", + llmOverall: llmOverall === null ? "" : String(llmOverall), + input: JSON.stringify({ + request: job.request, + }), + output: JSON.stringify(job.response), + derived: JSON.stringify(job.derived), + classified: JSON.stringify(job.classified), + validation: JSON.stringify(validation), + llmJudge: JSON.stringify(llmJudge), + evaluatedAt: new Date().toISOString(), + }; + + if (job.process) { + resultFields.process = JSON.stringify(job.process); + } + if (processJudge) { + resultFields.processJudge = JSON.stringify(processJudge); + } + await this.garnetClient.appendStream( EVAL_RESULTS_STREAM_KEY, - { - type: "shadow-eval-result", - requestId: job.request.requestId, - chatroomId: job.request.chatroomId, - userId: job.request.userId, - tier: String(job.derived.tier), - intent: job.derived.intent.kind, - responseMode: job.derived.responseMode, - validatorStatus: validation.status, - latencyMs: - job.latencyMs !== undefined ? String(job.latencyMs) : "", - llmOverall: - llmOverall === null ? "" : String(llmOverall), - input: JSON.stringify({ - request: job.request, - }), - output: JSON.stringify(job.response), - derived: JSON.stringify(job.derived), - classified: JSON.stringify(job.classified), - validation: JSON.stringify(validation), - llmJudge: JSON.stringify(llmJudge), - evaluatedAt: new Date().toISOString(), - }, + resultFields, { maxLen: getEvalStreamMaxLen(), }, ); } + private async judgeProcess( + job: ShadowEvalJob, + messageId: string, + ): Promise { + if (!job.process) { + throw new Error("Missing process trace"); + } + + const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process); + if ( + !getEvalProcessLlmJudgeEnabled() || + Math.random() >= getEvalProcessLlmJudgeSamplingRate() + ) { + return ruleResult; + } + + const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, { + request: job.request, + finalText: job.response.text, + }); + + return { + checks: [...ruleResult.checks, ...llmResult.checks], + passedCount: ruleResult.passedCount + llmResult.passedCount, + totalCount: ruleResult.totalCount + llmResult.totalCount, + score: Number( + ( + (ruleResult.score * ruleResult.totalCount + + llmResult.score * llmResult.totalCount) / + Math.max(1, ruleResult.totalCount + llmResult.totalCount) + ).toFixed(3), + ), + rationale: llmResult.rationale, + }; + } + private async processSummarize(message: StreamEntry): Promise { const chatroomId = message.message["chatroomId"]; const userId = message.message["userId"]; @@ -563,16 +673,23 @@ export class ShadowWorker { } private async runPeriodicChecksIfNeeded(): Promise { + const now = Date.now(); + + await this.runAggregationTickIfNeeded(now); + await this.runArchiveTickIfNeeded(now); + await this.runDriftTickIfNeeded(now); + await this.runPromoteTickIfNeeded(now); + await this.runReviewTickIfNeeded(now); + if (!this.alertSender) { return; } - const now = Date.now(); - if (now - this.lastAggregateAt < ALERT_TICK_MS) { + if (now - this.lastAlertCheckAt < ALERT_TICK_MS) { return; } - this.lastAggregateAt = now; + this.lastAlertCheckAt = now; this.pruneOldObservations(now); await this.checkLatencyAlert(now); @@ -581,6 +698,210 @@ export class ShadowWorker { await this.checkOpenCircuitAlerts(now); } + private async runAggregationTickIfNeeded(now: number): Promise { + if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) { + return; + } + + this.lastEvalAggregateAt = now; + const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY); + const result = await aggregateRecentEvalResults(this.garnetClient, { + sinceId: sinceId ?? "0-0", + nowMs: now, + }); + + if (result.updatedSlices > 0) { + this.logger.debug( + { + lastSeenId: result.lastSeenId, + updatedSlices: result.updatedSlices, + }, + "[ShadowWorker] Aggregated shadow eval results", + ); + } + } + + private warnMissingMongoOnce(): void { + if (this.warnedMissingMongo) { + return; + } + + this.warnedMissingMongo = true; + this.logger.warn( + "[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled", + ); + } + + private async runArchiveTickIfNeeded(now: number): Promise { + if (now - this.lastArchiveAt < getEvalArchiveTickMs()) { + return; + } + + this.lastArchiveAt = now; + if (!this.mongoConnection) { + this.warnMissingMongoOnce(); + return; + } + + const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY); + try { + const result = await archiveEvalResults( + this.garnetClient, + this.mongoConnection, + { + sinceId: sinceId ?? "0-0", + nowMs: now, + }, + ); + const labelSinceId = await this.garnetClient.getString( + EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, + ); + const labelResult = await archiveReviewLabels( + this.garnetClient, + this.mongoConnection, + { + sinceId: labelSinceId ?? "0-0", + nowMs: now, + }, + ); + this.archiveFailureCount = 0; + if (result.archivedCount > 0) { + this.logger.info( + { + archivedCount: result.archivedCount, + lastArchivedId: result.lastArchivedId, + }, + "[ShadowWorker] Archived shadow eval results", + ); + } + if (labelResult.archivedCount > 0) { + this.logger.info( + { + archivedCount: labelResult.archivedCount, + lastArchivedId: labelResult.lastArchivedId, + }, + "[ShadowWorker] Archived review labels", + ); + } + } catch (error) { + this.archiveFailureCount += 1; + this.logger.warn( + { + err: error, + failureCount: this.archiveFailureCount, + }, + "[ShadowWorker] Failed to archive shadow eval results", + ); + + if (this.archiveFailureCount >= 3 && this.alertSender) { + await this.alertSender.sendAlert( + { + kind: "archive-stalled", + metric: "shadow_eval.archive_failures", + severity: "warn", + suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.", + threshold: 3, + timestamp: new Date(now).toISOString(), + value: this.archiveFailureCount, + window: "consecutive", + }, + { + dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled", + }, + ); + } + } + } + + private async runDriftTickIfNeeded(now: number): Promise { + if (now - this.lastDriftAt < getEvalDriftTickMs()) { + return; + } + + this.lastDriftAt = now; + if (!this.mongoConnection) { + this.warnMissingMongoOnce(); + return; + } + if (!this.alertSender) { + return; + } + + const threshold = getAlertThresholdDriftPct(); + const reports = await computeDriftReports(this.mongoConnection, { nowMs: now }); + const significantReports = reports.filter((report) => report.significant); + + for (const report of significantReports) { + const slice = sliceKeyToString(report.slice); + await this.alertSender.sendAlert( + { + kind: "eval-drift", + metric: "shadow_eval.drift.llm_overall_drop_pct", + severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn", + suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`, + threshold, + timestamp: new Date(now).toISOString(), + value: report.dropPct, + window: "24h_vs_7d", + }, + { + dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`, + }, + ); + } + } + + private async runPromoteTickIfNeeded(now: number): Promise { + if (now - this.lastPromoteAt < getEvalPromoteTickMs()) { + return; + } + + this.lastPromoteAt = now; + if (!this.mongoConnection) { + this.warnMissingMongoOnce(); + return; + } + + const candidates = await selectPromotionCandidates(this.mongoConnection, { + nowMs: now, + }); + if (candidates.length === 0) { + return; + } + + const result = await appendCandidatesToJsonl(candidates); + this.logger.info( + { + appendedCount: result.appendedCount, + outputPath: result.outputPath, + }, + "[ShadowWorker] Promoted shadow eval candidates", + ); + } + + private async runReviewTickIfNeeded(now: number): Promise { + if (now - this.lastReviewAt < getEvalReviewTickMs()) { + return; + } + + this.lastReviewAt = now; + const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY); + const result = await routeReviewCandidates(this.garnetClient, { + sinceId: sinceId ?? "0-0", + nowMs: now, + }); + + if (result.enqueuedCount > 0) { + this.logger.info( + { + enqueuedCount: result.enqueuedCount, + lastSeenId: result.lastSeenId, + }, + "[ShadowWorker] Routed eval results to review queue", + ); + } + } + private async checkLatencyAlert(now: number): Promise { const latencySamples = this.recentEvalObservations.flatMap((observation) => observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs], diff --git a/src/eval/types.ts b/src/eval/types.ts index 20d44e5..8391364 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -5,6 +5,7 @@ import type { } from "@agent/types"; import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state"; +import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; export interface EvalRoomContextMessage { senderId: string; @@ -26,6 +27,12 @@ export interface EvalCaseExpected { intent?: string | undefined; mustIncludeAny?: string[] | undefined; mustNotInclude?: string[] | undefined; + processExpected?: { + expectedApproach?: string | undefined; + expectedTools?: string[] | undefined; + minSteps?: number | undefined; + toolMode?: "exact" | "subset" | undefined; + } | undefined; responseMode?: string | undefined; } @@ -82,6 +89,30 @@ export type LlmJudgeResult = raw: string; }; +export interface ProcessJudgeCheck { + detail?: string | undefined; + passed: boolean; + rule: + | "critic-decision-justified" + | "intent-match" + | "min-steps" + | "planner-approach" + | "tier-appropriate" + | "tool-args-valid" + | "tools-selected-correctly" + | "validate-no-reject"; + skipped: boolean; + step: string; +} + +export interface ProcessJudgeResult { + checks: ProcessJudgeCheck[]; + passedCount: number; + rationale?: string | undefined; + score: number; + totalCount: number; +} + export interface EvalDerivedSnapshot { allowShortResponse: boolean; intent: string; @@ -109,6 +140,8 @@ export interface EvalCaseResult { response: AgentResponse; ruleJudge: RuleJudgeResult; llmJudge: LlmJudgeResult; + processJudge?: ProcessJudgeResult | undefined; + processTrace?: ProcessTraceSnapshot | undefined; error?: string | undefined; } diff --git a/src/skills/onboarding/prompts/builder.ts b/src/skills/onboarding/prompts/builder.ts index 7020589..023baee 100644 --- a/src/skills/onboarding/prompts/builder.ts +++ b/src/skills/onboarding/prompts/builder.ts @@ -1,3 +1,6 @@ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + import type { ChatMessage } from "@llm/types"; import type { DerivedState } from "../state/onboarding-state.js"; import type { OnboardingSession, PromptContext } from "../types.js"; @@ -10,11 +13,104 @@ import { } from "./layers.js"; import { buildToolPolicy } from "./tool-policy.js"; +export type PromptLayerName = + | "context" + | "examples" + | "persona" + | "policy" + | "state" + | "tool-policy"; + +export type PromptLayerOperation = "append" | "remove" | "replace"; + +export interface PromptLayerDiff { + layer: PromptLayerName; + op: PromptLayerOperation; + reason?: string | undefined; + text?: string | undefined; +} + +export interface PromptVariantManifest { + description?: string | undefined; + diff: PromptLayerDiff[]; + expectedImprovement?: string | undefined; + rationale?: string | undefined; + variantId: string; +} + +const variantCache = new Map(); + +function getLayerTarget(layer: PromptLayerName): Exclude { + return layer === "tool-policy" ? "policy" : layer; +} + +function applyLayerDiff( + layers: Record, string>, + diff: PromptLayerDiff, +): void { + const target = getLayerTarget(diff.layer); + + if (diff.op === "remove") { + layers[target] = ""; + return; + } + + const text = diff.text?.trim(); + if (!text) { + return; + } + + if (diff.op === "replace") { + layers[target] = text; + return; + } + + layers[target] = layers[target] ? `${layers[target]}\n${text}` : text; +} + +export function loadPromptVariantManifest( + variantId: string, +): PromptVariantManifest | null { + const normalized = variantId.trim(); + if (!normalized) { + return null; + } + if (variantCache.has(normalized)) { + return variantCache.get(normalized) ?? null; + } + + const path = resolve( + process.cwd(), + "src", + "eval", + "proposals", + `${normalized}.json`, + ); + if (!existsSync(path)) { + variantCache.set(normalized, null); + return null; + } + + try { + const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest; + if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) { + variantCache.set(normalized, null); + return null; + } + variantCache.set(normalized, parsed); + return parsed; + } catch { + variantCache.set(normalized, null); + return null; + } +} + export class PromptBuilder { private constructor( private readonly derived: DerivedState, private readonly session: OnboardingSession | null, private readonly context: PromptContext, + private readonly variant?: PromptVariantManifest | undefined, ) {} static forDerived( @@ -25,7 +121,18 @@ export class PromptBuilder { return new PromptBuilder(derived, session, context); } - build(): ChatMessage[] { + applyVariant(variant: PromptVariantManifest | string): PromptBuilder { + const manifest = + typeof variant === "string" ? loadPromptVariantManifest(variant) : variant; + return new PromptBuilder( + this.derived, + this.session, + this.context, + manifest ?? undefined, + ); + } + + buildSystemPrompt(): string { const promptHistory = this.context.summary ? this.context.conversationHistory.slice(-5) : this.context.conversationHistory; @@ -34,21 +141,41 @@ export class PromptBuilder { conversationHistory: promptHistory, }; - const systemPrompt = [ - personaLayer(), - policyLayer(this.derived.intent, { + const layers = { + persona: personaLayer(), + policy: policyLayer(this.derived.intent, { toolPolicy: this.derived.responseMode === "llm-with-tools" ? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind) : undefined, }), - stateLayer(this.session, this.derived), - examplesLayer(this.derived.intent), - contextLayer(promptContext), - ].join("\n\n"); + state: stateLayer(this.session, this.derived), + examples: examplesLayer(this.derived.intent), + context: contextLayer(promptContext), + }; + + for (const diff of this.variant?.diff ?? []) { + applyLayerDiff(layers, diff); + } return [ - { role: "system", content: systemPrompt }, + layers.persona, + layers.policy, + layers.state, + layers.examples, + layers.context, + ] + .filter((layer) => layer.trim().length > 0) + .join("\n\n"); + } + + build(): ChatMessage[] { + const promptHistory = this.context.summary + ? this.context.conversationHistory.slice(-5) + : this.context.conversationHistory; + + return [ + { role: "system", content: this.buildSystemPrompt() }, ...promptHistory.map((message) => ({ role: message.role, content: message.content, diff --git a/src/skills/onboarding/runtime/critic.ts b/src/skills/onboarding/runtime/critic.ts index 8b16dbe..a12a0d7 100644 --- a/src/skills/onboarding/runtime/critic.ts +++ b/src/skills/onboarding/runtime/critic.ts @@ -6,6 +6,7 @@ import { getTier2CriticModel, } from "../../../config.js"; import type { DerivedState } from "../state/onboarding-state.js"; +import { getActiveProcessTrace } from "./trace.js"; export interface CritiqueResult { changed: boolean; @@ -55,6 +56,7 @@ export class OnboardingCritic { messages: ChatMessage[], options?: { deadlineMs?: number | undefined }, ): Promise { + const traceStep = getActiveProcessTrace()?.startStep("critic"); const response = await this.llmClient.chatCompletion( buildCriticMessages(draft, derived, messages), { @@ -69,12 +71,20 @@ export class OnboardingCritic { const text = response?.content.trim() ?? ""; if (!text || text.startsWith("[OK]")) { + traceStep?.finish({ + decision: "keep", + rationale: text || "empty critic response", + }); return { changed: false, text: draft, }; } + traceStep?.finish({ + decision: "rewrite", + rationale: text.slice(0, 200), + }); return { changed: true, text, diff --git a/src/skills/onboarding/runtime/orchestrator.ts b/src/skills/onboarding/runtime/orchestrator.ts index ad8227a..ccab56b 100644 --- a/src/skills/onboarding/runtime/orchestrator.ts +++ b/src/skills/onboarding/runtime/orchestrator.ts @@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js"; import { OnboardingCritic } from "./critic.js"; import type { PlannerOutput } from "./planner-schema.js"; import { PlannerOutputSchema } from "./planner-schema.js"; +import { getActiveProcessTrace } from "./trace.js"; import { runWithTools } from "./tool-loop.js"; export interface OrchestratorRunOptions { @@ -260,6 +261,17 @@ export class OnboardingOrchestrator { messages: ChatMessage[], ): Promise { const logger = getRequestLogger(); + const traceStep = getActiveProcessTrace()?.startStep("plan"); + const finishTrace = (plan: PlannerOutput): PlannerOutput => { + traceStep?.finish({ + approach: plan.approach, + mustAvoid: plan.mustAvoid, + mustInclude: plan.mustInclude, + suggestedTools: plan.suggestedTools, + targetLengthChars: plan.targetLengthChars, + }); + return plan; + }; const plannerMessages = buildPlannerMessages(derived, messages); const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, { deadlineMs: getRemainingDeadlineMs(), @@ -283,7 +295,7 @@ export class OnboardingOrchestrator { }, "[OnboardingOrchestrator] Planner output parsed", ); - return firstPlan; + return finishTrace(firstPlan); } const retryMessages = [ @@ -315,7 +327,7 @@ export class OnboardingOrchestrator { }, "[OnboardingOrchestrator] Planner output parsed", ); - return retryPlan; + return finishTrace(retryPlan); } const fallbackPlan = getDefaultPlan(derived); @@ -327,7 +339,7 @@ export class OnboardingOrchestrator { }, "[OnboardingOrchestrator] Planner fell back to default plan", ); - return fallbackPlan; + return finishTrace(fallbackPlan); } private async runTier0( diff --git a/src/skills/onboarding/runtime/tool-loop.ts b/src/skills/onboarding/runtime/tool-loop.ts index d9b274a..03c8c1c 100644 --- a/src/skills/onboarding/runtime/tool-loop.ts +++ b/src/skills/onboarding/runtime/tool-loop.ts @@ -4,6 +4,7 @@ import type { LlmClient } from "@llm/llm-client"; import type { ChatCompletionOptions, ChatMessage } from "@llm/types"; import { parseToolCalls } from "@tools/parser"; import type { Tool, ToolContext } from "@tools/tool"; +import { getActiveProcessTrace } from "./trace.js"; export interface ToolLoopOptions { deadlineMs?: number | undefined; @@ -107,15 +108,33 @@ export async function runWithTools( "[ToolLoop] Executing tool call", ); - const validated = tool.argsSchema.parse(call.arguments); - const result = await withToolSpan( - tool.name, - { - "tool.call.id": call.id, - "tool.name": tool.name, - }, - () => tool.execute(validated, options.toolContext), - ); + const traceStep = getActiveProcessTrace()?.startStep("tool-call"); + let result: unknown; + try { + const validated = tool.argsSchema.parse(call.arguments); + result = await withToolSpan( + tool.name, + { + "tool.call.id": call.id, + "tool.name": tool.name, + }, + () => tool.execute(validated, options.toolContext), + ); + traceStep?.finish({ + args: call.arguments, + name: tool.name, + ok: true, + resultSummary: summarizeToolResult(result), + }); + } catch (error) { + traceStep?.finish({ + args: call.arguments, + name: tool.name, + ok: false, + resultSummary: error instanceof Error ? error.message : String(error), + }); + throw error; + } logger.debug( { @@ -137,3 +156,11 @@ export async function runWithTools( return null; } + +function summarizeToolResult(result: unknown): string { + try { + return JSON.stringify(result).slice(0, 200); + } catch { + return String(result).slice(0, 200); + } +} diff --git a/src/skills/onboarding/runtime/trace.ts b/src/skills/onboarding/runtime/trace.ts new file mode 100644 index 0000000..d81c3a8 --- /dev/null +++ b/src/skills/onboarding/runtime/trace.ts @@ -0,0 +1,169 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +export type ProcessTraceStepName = + | "classify" + | "critic" + | "derive" + | "plan" + | "respond" + | "tool-call" + | "validate"; + +export interface ProcessTraceStep { + name: ProcessTraceStepName; + startedAtMs: number; + durationMs: number; + output: T; +} + +export interface ProcessTraceSnapshot { + steps: { + classify?: ProcessTraceStep<{ + confidence: number; + intent: string; + source: "regex" | "llm"; + }> | undefined; + derive?: ProcessTraceStep<{ + phase: string; + responseMode: string; + shouldRespond: boolean; + tier: 0 | 1 | 2; + }> | undefined; + plan?: ProcessTraceStep<{ + approach: string; + mustAvoid: string[]; + mustInclude: string[]; + suggestedTools: string[]; + targetLengthChars: number; + }> | undefined; + toolCalls?: Array< + ProcessTraceStep<{ + args: unknown; + name: string; + ok: boolean; + resultSummary: string; + }> + > | undefined; + respond?: ProcessTraceStep<{ + draftText: string; + tokensIn: number; + tokensOut: number; + }> | undefined; + critic?: ProcessTraceStep<{ + decision: "keep" | "rewrite"; + rationale: string; + }> | undefined; + validate?: ProcessTraceStep<{ + reasons: string[]; + status: "pass" | "reject" | "soft-warn"; + }> | undefined; + }; +} + +type ProcessTraceOutputByName = { + classify: NonNullable["output"]; + critic: NonNullable["output"]; + derive: NonNullable["output"]; + plan: NonNullable["output"]; + respond: NonNullable["output"]; + "tool-call": NonNullable< + NonNullable[number] + >["output"]; + validate: NonNullable["output"]; +}; + +const processTraceStorage = new AsyncLocalStorage(); + +export class ProcessTraceCollector { + private readonly steps: ProcessTraceSnapshot["steps"] = {}; + + startStep( + name: Name, + ): { + finish: (output: ProcessTraceOutputByName[Name]) => void; + } { + const startedAtMs = Date.now(); + let finished = false; + + return { + finish: (output: ProcessTraceOutputByName[Name]): void => { + if (finished) { + return; + } + finished = true; + + const step: ProcessTraceStep = { + name, + startedAtMs, + durationMs: Math.max(0, Date.now() - startedAtMs), + output, + }; + + if (name === "tool-call") { + this.steps.toolCalls = [ + ...(this.steps.toolCalls ?? []), + step as ProcessTraceStep< + ProcessTraceOutputByName["tool-call"] + >, + ]; + return; + } + + switch (name) { + case "classify": + this.steps.classify = step as ProcessTraceStep< + ProcessTraceOutputByName["classify"] + >; + return; + case "critic": + this.steps.critic = step as ProcessTraceStep< + ProcessTraceOutputByName["critic"] + >; + return; + case "derive": + this.steps.derive = step as ProcessTraceStep< + ProcessTraceOutputByName["derive"] + >; + return; + case "plan": + this.steps.plan = step as ProcessTraceStep< + ProcessTraceOutputByName["plan"] + >; + return; + case "respond": + this.steps.respond = step as ProcessTraceStep< + ProcessTraceOutputByName["respond"] + >; + return; + case "validate": + this.steps.validate = step as ProcessTraceStep< + ProcessTraceOutputByName["validate"] + >; + return; + } + }, + }; + } + + snapshot(): ProcessTraceSnapshot { + return { + steps: { + ...this.steps, + toolCalls: this.steps.toolCalls + ? [...this.steps.toolCalls] + : undefined, + }, + }; + } +} + +export function runWithProcessTrace( + collector: ProcessTraceCollector, + fn: () => T, +): T { + return processTraceStorage.run(collector, fn); +} + +export function getActiveProcessTrace(): ProcessTraceCollector | undefined { + return processTraceStorage.getStore(); +} diff --git a/src/skills/onboarding/runtime/validator.ts b/src/skills/onboarding/runtime/validator.ts index ac766a4..d4ae473 100644 --- a/src/skills/onboarding/runtime/validator.ts +++ b/src/skills/onboarding/runtime/validator.ts @@ -3,6 +3,7 @@ import { getMaxResponseLength } from "../../../config.js"; import { VALID_TIER_SET } from "../constants.js"; import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js"; import type { DerivedState } from "../state/onboarding-state.js"; +import { getActiveProcessTrace } from "./trace.js"; const ALLOWED_URL_PATTERNS = [ /discord\.gg\/lolympus/iu, @@ -118,6 +119,10 @@ function markValidation(result: ValidationResult): ValidationResult { "agent.validator.status": result.status, "agent.validator.warnings": result.reasons, }); + getActiveProcessTrace()?.startStep("validate").finish({ + status: result.status, + reasons: result.reasons, + }); return result; } diff --git a/src/skills/onboarding/session.ts b/src/skills/onboarding/session.ts index a5af6a5..fa1f252 100644 --- a/src/skills/onboarding/session.ts +++ b/src/skills/onboarding/session.ts @@ -61,7 +61,7 @@ export function collectNicknameAliases( for (const source of sources) { if (Array.isArray(source)) { - for (const value of source) { + for (const value of source as readonly string[]) { pushAlias(value); } continue; diff --git a/src/skills/onboarding/skill.ts b/src/skills/onboarding/skill.ts index d69da8a..65768f7 100644 --- a/src/skills/onboarding/skill.ts +++ b/src/skills/onboarding/skill.ts @@ -8,6 +8,7 @@ import type { ChatMessage } from '@llm/types'; import type { RoomContextService } from '@room/room-context-service'; import type { Skill } from '@skills/skill'; import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus'; +import { anonymizeProcessTrace } from '@eval/anonymize'; import { createBuiltInToolRegistry } from '@tools/builtins/index'; import type { Tool, ToolContext } from '@tools/tool'; import { PromptBuilder } from './prompts/builder.js'; @@ -27,12 +28,21 @@ import { getRemainingStatusMessage, getTimeoutWarningMessage, } from './onboarding-templates.js'; -import { getOnboardingSessionKeyPrefix, getLlmTemperature } from '../../config.js'; +import { + getOnboardingSessionKeyPrefix, + getLlmTemperature, + getPromptVariantId, +} from '../../config.js'; import { OnboardingBlackboardStore } from './state/blackboard.js'; import { deriveState } from './state/derive.js'; import { OnboardingIntentClassifier } from './state/classifier.js'; import type { DerivedState, Intent } from './state/onboarding-state.js'; import { OnboardingOrchestrator } from './runtime/orchestrator.js'; +import { + getActiveProcessTrace, + ProcessTraceCollector, + runWithProcessTrace, +} from './runtime/trace.js'; import { validateOnboardingResponse } from './runtime/validator.js'; import { normalizeOnboardingSession } from './session.js'; import type { OnboardingSession, PromptContext, StoredOnboardingSession } from './types.js'; @@ -71,19 +81,57 @@ export class OnboardingSkill implements Skill { async handle( request: AgentRequest, conversationHistory: ConversationMessage[], + ): Promise { + if (getActiveProcessTrace()) { + return this.handleWithTrace(request, conversationHistory); + } + + const collector = new ProcessTraceCollector(); + return runWithProcessTrace(collector, () => + this.handleWithTrace(request, conversationHistory), + ); + } + + private async handleWithTrace( + request: AgentRequest, + conversationHistory: ConversationMessage[], ): Promise { const logger = getRequestLogger(); + const trace = getActiveProcessTrace(); const session = await this.loadSession(request.chatroomId, request.userId); if (request.trigger === 'user-message' && !request.userMessage?.trim()) { return this.makeEmptyResponse(request.requestId); } + const classifyStep = trace?.startStep('classify'); const classified = await this.classifier.classify(request.userMessage, request.trigger); + classifyStep?.finish({ + confidence: classified.confidence, + intent: classified.intent.kind, + source: classified.source, + }); + + const deriveStep = trace?.startStep('derive'); const derived = deriveState(session, request.trigger, classified.intent); + deriveStep?.finish({ + phase: derived.phase.kind, + responseMode: derived.responseMode, + shouldRespond: derived.shouldRespond, + tier: derived.tier, + }); if (!derived.shouldRespond) { - return this.makeEmptyResponse(request.requestId); + const response = this.makeEmptyResponse(request.requestId); + trace?.startStep('respond').finish({ + draftText: response.text, + tokensIn: 0, + tokensOut: 0, + }); + validateOnboardingResponse('[NO_RESPONSE]', derived, { + allowShortResponse: true, + }); + return response; } let response: AgentResponse; @@ -102,6 +150,15 @@ export class OnboardingSkill implements Skill { } } + trace?.startStep('respond').finish({ + draftText: response.text, + tokensIn: 0, + tokensOut: 0, + }); + validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, { + allowShortResponse: derived.allowShortResponse, + }); + if (derived.intent.kind === 'discord-verify-question') { logger.debug( { @@ -122,6 +179,7 @@ export class OnboardingSkill implements Skill { createdAt: new Date().toISOString(), derived, latencyMs: this.getCurrentLatencyMs(), + process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, }); @@ -185,7 +243,9 @@ export class OnboardingSkill implements Skill { userMessage: this.buildPromptUserMessage(request, derived), }; - return PromptBuilder.forDerived(derived, session, context).build(); + const builder = PromptBuilder.forDerived(derived, session, context); + const promptVariantId = getPromptVariantId(); + return promptVariantId ? builder.applyVariant(promptVariantId).build() : builder.build(); } private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string { diff --git a/src/skills/onboarding/state/transitions.ts b/src/skills/onboarding/state/transitions.ts index 9343cb8..1a80cfc 100644 --- a/src/skills/onboarding/state/transitions.ts +++ b/src/skills/onboarding/state/transitions.ts @@ -1,7 +1,7 @@ import { DEADLINE_NEAR_THRESHOLD_MIN } from "../constants.js"; import { analyzeNicknameIssues } from "../onboarding-templates.js"; import type { OnboardingSession } from "../types.js"; -import type { OnboardingPhase, RemainingStep } from "./onboarding-state.js"; +import type { OnboardingPhase } from "./onboarding-state.js"; import { getRemainingSteps, isOnboardingCompleted, diff --git a/src/tools/builtins/check-onboarding-progress.ts b/src/tools/builtins/check-onboarding-progress.ts index d704e3f..ffaf61e 100644 --- a/src/tools/builtins/check-onboarding-progress.ts +++ b/src/tools/builtins/check-onboarding-progress.ts @@ -8,7 +8,6 @@ import { isStep1Completed, } from "@skills/onboarding/state/progress"; import type { - OnboardingSession, StoredOnboardingSession, } from "@skills/onboarding/types"; import type { Tool } from "../tool.js"; diff --git a/src/tools/builtins/verify-discord-nickname.test.ts b/src/tools/builtins/verify-discord-nickname.test.ts index 481d4bf..6d1b767 100644 --- a/src/tools/builtins/verify-discord-nickname.test.ts +++ b/src/tools/builtins/verify-discord-nickname.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { verifyDiscordNicknameTool } from "./verify-discord-nickname.js"; import type { OnboardingSession } from "@skills/onboarding/types"; import type { DiscordMemberLookupResponse } from "@common/nats-subjects"; +import { KAKAOTALK_NICKNAME_FORMAT_EXAMPLE } from "@skills/onboarding/constants"; function makeSession( overrides?: Partial, @@ -35,7 +36,7 @@ describe("verifyDiscordNicknameTool", () => { ); expect(result.status).toBe("invalid-nickname-format"); - expect(result.formatExample).toBe("홍길동/95/남/G"); + expect(result.formatExample).toBe(KAKAOTALK_NICKNAME_FORMAT_EXAMPLE); expect(result.issueMessages?.length).toBeGreaterThan(0); }); @@ -56,11 +57,11 @@ describe("verifyDiscordNicknameTool", () => { joinedAfterMatches: [], }; const request = vi.fn( - async ( + ( _subject: string, _payload: TRequest, _options?: { timeoutMs?: number }, - ): Promise => response as TResponse, + ): Promise => Promise.resolve(response as TResponse), ); const session = makeSession(); const result = await verifyDiscordNicknameTool.execute( @@ -102,11 +103,11 @@ describe("verifyDiscordNicknameTool", () => { joinedAfterMatches: [], }; const request = vi.fn( - async ( + ( _subject: string, _payload: TRequest, _options?: { timeoutMs?: number }, - ): Promise => response as TResponse, + ): Promise => Promise.resolve(response as TResponse), ); const result = await verifyDiscordNicknameTool.execute( @@ -141,11 +142,11 @@ describe("verifyDiscordNicknameTool", () => { ], }; const request = vi.fn( - async ( + ( _subject: string, _payload: TRequest, _options?: { timeoutMs?: number }, - ): Promise => response as TResponse, + ): Promise => Promise.resolve(response as TResponse), ); const result = await verifyDiscordNicknameTool.execute( diff --git a/src/workers/background.ts b/src/workers/background.ts index b15ddcd..e0ecacb 100644 --- a/src/workers/background.ts +++ b/src/workers/background.ts @@ -21,6 +21,7 @@ const { getRawSecret } = await import("../common/secret-manager.js"); const { ConversationStore } = await import("../conversation/index.js"); const { ShadowWorker } = await import("../eval/shadow-worker.js"); const { AlertService } = await import("../observability/alert.js"); +const { initMongoDB, closeMongoDB } = await import("../common/mongodb.js"); function firstNonEmpty( ...values: Array @@ -175,15 +176,30 @@ async function bootstrap(): Promise { }); const conversationStore = new ConversationStore(garnetClient); const alertService = new AlertService(garnetClient, natsClient); + const mongoConnectionString = firstNonEmpty( + getRawSecret("MONGODB_CONNECTION_STRING"), + process.env["MONGODB_CONNECTION_STRING"], + ); + const mongoConnection = mongoConnectionString + ? await initMongoDB(mongoConnectionString) + : undefined; + if (!mongoConnection) { + logger.warn( + "[Background] MongoDB connection string missing; eval archive/drift/promote disabled", + ); + } const worker = new ShadowWorker( garnetClient, llmClient, conversationStore, alertService, + undefined, + mongoConnection, ); installShutdownHooks(async () => { await worker.stop(); + await closeMongoDB(); await natsClient.close(); await garnetClient.close(); await shutdownTracing();