Improve onboarding verification and routing controls
This commit is contained in:
1 parent
bc0f8e0773
commit
d05de52cd9
30 files changed
+3021
-159
No files matched your search
@@ -257,6 +257,18 @@ npm run review:list -- --reason=low-confidence --limit=10
|
||||
|
||||
Resolution history는 stream에 쌓이며, compatibility hash는 sunset 전까지 read fallback으로 유지된다. Sunset 조건과 절차는 [resolution-hash-sunset-procedure.md](resolution-hash-sunset-procedure.md)를 따른다.
|
||||
|
||||
### 10.2 Tool Audit
|
||||
|
||||
Tool call은 성공과 실패 모두 `lolympus-agent:tool-calls` Stream에 남긴다. 원본 args는 저장하지 않고, 민감 키를 제거한 뒤 hash만 기록한다.
|
||||
|
||||
```bash
|
||||
npm run tool-audit:list -- --limit=20
|
||||
npm run tool-audit:list -- --skill=onboarding --since=24h
|
||||
npm run tool-audit:show -- <callId>
|
||||
```
|
||||
|
||||
Stream은 `AGENT_TOOL_AUDIT_MAX_LEN` 기준으로 trim된다. 기본값은 `100000`이다.
|
||||
|
||||
## 11. Calibration 흐름
|
||||
|
||||
Calibration은 사람 점수와 LLM judge 점수의 상관을 계산한다.
|
||||
@@ -460,6 +472,13 @@ npm test
|
||||
| `EVAL_TOURNAMENT_MIN_CASES` | 10 | proposal 최소 archive cases |
|
||||
| `PROMPT_VARIANT_ID` | unset | 운영 prompt variant 선택 |
|
||||
| `AGENT_ROUTER_WEIGHTS_PATH` | unset | learned router weights path |
|
||||
| `AGENT_BUDGET_LLM_COST_ENABLED` | true | LLM token cost를 request budget에 누적 |
|
||||
| `AGENT_COST_AWARE_TIER_ENABLED` | true | confidence/cost-aware tier selection |
|
||||
| `AGENT_SUPERVISOR_FAST_PATH_ENABLED` | true | supervisor regex fast-path 사용 |
|
||||
| `SUPERVISOR_FAST_PATH_CONFIDENCE` | 0.85 | fast-path 최소 규칙 confidence |
|
||||
| `AGENT_TOOL_SANDBOX_ENABLED` | false | tool context 최소 권한 강제 여부 |
|
||||
| `AGENT_TOOL_AUDIT_MAX_LEN` | 100000 | tool audit Stream maxlen |
|
||||
| `AGENT_NATS_ACL_MODE` | log-only | NATS subject ACL 모드 |
|
||||
| `AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED` | true | supervisor LLM userMessage 사용 여부 |
|
||||
| `PHOENIX_OTLP_ENDPOINT` | unset | Phoenix OTLP trace endpoint |
|
||||
| `PHOENIX_LOG_PROMPTS` | false | LLM prompt/completion content trace 기록 |
|
||||
@@ -519,7 +538,15 @@ ALERT_THRESHOLD_DRIFT_PCT=15
|
||||
예를 들어 onboarding latency alert와 moderation latency alert는 서로 throttle하지 않는다.
|
||||
skill이 없는 cross-system alert는 기존처럼 글로벌 dedupe를 사용한다.
|
||||
|
||||
### 16.2 Phoenix trace
|
||||
### 16.2 Secret, Tool Sandbox, NATS ACL
|
||||
|
||||
Secret은 Infisical을 우선으로 로드한다. Infisical credential은 로컬 env 또는 IMDS에서 읽고, 새 secret env를 추가할 때는 `src/common/known-secret-keys.ts`에 함께 등록해야 로그 redaction 대상에 들어간다. LLM, embedding, Phoenix API key도 redaction 대상이다.
|
||||
|
||||
Tool은 `requires` 선언으로 필요한 context만 받는다. `AGENT_TOOL_SANDBOX_ENABLED=false`가 기본이라 운영 초기는 호환성을 유지한다. 1주 정도 audit을 본 뒤 false positive가 없으면 true로 전환한다.
|
||||
|
||||
NATS subject ACL은 `src/common/nats-skill-acl.ts`의 `DEFAULT_NATS_ACL`에서 관리한다. 기본 `AGENT_NATS_ACL_MODE=log-only`는 위반을 warn으로만 남기고 publish/request는 그대로 진행한다. 1개월 운영 후 false positive가 없으면 `enforce`로 전환한다.
|
||||
|
||||
### 16.3 Phoenix trace
|
||||
|
||||
LLM 호출, skill 실행, tool 실행 span은 OpenTelemetry trace로 남긴다.
|
||||
`PHOENIX_OTLP_ENDPOINT`가 없으면 기존 설정을 따른다. 개발 환경 기본값은 trace exporter 비활성화이고, `OTEL_EXPORTER=stdout`이면 콘솔 exporter를 쓴다.
|
||||
@@ -544,7 +571,7 @@ Phoenix UI에서는 `service.name=lolympus-agent`, `gen_ai.system`, `gen_ai.requ
|
||||
Prompt와 completion 본문은 기본적으로 기록하지 않는다.
|
||||
디버깅 때문에 필요할 때만 `PHOENIX_LOG_PROMPTS=true`를 켠다. 이 경우 email, 전화번호, token/API key 형태의 값은 span 기록 전에 masking된다.
|
||||
|
||||
### 16.3 Prometheus scrape
|
||||
### 16.4 Prometheus scrape
|
||||
|
||||
Agent process는 `METRICS_PORT`에서 HTTP endpoint를 연다.
|
||||
|
||||
@@ -596,7 +623,7 @@ sum by (skill, stage) (rate(agent_validator_fail_total[5m]))
|
||||
| eval drift | 15% 이하 |
|
||||
| archive stalled | 3회 연속 실패 0건 |
|
||||
|
||||
### 16.4 Slack webhook 설정
|
||||
### 16.5 Slack webhook 설정
|
||||
|
||||
Slack incoming webhook URL을 env로 설정한다.
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+23
-7
@@ -16,20 +16,36 @@ if ($isMacOS -or $isLinux) {
|
||||
$env:ENV_FILE = "$PSScriptRoot\env-vars\$env:NODE_ENV.env"
|
||||
}
|
||||
|
||||
# 환경 변수 파일 확인
|
||||
if (-Not (Test-Path $env:ENV_FILE)) {
|
||||
Write-Error "The file '$env:ENV_FILE' does not exist."
|
||||
exit 1
|
||||
}
|
||||
$GLOBAL_ENV_FILE = Join-Path $HOME "source/global-env-vars.txt"
|
||||
|
||||
# 'dahyeon-local.env' 파일에서 환경 변수 설정
|
||||
Get-Content $env:ENV_FILE | ForEach-Object {
|
||||
function Import-EnvFileIfPresent {
|
||||
param(
|
||||
[string]$Path,
|
||||
[switch]$Required
|
||||
)
|
||||
|
||||
if (-Not (Test-Path $Path)) {
|
||||
if ($Required) {
|
||||
Write-Error "The file '$Path' does not exist."
|
||||
exit 1
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Get-Content $Path | ForEach-Object {
|
||||
if ($_ -match '^\s*([^#]+?)\s*=\s*(.*)$') {
|
||||
$key, $value = $matches[1].Trim(), $matches[2].Trim().Trim("`"")
|
||||
if (-not [System.Environment]::GetEnvironmentVariable($key)) {
|
||||
[System.Environment]::SetEnvironmentVariable($key, $value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# 환경 변수 파일 확인
|
||||
Import-EnvFileIfPresent -Path $GLOBAL_ENV_FILE
|
||||
Import-EnvFileIfPresent -Path $env:ENV_FILE -Required
|
||||
|
||||
Get-ChildItem Env: | Where-Object {
|
||||
$_.Name -eq "NODE_ENV" -or
|
||||
$_.Name -like "IMDS_*" -or
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import * as dotenv from "dotenv";
|
||||
@@ -34,6 +35,11 @@ function resolveEnvFile(mode: NodeEnvMode): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveGlobalEnvFile(): string | null {
|
||||
const candidate = path.resolve(os.homedir(), "source", "global-env-vars.txt");
|
||||
return fs.existsSync(candidate) ? candidate : null;
|
||||
}
|
||||
|
||||
function loadDotenvFile(filePath: string, label: string): void {
|
||||
const result = dotenv.config({ path: filePath, override: false });
|
||||
if (result.error) {
|
||||
@@ -48,6 +54,11 @@ export function loadEnvironmentFromNodeEnv(): void {
|
||||
const mode = normalizeMode(process.env["NODE_ENV"]);
|
||||
process.env["NODE_ENV"] = mode;
|
||||
|
||||
const globalEnvFile = resolveGlobalEnvFile();
|
||||
if (globalEnvFile) {
|
||||
loadDotenvFile(globalEnvFile, "global-env-vars.txt");
|
||||
}
|
||||
|
||||
const envFile = resolveEnvFile(mode);
|
||||
if (!envFile) {
|
||||
logger.warn(`[Env] env-vars/${mode}.env not found. Skipping dotenv load.`);
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { GarnetClient } from "./garnet.js";
|
||||
import type { GarnetClient as GarnetClientType } from "./garnet.js";
|
||||
|
||||
export async function connectGarnetForCli(): Promise<{
|
||||
close: () => Promise<void>;
|
||||
garnetClient: GarnetClient;
|
||||
garnetClient: GarnetClientType;
|
||||
}> {
|
||||
const { GarnetClient } = await import("./garnet.js");
|
||||
const garnetClient = new GarnetClient();
|
||||
await garnetClient.connect();
|
||||
return {
|
||||
|
||||
@@ -34,6 +34,12 @@ export type DiscordMemberMatchSource =
|
||||
| "username"
|
||||
| "joined-after";
|
||||
|
||||
export type DiscordResidentApplicationMessageMatchSource =
|
||||
| "author-id"
|
||||
| "author-name"
|
||||
| "content"
|
||||
| "joined-after";
|
||||
|
||||
export interface DiscordMemberCandidate {
|
||||
memberId: string;
|
||||
joinedAt: string;
|
||||
@@ -43,8 +49,20 @@ export interface DiscordMemberCandidate {
|
||||
username: string;
|
||||
}
|
||||
|
||||
export interface DiscordResidentApplicationMessageCandidate {
|
||||
authorDisplayName?: string | undefined;
|
||||
authorGuildNickname?: string | undefined;
|
||||
authorId: string;
|
||||
channelId: string;
|
||||
content: string;
|
||||
createdAt: string;
|
||||
matchedBy?: DiscordResidentApplicationMessageMatchSource | undefined;
|
||||
messageId: string;
|
||||
}
|
||||
|
||||
export interface DiscordMemberLookupRequest {
|
||||
chatroomId: string;
|
||||
includeResidentApplicationMessages?: boolean | undefined;
|
||||
joinedAfter: string;
|
||||
searchName?: string | undefined;
|
||||
userId: string;
|
||||
@@ -55,6 +73,9 @@ export type DiscordMemberLookupResponse =
|
||||
status: "ok";
|
||||
joinedAfterMatches: DiscordMemberCandidate[];
|
||||
nicknameMatches: DiscordMemberCandidate[];
|
||||
residentApplicationMessageMatches?:
|
||||
| DiscordResidentApplicationMessageCandidate[]
|
||||
| undefined;
|
||||
}
|
||||
| {
|
||||
status: "error";
|
||||
|
||||
@@ -11,7 +11,12 @@ import {
|
||||
getAgentToolSandboxEnabled,
|
||||
getEmbeddingModel,
|
||||
getEmbeddingProvider,
|
||||
getEvalCaseDeadlineMs,
|
||||
getEvalLlmProvider,
|
||||
getEvalRuntimeLlmEnabled,
|
||||
getLongTermSummaryEnabled,
|
||||
getLlmModelRoundRobinEnabled,
|
||||
getRuntimeLlmModelsForProvider,
|
||||
getMemoryEvictionEnabled,
|
||||
getNatsAclMode,
|
||||
getPhoenixLogPrompts,
|
||||
@@ -56,6 +61,12 @@ const ENV_KEYS = [
|
||||
"SLACK_ALERT_WEBHOOK_URL",
|
||||
"SLACK_WEBHOOK_TIMEOUT_MS",
|
||||
"AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED",
|
||||
"CLIPROXY_TIER1_PLANNER_MODEL",
|
||||
"EVAL_CASE_DEADLINE_MS",
|
||||
"EVAL_LLM_PROVIDER",
|
||||
"EVAL_RUNTIME_LLM_ENABLED",
|
||||
"LLM_MODEL_ROUND_ROBIN_ENABLED",
|
||||
"OPENROUTER_TIER1_RESPONDER_MODEL",
|
||||
] as const;
|
||||
|
||||
describe("config", () => {
|
||||
@@ -169,4 +180,71 @@ describe("config", () => {
|
||||
expect(getToolAuditMaxLen()).toBe(42);
|
||||
expect(getNatsAclMode()).toBe("enforce");
|
||||
});
|
||||
|
||||
it("reads eval case deadline configuration", () => {
|
||||
expect(getEvalCaseDeadlineMs()).toBe(4_500);
|
||||
|
||||
vi.stubEnv("EVAL_CASE_DEADLINE_MS", "45000");
|
||||
|
||||
expect(getEvalCaseDeadlineMs()).toBe(45_000);
|
||||
});
|
||||
|
||||
it("enables eval runtime LLM by default with env override", () => {
|
||||
expect(getEvalRuntimeLlmEnabled()).toBe(true);
|
||||
|
||||
vi.stubEnv("EVAL_RUNTIME_LLM_ENABLED", "false");
|
||||
|
||||
expect(getEvalRuntimeLlmEnabled()).toBe(false);
|
||||
});
|
||||
|
||||
it("pins eval LLM provider to cliproxy by default", () => {
|
||||
expect(getEvalLlmProvider()).toBe("cliproxy");
|
||||
|
||||
vi.stubEnv("EVAL_LLM_PROVIDER", "all");
|
||||
expect(getEvalLlmProvider()).toBe("all");
|
||||
|
||||
vi.stubEnv("EVAL_LLM_PROVIDER", "openrouter");
|
||||
expect(getEvalLlmProvider()).toBe("openrouter");
|
||||
|
||||
vi.stubEnv("EVAL_LLM_PROVIDER", "invalid");
|
||||
expect(getEvalLlmProvider()).toBe("cliproxy");
|
||||
});
|
||||
|
||||
it("reads provider-scoped runtime LLM models", () => {
|
||||
vi.stubEnv("CLIPROXY_TIER1_PLANNER_MODEL", "cliproxy-planner");
|
||||
vi.stubEnv(
|
||||
"OPENROUTER_TIER1_RESPONDER_MODEL",
|
||||
"openrouter-fast, openrouter-backup",
|
||||
);
|
||||
|
||||
expect(
|
||||
getRuntimeLlmModelsForProvider(
|
||||
"cliproxy",
|
||||
{ role: "planner", tier: 1 },
|
||||
"local-qwen",
|
||||
),
|
||||
).toEqual(["cliproxy-planner"]);
|
||||
expect(
|
||||
getRuntimeLlmModelsForProvider(
|
||||
"openrouter",
|
||||
{ role: "responder", tier: 1 },
|
||||
"local-qwen",
|
||||
),
|
||||
).toEqual(["openrouter-fast", "openrouter-backup"]);
|
||||
expect(
|
||||
getRuntimeLlmModelsForProvider(
|
||||
"llama",
|
||||
{ role: "responder", tier: 1 },
|
||||
"local-qwen",
|
||||
),
|
||||
).toEqual(["local-qwen"]);
|
||||
});
|
||||
|
||||
it("keeps model round-robin disabled by default", () => {
|
||||
expect(getLlmModelRoundRobinEnabled()).toBe(false);
|
||||
|
||||
vi.stubEnv("LLM_MODEL_ROUND_ROBIN_ENABLED", "true");
|
||||
|
||||
expect(getLlmModelRoundRobinEnabled()).toBe(true);
|
||||
});
|
||||
});
|
||||
+130
-8
@@ -77,6 +77,23 @@ function getBoolean(envKey: string, fallback: boolean): boolean {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function getStringListFromEnv(...envKeys: string[]): string[] | undefined {
|
||||
for (const envKey of envKeys) {
|
||||
const raw = process.env[envKey]?.trim();
|
||||
if (!raw) continue;
|
||||
|
||||
const values = raw
|
||||
.split(/[\n,]/u)
|
||||
.map((value) => value.trim())
|
||||
.filter((value) => value.length > 0);
|
||||
if (values.length > 0) {
|
||||
return values;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function getNodeEnv(): "dev" | "test" | "prod" {
|
||||
const value = process.env["NODE_ENV"]?.trim();
|
||||
return value === "test" || value === "prod" ? value : "dev";
|
||||
@@ -390,6 +407,23 @@ export function getSupervisorLlmResponseEnabled(): boolean {
|
||||
return getBoolean("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", true);
|
||||
}
|
||||
|
||||
export type EvalLlmProvider = "all" | "cliproxy" | "llama" | "openrouter";
|
||||
|
||||
const EVAL_LLM_PROVIDERS = new Set<EvalLlmProvider>([
|
||||
"all",
|
||||
"cliproxy",
|
||||
"llama",
|
||||
"openrouter",
|
||||
]);
|
||||
|
||||
export function getEvalLlmProvider(): EvalLlmProvider {
|
||||
const raw = process.env["EVAL_LLM_PROVIDER"]?.trim().toLowerCase();
|
||||
if (raw && EVAL_LLM_PROVIDERS.has(raw as EvalLlmProvider)) {
|
||||
return raw as EvalLlmProvider;
|
||||
}
|
||||
return "cliproxy";
|
||||
}
|
||||
|
||||
export function getEvalCalibrationMinCorr(): number {
|
||||
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
||||
}
|
||||
@@ -515,6 +549,10 @@ export function getLlmCircuitCooldownMs(): number {
|
||||
return getPositiveNumber("LLM_CIRCUIT_COOLDOWN_MS", 60_000);
|
||||
}
|
||||
|
||||
export function getLlmModelRoundRobinEnabled(): boolean {
|
||||
return getBoolean("LLM_MODEL_ROUND_ROBIN_ENABLED", false);
|
||||
}
|
||||
|
||||
export function getClassifierMaxTokens(): number {
|
||||
return getPositiveNumber("CLASSIFIER_MAX_TOKENS", 16);
|
||||
}
|
||||
@@ -551,6 +589,14 @@ export function getEvalPrmMinLabels(): number {
|
||||
return getPositiveNumber("EVAL_PRM_MIN_LABELS", 500);
|
||||
}
|
||||
|
||||
export function getEvalCaseDeadlineMs(): number {
|
||||
return getPositiveNumber("EVAL_CASE_DEADLINE_MS", 4_500);
|
||||
}
|
||||
|
||||
export function getEvalRuntimeLlmEnabled(): boolean {
|
||||
return getBoolean("EVAL_RUNTIME_LLM_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getEvalAttributionTickMs(): number {
|
||||
return getPositiveNumber("EVAL_ATTRIBUTION_TICK_MS", 6 * 60 * 60 * 1000);
|
||||
}
|
||||
@@ -606,15 +652,91 @@ export function getTier2CriticModel(): string {
|
||||
}
|
||||
|
||||
export function getOpenRouterModels(): string[] {
|
||||
const raw = process.env["OPENROUTER_MODEL"]?.trim();
|
||||
if (!raw) {
|
||||
return ["openai/gpt-5.3-codex-spark"];
|
||||
return (
|
||||
getStringListFromEnv("OPENROUTER_MODEL", "OPENROUTER_MODELS") ?? [
|
||||
"openai/gpt-5.3-codex-spark",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
type RuntimeLlmProvider = "cliproxy" | "llama" | "openrouter";
|
||||
type RuntimeLlmRole = "critic" | "planner" | "responder" | undefined;
|
||||
type RuntimeLlmTier = 0 | 1 | 2 | undefined;
|
||||
|
||||
function getRuntimeRouteSuffix(args: {
|
||||
role?: RuntimeLlmRole;
|
||||
tier?: RuntimeLlmTier;
|
||||
}): string {
|
||||
if (args.role === "planner") {
|
||||
return "TIER1_PLANNER";
|
||||
}
|
||||
const models = raw
|
||||
.split(",")
|
||||
.map((value) => value.trim())
|
||||
.filter((value) => value.length > 0);
|
||||
return models.length > 0 ? models : ["openai/gpt-5.3-codex-spark"];
|
||||
if (args.role === "critic") {
|
||||
return "TIER2_CRITIC";
|
||||
}
|
||||
if (args.tier === 0) {
|
||||
return "TIER0";
|
||||
}
|
||||
return "TIER1_RESPONDER";
|
||||
}
|
||||
|
||||
function getProviderPrefix(provider: RuntimeLlmProvider): string {
|
||||
return provider.toUpperCase();
|
||||
}
|
||||
|
||||
function getProviderDefaultModels(
|
||||
provider: RuntimeLlmProvider,
|
||||
llamaFallbackModel: string,
|
||||
): string[] {
|
||||
if (provider === "cliproxy") {
|
||||
return (
|
||||
getStringListFromEnv(
|
||||
"CLIPROXY_MODEL",
|
||||
"CLIPROXY_MODELS",
|
||||
"CLI_PROXY_LLM_MODEL",
|
||||
) ?? ["gpt-5.3-codex-spark"]
|
||||
);
|
||||
}
|
||||
|
||||
if (provider === "openrouter") {
|
||||
return getOpenRouterModels();
|
||||
}
|
||||
|
||||
return (
|
||||
getStringListFromEnv("LLAMA_MODEL", "LLAMA_MODELS", "AGENT_LLM_MODEL") ?? [
|
||||
llamaFallbackModel,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
export function getRuntimeLlmModelsForProvider(
|
||||
provider: RuntimeLlmProvider,
|
||||
args: {
|
||||
role?: RuntimeLlmRole;
|
||||
tier?: RuntimeLlmTier;
|
||||
},
|
||||
llamaFallbackModel: string,
|
||||
): string[] {
|
||||
const suffix = getRuntimeRouteSuffix(args);
|
||||
const providerPrefix = getProviderPrefix(provider);
|
||||
const providerScopedModels = getStringListFromEnv(
|
||||
`${providerPrefix}_${suffix}_MODEL`,
|
||||
`${providerPrefix}_${suffix}_MODELS`,
|
||||
);
|
||||
if (providerScopedModels) {
|
||||
return providerScopedModels;
|
||||
}
|
||||
|
||||
if (provider === "cliproxy") {
|
||||
const legacyModels = getStringListFromEnv(
|
||||
`${suffix}_MODEL`,
|
||||
`${suffix}_MODELS`,
|
||||
);
|
||||
if (legacyModels) {
|
||||
return legacyModels;
|
||||
}
|
||||
}
|
||||
|
||||
return getProviderDefaultModels(provider, llamaFallbackModel);
|
||||
}
|
||||
|
||||
export function getLlmMaxTokens(): number {
|
||||
|
||||
@@ -138,3 +138,63 @@
|
||||
{"id":"case_acd612e0","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778740258770-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_7169cd5a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_81f2ad8f","nickname":"<nick:ba29bb75>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_6cd7106c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778718965990-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse={\"reply\":\"안녕하세요!\"}","input":{"request":{"requestId":"req_1477394f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_24cff593","nickname":"<nick:d455d0e4>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_de51735e","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778674030965-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_32cc22d6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cd95eff0","nickname":"<nick:7feae321>","trigger":"user-message","userMessage":"안녕하세오"}},"expected":{}}
|
||||
{"id":"case_4f45294b","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778756106669-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 여기 채팅방에 올리면 돼요. \n채팅방 오른쪽 상단 메뉴의 '커버 보기'로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷으로 보내면 됩니다.","input":{"request":{"requestId":"req_e6e21e71","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0dd56d1c","nickname":"<nick:7724addd>","trigger":"user-message","userMessage":"하트인증\n여따 올리면 되요?"}},"expected":{}}
|
||||
{"id":"case_e3ff18c6","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778756063255-0; llmOverall=2.75; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 확인했어요. 천천히 진행해 주세요.","input":{"request":{"requestId":"req_7b05c872","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0dd56d1c","nickname":"<nick:7724addd>","trigger":"user-message","userMessage":"네"}},"expected":{}}
|
||||
{"id":"case_1b906220","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778763820454-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 순서는 이름/출생연도/성별/티어로 맞게 돼 있고, 안 되는 부분은 마지막 티어 코드뿐이에요. \n현재의 '플'은 허용 티어 코드가 아니므로 GM, I, B, S, G, P, E, D, M, C, U 중 한 글자(예: D)로 바꿔주세요.","input":{"request":{"requestId":"req_a8ba4ffe","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_e3e5b006","nickname":"<nick:405abf95>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_4ff31f85","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778756157817-0; llmOverall=n/a; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 확인했어요. 천천히 진행해 주세요.","input":{"request":{"requestId":"req_c78edc0e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0dd56d1c","nickname":"<nick:7724addd>","trigger":"user-message","userMessage":"오"}},"expected":{}}
|
||||
{"id":"case_0fd45691","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778874307141-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=닉네임이 안 되는 이유는 형식이 맞지 않아서예요. \n현재 \"위례\"는 이름만 있고, 연도/성별/티어가 없어 형식이 깨졌습니다. \n카카오톡 닉네임은 [이름]/[출생연도 두 자리]/[남 또는 여]/[티어]로 바꿔주세요.","input":{"request":{"requestId":"req_b6f69c24","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"<nick:16fb6789>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_408c317c","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778874400236-0; llmOverall=2.25; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=채팅방 오른쪽 상단 메뉴 버튼을 눌러 “커버 보기”로 들어가면 하트 인증 화면이 보여요.","input":{"request":{"requestId":"req_1db51665","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"<nick:ad75d929>","trigger":"user-message","userMessage":"하트인증은 어디다 하나요?"}},"expected":{}}
|
||||
{"id":"case_cf22a9d8","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778874533535-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_60f5e74a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"<nick:ad75d929>","trigger":"user-message","userMessage":"넵 감사합니다"}},"expected":{}}
|
||||
{"id":"case_fd404eb4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778874508177-0; llmOverall=5; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=네, 가능합니다. 카카오톡 1단계가 완료된 상태라면 디스코드는 저녁에도 접속해 진행하시면 돼요.","input":{"request":{"requestId":"req_d814bae6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"<nick:ad75d929>","trigger":"user-message","userMessage":"디스코드는 저녁에 해도될까요?"}},"expected":{}}
|
||||
{"id":"case_d42b8624","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778874344992-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_47d100d2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0558bf70","nickname":"<nick:ad75d929>","trigger":"user-message","userMessage":"안녕하세여"}},"expected":{}}
|
||||
{"id":"case_bf09ad3a","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778954734843-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=현재 남은 절차는 카카오톡 닉네임 변경과 방 하트 인증입니다.","input":{"request":{"requestId":"req_010646c0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a2e8fc0b","nickname":"<nick:a9bad7ab>","trigger":"user-message","userMessage":"혹시 이시간엔 내전 안돌아가죠?"}},"expected":{}}
|
||||
{"id":"case_275402ce","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778950244576-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기예요.","input":{"request":{"requestId":"req_bc7c8aae","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"<nick:e448bd8b>","trigger":"user-message","userMessage":"5대5 하려고 하는데 가능 한지 해서용"}},"expected":{}}
|
||||
{"id":"case_0e56499c","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778950227521-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기예요.","input":{"request":{"requestId":"req_593ffaff","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"<nick:e448bd8b>","trigger":"user-message","userMessage":"아 지금 들어와 있는 관리자 분이용"}},"expected":{}}
|
||||
{"id":"case_c8398f09","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778949416363-0; llmOverall=1; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=웅/97/남/M님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_b0f0648f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"<nick:e448bd8b>","trigger":"user-message","userMessage":"혹시 지 외전 문의 드리려 하는데"}},"expected":{}}
|
||||
{"id":"case_0a460498","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778950169079-0; llmOverall=2; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=관리분(관리자)은 있어요.","input":{"request":{"requestId":"req_6bffed4f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"<nick:e448bd8b>","trigger":"user-message","userMessage":"관리분 있으신가요?"}},"expected":{}}
|
||||
{"id":"case_002ac417","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778931066262-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 \"강서/93/남\"은 티어가 빠져서 형식이 안 맞아요. \n강서/93/남은 티어가 빠져서 형식이 안 맞아요. 마지막에 티어 한 글자만 추가하면 됩니다. \n예: 강서/93/남/D","input":{"request":{"requestId":"req_ed284edb","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0d05dfd2","nickname":"<nick:1ec0deae>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_3e163e2e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778949203127-0; llmOverall=2.75; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 형식이 안 맞아요. \n카톡 닉네임은 구분자로 '/'를 써야 하고 4개 항목이 필요해요. 현재는 웅 97 남 M라 안 되고, 현재 닉네임은 웅/97/남/M 이렇게 고치면 됩니다.","input":{"request":{"requestId":"req_ab630198","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"<nick:6334c82e>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_1db9c911","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778958194603-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_f3d19239","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3e4be23b","nickname":"<nick:b09afa6a>","trigger":"user-message","userMessage":"안녕하세여"}},"expected":{}}
|
||||
{"id":"case_ee7ae44a","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778955694984-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_fea5fd76","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3592a95a","nickname":"<nick:437c8fc3>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_d6bd160f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778950199447-0; llmOverall=2.25; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=운영진께, 특히 관리자님께 문의하시면 돼요.","input":{"request":{"requestId":"req_5fbe8788","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"<nick:e448bd8b>","trigger":"user-message","userMessage":"누구한테 여쭤봐야 할까요"}},"expected":{}}
|
||||
{"id":"case_36b3ac9c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778949423115-0; llmOverall=n/a; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=웅/97/남/M님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_35e659fa","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ecdda046","nickname":"<nick:e448bd8b>","trigger":"user-message","userMessage":"어디에 문의 해야 할까용"}},"expected":{}}
|
||||
{"id":"case_f7a5d661","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779019400869-0; llmOverall=1; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_4861ee12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"<nick:444d812a>","trigger":"user-message","userMessage":"듀랭하실분들 있을까요"}},"expected":{}}
|
||||
{"id":"case_74eac586","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779019394267-0; llmOverall=1.25; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=톡 게시글의 공지를 참고하셔서 먼저 카카오톡 닉네임 변경과 하트 인증을 진행해 주세요!","input":{"request":{"requestId":"req_743e0456","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"<nick:444d812a>","trigger":"user-message","userMessage":"혹시"}},"expected":{}}
|
||||
{"id":"case_9e9195bb","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779019388776-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=@벽 넘어가는 탈론 00/남/E3님 닉네임에서 수정할 부분이 있어요!\n나이를 두 자리 숫자로 입력해주세요 (예: 25)\n성별(남/여)과 티어 코드를 '/'로 구분해서 입력해주세요","input":{"request":{"requestId":"req_568d875b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"<nick:444d812a>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_8acd1f91","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779011411116-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임에서 안 된 부분은 마지막 티어예요. \nG2는 허용 티어가 아니라서, 뒤의 G2를 GM, I, B, S, G, P, E, D, M, C, U 중 하나(예: G)로만 바꾸면 됩니다.","input":{"request":{"requestId":"req_9860e773","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40745cd2","nickname":"<nick:666d8731>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_d6824fdb","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779023514843-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_4d45f87c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bc77847e","nickname":"<nick:9ee38f43>","trigger":"user-message","userMessage":"안녕하세요 공지읽고오겠습니다"}},"expected":{}}
|
||||
{"id":"case_83136947","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779023414663-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요, 반갑습니다.","input":{"request":{"requestId":"req_d55e086e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9ca5eb1","nickname":"<nick:ebf85b46>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_1686cd08","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779019324629-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 /로 구분된 4개 항목(이름/출생연도 두 자리/성별/티어)이 아니어서 안 돼요. 지금은 항목이 2개로 잘못되어 있어요. \n고쳐야 할 부분은 연도/성별/티어를 다시 /로 정확히 나눠서 넣는 거예요. \n성별은 남 또는 여 한 글자, 티어는 GM/I/B/S/G/P/E/D/M/C/U 중 한 글자여야 해요.","input":{"request":{"requestId":"req_7cb58844","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bea0f163","nickname":"<nick:23dc710b>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_0f722e08","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779015436203-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_a6a1d38f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bc35f985","nickname":"<nick:c3de5ead>","trigger":"user-message","userMessage":"안녕하세요~"}},"expected":{}}
|
||||
{"id":"case_009d9558","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779011507015-0; llmOverall=2.25; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 \n카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널 메시지 남기기 입니다.","input":{"request":{"requestId":"req_18174c18","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40745cd2","nickname":"<nick:94137b1e>","trigger":"user-message","userMessage":"이러면되나요"}},"expected":{}}
|
||||
{"id":"case_b51a6dfb","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779011451230-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_5d3182b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40745cd2","nickname":"<nick:94137b1e>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_8f87e018","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779562992008-0; llmOverall=1; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=고구마는보라색/98/남/D님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_d9ff0f8b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ab97b973","nickname":"<nick:b410b374>","trigger":"user-message","userMessage":"봇인가요? 어케 이렇게 빠르지"}},"expected":{}}
|
||||
{"id":"case_51fec5d7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779563007004-0; llmOverall=1.25; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=고구마는보라색/98/남/D님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_058c9557","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ab97b973","nickname":"<nick:b410b374>","trigger":"user-message","userMessage":"아하"}},"expected":{}}
|
||||
{"id":"case_47db12bc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779544865402-0; llmOverall=1.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=방 하트 인증은 채팅방 오른쪽 상단 메뉴 버튼에서 \"커버 보기\"를 눌러 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.","input":{"request":{"requestId":"req_887be307","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_e35470e6","nickname":"<nick:01e9e582>","trigger":"user-message","userMessage":"어떻게 인증하나요 ?"}},"expected":{}}
|
||||
{"id":"case_8aa3d0bc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779574878821-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=아직 승인해드리기 어려워요. 남은 항목이 카카오톡 본인인증방 입장과 디스코드 주민신청 채널 메시지라서, 이 두 가지만 먼저 완료해주시면 될 것 같아요.","input":{"request":{"requestId":"req_117cdf74","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"<nick:f29257ca>","trigger":"user-message","userMessage":"디코 승인 부탁드려도 될까요?"}},"expected":{}}
|
||||
{"id":"case_0191603e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779574708797-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디스코드 관련 안내입니다. 디스코드 닉네임은 닉네임#태그/출생 연도 두 자리/티어 형식으로 수정해 주세요. 예시: 할리#프렌치불독/88/D \n디스코드 서버에서 이 닉네임은 본인 식별용이라 꼭 맞춰야 하고, 주민신청 채널에는 서버 소개 메시지도 남겨야 해요.","input":{"request":{"requestId":"req_17d4f38d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"<nick:f29257ca>","trigger":"user-message","userMessage":"/디코"}},"expected":{}}
|
||||
{"id":"case_214e18f8","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779562888510-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=@고구마는보라색/98/D/남님 닉네임에서 수정할 부분이 있어요!\n성별을 '남' 또는 '여'로 입력해주세요\n티어 코드(GM, I, B, S, G, P, E, D, M, C, U 중 하나)를 입력해주세요","input":{"request":{"requestId":"req_dd3e072b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ab97b973","nickname":"<nick:f7254b29>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_b90d32a5","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779535490115-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=네, 다시 보내주시면 돼요. \n방에서 오른쪽 상단 메뉴 버튼 열고 커버 보기로 들어간 뒤 하트를 눌러 빨간색으로 바꿔주세요. \n그 상태 화면을 채팅방에서 바로 캡처해서 편집(자르기/필터/화면 편집) 없이 원본 그대로 보내주시면 됩니다. \n참고로 회색 하트면 아직 인증으로 안 된 상태예요.","input":{"request":{"requestId":"req_8ac4ab90","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7d8e9f5b","nickname":"<nick:949aeac9>","trigger":"photo-failed"}},"expected":{}}
|
||||
{"id":"case_dc095d27","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779535154924-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=네, 다시 시도할 수 있어요. \n방 하트를 빨간색으로 바꾼 뒤, 채팅방에서 바로 캡처한 원본 화면(편집/자르기/필터 없음)만 새로 보내주시면 됩니다.","input":{"request":{"requestId":"req_bb9360ad","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7d8e9f5b","nickname":"<nick:949aeac9>","trigger":"photo-failed"}},"expected":{}}
|
||||
{"id":"case_c33b0453","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779535131374-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 하트는 눌렀다면 하트가 빨간색인 화면을 편집 없이 원본 스크린샷으로 올려주시면 돼요.","input":{"request":{"requestId":"req_5c1b0e13","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7d8e9f5b","nickname":"<nick:949aeac9>","trigger":"user-message","userMessage":"하트는 눌렀는대.."}},"expected":{}}
|
||||
{"id":"case_955dc7df","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1779605646714-0; llmOverall=2.5; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=오케이, 하트 인증은 화면이 제대로 올라오면 바로 확인돼요. \n지금 올리신 걸로 보냈다면 잠깐 확인 기다려볼게요. \n만약 아직 안 보인다면, 하트를 빨간색으로 바꾼 그대로의 원본 캡처만 한 번 더 보내주세요.","input":{"request":{"requestId":"req_fb38ace6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0ea265d7","nickname":"<nick:7a45b00c>","trigger":"user-message","userMessage":"위에 인증햇어여"}},"expected":{}}
|
||||
{"id":"case_e75e191e","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779604227776-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=다시 시도하려면 이렇게 하면 돼요.\n\n채팅방 오른쪽 상단 메뉴 버튼에서 커버 보기로 들어가서 하트를 빨간색으로 바꾼 뒤, \n그 장면이 보이게 편집 없이 원본 화면 캡처로 다시 보내주면 돼요.","input":{"request":{"requestId":"req_4368c115","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0ea265d7","nickname":"<nick:d0f3633e>","trigger":"photo-failed"}},"expected":{}}
|
||||
{"id":"case_f485439f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779593282207-0; llmOverall=n/a; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=Summer/95/남/S님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_a33cd695","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_18313eb7","nickname":"<nick:7ae6ee97>","trigger":"user-message","userMessage":"가입했습니다"}},"expected":{}}
|
||||
{"id":"case_75bc3458","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779593274596-0; llmOverall=n/a; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=Summer/95/남/S님 이제 추가 절차만 남았어요. 카카오톡 본인인증방 입장과 디스코드 주민신청 채널에 메시지 남기기을 확인해 주세요.","input":{"request":{"requestId":"req_5bb3e408","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_18313eb7","nickname":"<nick:7ae6ee97>","trigger":"user-message","userMessage":"디스코드"}},"expected":{}}
|
||||
{"id":"case_c1016e76","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779592708891-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 하세요.","input":{"request":{"requestId":"req_179a599b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_18313eb7","nickname":"<nick:50cd33f9>","trigger":"user-message","userMessage":"엇 잠시만용"}},"expected":{}}
|
||||
{"id":"case_12f3372d","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779574484786-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 바로 여기 채팅방에 올리면 됩니다. \n채팅방 오른쪽 상단 메뉴의 \"커버 보기\"에서 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷으로 보내주세요.","input":{"request":{"requestId":"req_b7b755bf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"<nick:f29257ca>","trigger":"user-message","userMessage":"하트인증은 여기다 올리면 될까요?"}},"expected":{}}
|
||||
{"id":"case_9f0cb2f4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779574363525-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_b822a53e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_53e8e877","nickname":"<nick:90b41f68>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_885a35a9","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629981088-0; llmOverall=1; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=톡 게시글의 공지를 참고하셔서 먼저 카카오톡 닉네임 변경과 하트 인증을 진행해 주세요!","input":{"request":{"requestId":"req_42415f0f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"<nick:6110a7e7>","trigger":"user-message","userMessage":"진짜 잘하시네요"}},"expected":{}}
|
||||
{"id":"case_4603c0cc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629976794-0; llmOverall=1; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_5400a31f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"<nick:6110a7e7>","trigger":"user-message","userMessage":"챌린저 ㄷㄷ"}},"expected":{}}
|
||||
{"id":"case_2b38adb8","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629930176-0; llmOverall=1; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=채팅방 프로필에서 하트를 눌러 빨간색으로 만든 다음, 그 화면이 보이게 원본 스크린샷을 보내주시면 인증돼요.","input":{"request":{"requestId":"req_bf140842","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"<nick:68a7fbf0>","trigger":"user-message","userMessage":"어떻게 하나여"}},"expected":{}}
|
||||
{"id":"case_f04b5287","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779633055712-0; llmOverall=1.25; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요, 오늘도 편하게 진행해요.","input":{"request":{"requestId":"req_bdd1114b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"<nick:c673b3e2>","trigger":"user-message","userMessage":"안녕하세요!"}},"expected":{}}
|
||||
{"id":"case_3e077513","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779630086906-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 그렇게 해주시면 돼요.","input":{"request":{"requestId":"req_bbc426e2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"<nick:6110a7e7>","trigger":"user-message","userMessage":"아항"}},"expected":{}}
|
||||
{"id":"case_274fec57","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629952313-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_c69053e8","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"<nick:68a7fbf0>","trigger":"user-message","userMessage":"보내면 되나여?"}},"expected":{}}
|
||||
{"id":"case_29f02c6e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1779629936899-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=톡 게시글의 공지를 참고하셔서 먼저 카카오톡 닉네임 변경과 하트 인증을 진행해 주세요!","input":{"request":{"requestId":"req_be0e7f4a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"<nick:68a7fbf0>","trigger":"user-message","userMessage":"여기 채팅방에"}},"expected":{}}
|
||||
{"id":"case_59c880e4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779633743117-0; llmOverall=4.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=카카오톡에서 디스코드 초대 안내를 받았던 메시지를 찾아 초대 수락을 눌러 들어가면 돼요.","input":{"request":{"requestId":"req_705c9e3f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"<nick:c673b3e2>","trigger":"user-message","userMessage":"저 디코는 어떻게 들어가요? 주민신청까지 해뒀습니다"}},"expected":{}}
|
||||
{"id":"case_4fe5864d","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779632430014-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_781682ec","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"<nick:658a6c82>","trigger":"user-message","userMessage":"넵"}},"expected":{}}
|
||||
{"id":"case_9a3640ce","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779632381253-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=안 되는 이유는 티어가 허용 코드(GM, I, B, S, G, P, E, D, M, C, U) 중 하나의 대문자로 되어야 하는데 p가 소문자여서예요. \n카카오톡 닉네임의 마지막 부분만 P로 바꿔서 세나/95/여/P로 수정해 주세요.","input":{"request":{"requestId":"req_7368778f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ade16621","nickname":"<nick:1f050511>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_5ea6c4d2","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779629969676-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=환영해요! 반갑습니다.","input":{"request":{"requestId":"req_9352c82d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_41e5d293","nickname":"<nick:6110a7e7>","trigger":"user-message","userMessage":"안녕하세요!!"}},"expected":{}}
|
||||
{"id":"case_207dec80","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779629904020-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=방 하트 인증은 채팅방 오른쪽 상단 메뉴 버튼의 '커버 보기'에서 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 화면이 보이게 스크린샷을 보내주시면 돼요.","input":{"request":{"requestId":"req_37928f7d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_80c06d53","nickname":"<nick:68a7fbf0>","trigger":"user-message","userMessage":"방하트 인증은"}},"expected":{}}
|
||||
{"id":"case_b42ca588","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779622035543-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=어서오세요!","input":{"request":{"requestId":"req_e9dcff68","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3adc9936","nickname":"<nick:ccb74517>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_d96d1b98","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1779619834487-0; llmOverall=n/a; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 본인인증이 완료됐다면 본인인증방은 나가셔도 돼요. \n현재 남은 절차는 디스코드 주민신청 채널 메시지 남기기 하나예요.","input":{"request":{"requestId":"req_e6bc023c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_d9c80f01","nickname":"<nick:9c5d5e3a>","trigger":"user-message","userMessage":"본인인증방은이제나가도돼요?"}},"expected":{}}
|
||||
@@ -93,6 +93,29 @@ describe("judgeOnboardingWithLlm", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps scores when rationale is longer than the prompt asks for", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient(
|
||||
JSON.stringify({
|
||||
friendliness: 3,
|
||||
accuracy: 4,
|
||||
concision: 5,
|
||||
scope: 4,
|
||||
rationale: "가".repeat(800),
|
||||
}),
|
||||
),
|
||||
evalCase,
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
derived,
|
||||
);
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.scores.overall).toBe(4);
|
||||
expect(result.rationale).toHaveLength(500);
|
||||
}
|
||||
});
|
||||
|
||||
it("skips when the judge returns invalid JSON", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient('{"friendliness":"bad"}'),
|
||||
|
||||
@@ -9,6 +9,8 @@ import type {
|
||||
} from "../types.js";
|
||||
import { getSkillJudgeConfig } from "./skill-config.js";
|
||||
|
||||
const MAX_RATIONALE_CHARS = 500;
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
@@ -18,10 +20,14 @@ function createJudgeSchema(axes: readonly string[]): z.ZodObject<z.ZodRawShape>
|
||||
...Object.fromEntries(
|
||||
axes.map((axis) => [axis, z.number().int().min(1).max(5)]),
|
||||
),
|
||||
rationale: z.string().trim().min(1).max(200),
|
||||
rationale: z.string().trim().min(1),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeRationale(value: unknown): string {
|
||||
return String(value).slice(0, MAX_RATIONALE_CHARS);
|
||||
}
|
||||
|
||||
function buildJudgePrompt(
|
||||
skillName: string,
|
||||
evalCase: EvalCaseDefinition,
|
||||
@@ -35,6 +41,7 @@ function buildJudgePrompt(
|
||||
`다음 ${skillName} 에이전트 응답을 평가해 주세요.`,
|
||||
`평가 축은 ${axes} 입니다.`,
|
||||
"각 점수는 1~5의 정수만 사용하세요.",
|
||||
"rationale은 200자 이하로 짧게 작성하세요.",
|
||||
config.promptTemplate ??
|
||||
"accuracy는 기대 조건과 실제 답변의 사실성을, concision은 과도한 부가 설명 여부를 포함해 평가하세요.",
|
||||
"",
|
||||
@@ -129,7 +136,7 @@ export async function judgeSkillWithLlm(
|
||||
endpoint: response.endpoint,
|
||||
provider: response.provider,
|
||||
scores,
|
||||
rationale: String(parsed["rationale"]),
|
||||
rationale: normalizeRationale(parsed["rationale"]),
|
||||
raw: response.content,
|
||||
};
|
||||
} catch {
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"description": "deadline-near 상태에서도 사용자가 묻지 않은 남은 등록 절차를 자동으로 붙이지 않도록 우선순위 강화",
|
||||
"diff": [
|
||||
{
|
||||
"layer": "policy",
|
||||
"op": "append",
|
||||
"reason": "첫 번째 실패에서 현재 상태의 remaining 정보가 질문보다 우선되어 무관한 절차 안내가 출력됐다. deadline-near에서도 질문 우선 원칙을 강화한다.",
|
||||
"text": "[deadline-near 응답 범위 제한]\n- phase가 deadline-near여도 마지막 메시지가 가능 여부/확인 주체/확인 여부를 묻는 질문이면, 독촉성 남은 절차 안내를 붙이지 마.\n- remaining 상태는 사용자가 \"뭐가 남았나요\", \"다 했나요\", \"완료인가요\"처럼 진행 상태를 물을 때만 사용해.\n- 답변 전 마지막으로 확인: 내 응답 첫 문장이 사용자의 질문에 대한 '네/아니요/아직/관리자가 확인' 중 직접 답인지 점검해."
|
||||
}
|
||||
],
|
||||
"expectedImprovement": "무관한 카카오톡/디스코드 절차 나열이 줄고, 사용자의 마지막 질문 중심으로 1문장 내외의 답변이 생성된다.",
|
||||
"rationale": "현재 스냅샷에는 remaining 정보가 강하게 주입되어 있어, 사용자가 재등록 가능 여부만 묻는 상황에서도 남은 절차를 안내할 위험이 높다. 진행 상태 질문이 아닐 때 remaining을 쓰지 않게 하면 범위 이탈을 줄일 수 있다.",
|
||||
"skill": "onboarding",
|
||||
"variantId": "579dcc7d",
|
||||
"trustTier": "T2"
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"description": "디스코드 확인 주체/재등록 가능 여부 질문에는 상태 나열이나 불확실 회피 대신 짧게 직접 답하도록 하는 예외 규칙 추가",
|
||||
"diff": [
|
||||
{
|
||||
"layer": "policy",
|
||||
"op": "append",
|
||||
"reason": "실패 케이스가 모두 사용자의 단일 질문에 답하지 못하고, 확인 불가 회피 또는 무관한 남은 절차 나열로 이탈했다. 해당 slice에서 반복되는 직접 질문 패턴을 좁게 처리한다.",
|
||||
"text": "[discord-verify-question 직접응답 보강]\n- 사용자가 \"디코는 누가 확인하나요\", \"관리자가 확인하나요\", \"관리자 형님들이 확인해야 하나요\"처럼 디스코드 확인 주체를 물으면, 확인되지 않았다고 회피하지 말고 \"네, 디스코드 관련 확인은 관리자/운영진이 확인해요.\"라고 짧게 답해.\n- 사용자가 \"계정 확인 후 다시 등록해도 되나요\", \"제 명의 계정 확인 후 다시 등록해도 될까요\"처럼 본인 계정/명의 확인 뒤 재등록 가능 여부를 물으면, 다른 절차를 나열하지 말고 \"네, 본인 명의 계정 확인 후 다시 등록해 주시면 돼요.\"라고 답해.\n- 위 두 유형에서는 카카오톡 완료 여부, 남은 단계, 디스코드 닉네임 형식, 다음 절차를 사용자가 묻지 않는 한 덧붙이지 마."
|
||||
}
|
||||
],
|
||||
"expectedImprovement": "디스코드 확인 주체 질문에는 '관리자/운영진이 확인한다'고 정확히 답하고, 계정 확인 후 재등록 질문에는 '가능하다'고 직접 답해 accuracy와 scope가 개선된다.",
|
||||
"rationale": "현재 프롬프트의 '확인되지 않았으면 확인되지 않았다고만 답'과 '세션 상태' 정보가 과하게 작동해, 실제로는 일반 운영 규칙으로 답할 수 있는 질문에도 회피하거나 상태 목록을 출력한다. 확인 주체와 재등록 가능 여부는 온보딩 절차의 일반 질문이므로 짧은 정답 템플릿을 추가하는 것이 가장 작은 수정이다.",
|
||||
"skill": "onboarding",
|
||||
"variantId": "991bfb18",
|
||||
"trustTier": "T2"
|
||||
}
|
||||
+52
-16
@@ -31,7 +31,13 @@ import type {
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
||||
import {
|
||||
getEvalCaseDeadlineMs,
|
||||
getEvalLlmProvider,
|
||||
getEvalRuntimeLlmEnabled,
|
||||
getOnboardingSessionKeyPrefix,
|
||||
getOpenRouterApiUrl,
|
||||
} from "../config.js";
|
||||
import { runEvalDispatch } from "./handoff-dispatch.js";
|
||||
import { judgeHandoff } from "./judges/handoff-judge.js";
|
||||
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||
@@ -285,6 +291,9 @@ function toProviderEndpoints(
|
||||
}
|
||||
|
||||
export function createEvalLlmClient(): LlmClient {
|
||||
const evalProvider = getEvalLlmProvider();
|
||||
const includeProvider = (provider: ProviderName): boolean =>
|
||||
evalProvider === "all" || evalProvider === provider;
|
||||
const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]);
|
||||
const llamaEndpoints = parseEndpointList(
|
||||
process.env["AGENT_LLM_API_URLS"],
|
||||
@@ -301,7 +310,14 @@ export function createEvalLlmClient(): LlmClient {
|
||||
);
|
||||
const providerApiKeys = Object.fromEntries(
|
||||
[
|
||||
["cliproxy", firstNonEmpty(process.env["CLI_PROXY_API_KEY"])],
|
||||
[
|
||||
"cliproxy",
|
||||
firstNonEmpty(
|
||||
process.env["CLI_PROXY_API_KEY"],
|
||||
process.env["LLAMA_SERVER_API_KEY"],
|
||||
process.env["AGENT_LLM_API_KEY"],
|
||||
),
|
||||
],
|
||||
["llama", llmApiKey],
|
||||
["openrouter", openRouterApiKey],
|
||||
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
||||
@@ -309,9 +325,15 @@ export function createEvalLlmClient(): LlmClient {
|
||||
|
||||
return new LlmClient({
|
||||
endpoints: [
|
||||
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
|
||||
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
|
||||
...toProviderEndpoints(llamaEndpoints, "llama"),
|
||||
...(includeProvider("cliproxy")
|
||||
? toProviderEndpoints(cliproxyEndpoints, "cliproxy")
|
||||
: []),
|
||||
...(includeProvider("openrouter")
|
||||
? toProviderEndpoints(openRouterEndpoints, "openrouter")
|
||||
: []),
|
||||
...(includeProvider("llama")
|
||||
? toProviderEndpoints(llamaEndpoints, "llama")
|
||||
: []),
|
||||
],
|
||||
apiKey: llmApiKey,
|
||||
model: firstNonEmpty(process.env["AGENT_LLM_MODEL"]),
|
||||
@@ -846,7 +868,8 @@ export async function loadEvalCases(
|
||||
|
||||
async function evaluateCase(
|
||||
evalCase: EvalCaseDefinition,
|
||||
llmClient: LlmClient,
|
||||
runtimeLlmClient: LlmClient,
|
||||
judgeLlmClient: LlmClient,
|
||||
enableLlmJudge: boolean,
|
||||
skillName: string,
|
||||
): Promise<EvalCaseResult> {
|
||||
@@ -860,7 +883,7 @@ async function evaluateCase(
|
||||
: undefined;
|
||||
const deps = {
|
||||
garnet: inMemoryGarnet,
|
||||
llm: llmClient,
|
||||
llm: runtimeLlmClient,
|
||||
room: roomContextService,
|
||||
};
|
||||
if (skillName === "handoff") {
|
||||
@@ -868,7 +891,7 @@ async function evaluateCase(
|
||||
evalCase,
|
||||
input,
|
||||
Object.values(SKILL_RUNNER_FACTORIES).map((factory) => factory(deps)),
|
||||
llmClient,
|
||||
judgeLlmClient,
|
||||
enableLlmJudge,
|
||||
);
|
||||
}
|
||||
@@ -883,7 +906,7 @@ async function evaluateCase(
|
||||
return runWithContext(
|
||||
{
|
||||
chatroomId: input.request.chatroomId,
|
||||
deadlineAt: new Date(Date.now() + 4_500).toISOString(),
|
||||
deadlineAt: new Date(Date.now() + getEvalCaseDeadlineMs()).toISOString(),
|
||||
requestId: input.request.requestId,
|
||||
skill: input.request.skill,
|
||||
startedAt: new Date().toISOString(),
|
||||
@@ -896,7 +919,8 @@ async function evaluateCase(
|
||||
evalCase,
|
||||
input,
|
||||
skill,
|
||||
llmClient,
|
||||
runtimeLlmClient,
|
||||
judgeLlmClient,
|
||||
enableLlmJudge,
|
||||
effectiveSkill,
|
||||
)
|
||||
@@ -904,7 +928,7 @@ async function evaluateCase(
|
||||
evalCase,
|
||||
input,
|
||||
skill,
|
||||
llmClient,
|
||||
judgeLlmClient,
|
||||
enableLlmJudge,
|
||||
effectiveSkill,
|
||||
),
|
||||
@@ -915,11 +939,12 @@ async function evaluateOnboardingCase(
|
||||
evalCase: EvalCaseDefinition,
|
||||
input: EvalCaseInput,
|
||||
skill: Skill,
|
||||
llmClient: LlmClient,
|
||||
runtimeLlmClient: LlmClient,
|
||||
judgeLlmClient: LlmClient,
|
||||
enableLlmJudge: boolean,
|
||||
skillName: string,
|
||||
): Promise<EvalCaseResult> {
|
||||
const classifier = new OnboardingIntentClassifier(llmClient);
|
||||
const classifier = new OnboardingIntentClassifier(runtimeLlmClient);
|
||||
const processTrace = new ProcessTraceCollector();
|
||||
const judgeConfig = getSkillJudgeConfig(skillName);
|
||||
|
||||
@@ -950,7 +975,7 @@ async function evaluateOnboardingCase(
|
||||
});
|
||||
const llmJudge = enableLlmJudge
|
||||
? await judgeSkillWithLlm(
|
||||
llmClient,
|
||||
judgeLlmClient,
|
||||
skillName,
|
||||
evalCase,
|
||||
response.text,
|
||||
@@ -1262,12 +1287,23 @@ export async function runEval(
|
||||
limit: options?.limit,
|
||||
});
|
||||
|
||||
const llmClient = createEvalLlmClient();
|
||||
const judgeLlmClient = createEvalLlmClient();
|
||||
const runtimeLlmClient = getEvalRuntimeLlmEnabled()
|
||||
? judgeLlmClient
|
||||
: new LlmClient({ endpoints: [] });
|
||||
const enableLlmJudge = options?.enableLlmJudge ?? true;
|
||||
const caseResults: EvalCaseResult[] = [];
|
||||
|
||||
for (const evalCase of evalCases) {
|
||||
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge, skill));
|
||||
caseResults.push(
|
||||
await evaluateCase(
|
||||
evalCase,
|
||||
runtimeLlmClient,
|
||||
judgeLlmClient,
|
||||
enableLlmJudge,
|
||||
skill,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const summary = createSummary(caseResults);
|
||||
|
||||
+18
-1
@@ -194,9 +194,10 @@ export class LlmClient {
|
||||
continue;
|
||||
}
|
||||
|
||||
const configuredRequestTimeoutMs = getLlmRequestTimeoutMs();
|
||||
const requestTimeoutMs = Math.max(
|
||||
1,
|
||||
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
|
||||
Math.min(configuredRequestTimeoutMs, options?.deadlineMs ?? Number.POSITIVE_INFINITY),
|
||||
);
|
||||
const models = selectModelsForProvider(provider.name, options, this.llamaModel);
|
||||
let endpointSucceeded = false;
|
||||
@@ -241,6 +242,9 @@ export class LlmClient {
|
||||
model: request.model,
|
||||
status: response.status,
|
||||
body,
|
||||
configuredTimeoutMs: configuredRequestTimeoutMs,
|
||||
deadlineMs: options?.deadlineMs,
|
||||
timeoutMs: requestTimeoutMs,
|
||||
},
|
||||
"[LLM] Request failed",
|
||||
);
|
||||
@@ -304,6 +308,16 @@ export class LlmClient {
|
||||
if (llmResult) {
|
||||
await this.circuit.recordSuccess(endpoint.completionUrl);
|
||||
endpointSucceeded = true;
|
||||
logger.debug(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
model: request.model,
|
||||
provider: provider.name,
|
||||
role: options?.role,
|
||||
tier: options?.tier,
|
||||
},
|
||||
"[LLM] Response received",
|
||||
);
|
||||
return llmResult;
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -313,6 +327,9 @@ export class LlmClient {
|
||||
provider: provider.name,
|
||||
model,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
configuredTimeoutMs: configuredRequestTimeoutMs,
|
||||
deadlineMs: options?.deadlineMs,
|
||||
timeoutMs: requestTimeoutMs,
|
||||
},
|
||||
"[LLM] Request error",
|
||||
);
|
||||
|
||||
+63
-6
@@ -1,6 +1,7 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { LlmEndpointConfig } from "./types.js";
|
||||
import {
|
||||
resetLlmRoutingRoundRobinForTests,
|
||||
selectMaxTokensForTier,
|
||||
selectModelsForProvider,
|
||||
selectProviderOrder,
|
||||
@@ -8,6 +9,12 @@ import {
|
||||
|
||||
describe("llm routing", () => {
|
||||
afterEach(() => {
|
||||
delete process.env["CLIPROXY_TIER0_MODEL"];
|
||||
delete process.env["CLIPROXY_TIER1_PLANNER_MODEL"];
|
||||
delete process.env["CLIPROXY_TIER2_CRITIC_MODEL"];
|
||||
delete process.env["LLAMA_TIER2_CRITIC_MODEL"];
|
||||
delete process.env["LLM_MODEL_ROUND_ROBIN_ENABLED"];
|
||||
delete process.env["OPENROUTER_TIER1_RESPONDER_MODEL"];
|
||||
delete process.env["OPENROUTER_MODEL"];
|
||||
delete process.env["TIER0_MODEL"];
|
||||
delete process.env["TIER1_PLANNER_MODEL"];
|
||||
@@ -16,6 +23,7 @@ describe("llm routing", () => {
|
||||
delete process.env["TIER0_MAX_TOKENS"];
|
||||
delete process.env["TIER1_MAX_TOKENS"];
|
||||
delete process.env["TIER2_MAX_TOKENS"];
|
||||
resetLlmRoutingRoundRobinForTests();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -93,9 +101,12 @@ describe("llm routing", () => {
|
||||
).toEqual(["cliproxy", "openrouter"]);
|
||||
});
|
||||
|
||||
it("returns role-aware model for cliproxy and llama fallback as single-item list", () => {
|
||||
process.env["TIER1_PLANNER_MODEL"] = "cliproxy-planner";
|
||||
process.env["TIER2_CRITIC_MODEL"] = "cliproxy-critic";
|
||||
it("returns provider-aware models for runtime roles and tiers", () => {
|
||||
process.env["CLIPROXY_TIER1_PLANNER_MODEL"] = "cliproxy-planner";
|
||||
process.env["CLIPROXY_TIER2_CRITIC_MODEL"] = "cliproxy-critic";
|
||||
process.env["OPENROUTER_TIER1_RESPONDER_MODEL"] =
|
||||
"openrouter-fast, openrouter-backup";
|
||||
process.env["LLAMA_TIER2_CRITIC_MODEL"] = "llama-critic";
|
||||
|
||||
expect(
|
||||
selectModelsForProvider("cliproxy", { role: "planner", tier: 1 }, "local-qwen"),
|
||||
@@ -103,9 +114,12 @@ describe("llm routing", () => {
|
||||
expect(
|
||||
selectModelsForProvider("cliproxy", { role: "critic", tier: 2 }, "local-qwen"),
|
||||
).toEqual(["cliproxy-critic"]);
|
||||
expect(selectModelsForProvider("llama", { tier: 2 }, "local-qwen")).toEqual([
|
||||
"local-qwen",
|
||||
]);
|
||||
expect(
|
||||
selectModelsForProvider("openrouter", { role: "responder", tier: 1 }, "local-qwen"),
|
||||
).toEqual(["openrouter-fast", "openrouter-backup"]);
|
||||
expect(
|
||||
selectModelsForProvider("llama", { role: "critic", tier: 2 }, "local-qwen"),
|
||||
).toEqual(["llama-critic"]);
|
||||
});
|
||||
|
||||
it("splits OPENROUTER_MODEL by comma and preserves priority order", () => {
|
||||
@@ -124,6 +138,49 @@ describe("llm routing", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("round-robins models within the same provider role and tier when enabled", () => {
|
||||
process.env["LLM_MODEL_ROUND_ROBIN_ENABLED"] = "true";
|
||||
process.env["CLIPROXY_TIER0_MODEL"] =
|
||||
"gemini-3.1-flash-lite, gpt-5.3-codex-spark";
|
||||
|
||||
expect(
|
||||
selectModelsForProvider("cliproxy", { tier: 0 }, "local-qwen"),
|
||||
).toEqual(["gemini-3.1-flash-lite", "gpt-5.3-codex-spark"]);
|
||||
expect(
|
||||
selectModelsForProvider("cliproxy", { tier: 0 }, "local-qwen"),
|
||||
).toEqual(["gpt-5.3-codex-spark", "gemini-3.1-flash-lite"]);
|
||||
expect(
|
||||
selectModelsForProvider("cliproxy", { tier: 0 }, "local-qwen"),
|
||||
).toEqual(["gemini-3.1-flash-lite", "gpt-5.3-codex-spark"]);
|
||||
});
|
||||
|
||||
it("keeps provider order fixed even when model round-robin is enabled", () => {
|
||||
process.env["LLM_MODEL_ROUND_ROBIN_ENABLED"] = "true";
|
||||
const endpoints: LlmEndpointConfig[] = [
|
||||
{
|
||||
completionUrl: "http://llama.test/v1/chat/completions",
|
||||
healthUrl: "http://llama.test/health",
|
||||
provider: "llama",
|
||||
},
|
||||
{
|
||||
completionUrl: "http://openrouter.test/v1/chat/completions",
|
||||
healthUrl: "http://openrouter.test",
|
||||
provider: "openrouter",
|
||||
},
|
||||
{
|
||||
completionUrl: "http://cliproxy.test/v1/chat/completions",
|
||||
healthUrl: "http://cliproxy.test",
|
||||
provider: "cliproxy",
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
selectProviderOrder(endpoints, { role: "responder", tier: 1 }).map(
|
||||
(endpoint) => endpoint.provider,
|
||||
),
|
||||
).toEqual(["cliproxy", "openrouter", "llama"]);
|
||||
});
|
||||
|
||||
it("chooses tier defaults for max tokens when explicit maxTokens is absent", () => {
|
||||
process.env["TIER0_MAX_TOKENS"] = "111";
|
||||
process.env["TIER1_MAX_TOKENS"] = "222";
|
||||
|
||||
+37
-26
@@ -1,11 +1,8 @@
|
||||
import {
|
||||
getOpenRouterModels,
|
||||
getLlmModelRoundRobinEnabled,
|
||||
getRuntimeLlmModelsForProvider,
|
||||
getTier0MaxTokens,
|
||||
getTier0Model,
|
||||
getTier1MaxTokens,
|
||||
getTier1PlannerModel,
|
||||
getTier1ResponderModel,
|
||||
getTier2CriticModel,
|
||||
getTier2MaxTokens,
|
||||
getLlmMaxTokens,
|
||||
} from "../config.js";
|
||||
@@ -13,6 +10,7 @@ import type { ChatCompletionOptions, LlmEndpointConfig, ProviderName } from "./t
|
||||
|
||||
const PROVIDER_PRIORITY: readonly ProviderName[] = ["cliproxy", "openrouter", "llama"];
|
||||
const TEMPLATE_ONLY_PROVIDER_PRIORITY: readonly ProviderName[] = ["llama"];
|
||||
const modelRoundRobinOffsets = new Map<string, number>();
|
||||
|
||||
function getProviderPriority(
|
||||
options: ChatCompletionOptions | undefined,
|
||||
@@ -33,27 +31,15 @@ export function selectModelsForProvider(
|
||||
return [options.modelOverride.trim()];
|
||||
}
|
||||
|
||||
if (provider === "llama") {
|
||||
return [llamaFallbackModel];
|
||||
}
|
||||
|
||||
if (provider === "openrouter") {
|
||||
return getOpenRouterModels();
|
||||
}
|
||||
|
||||
if (options?.role === "planner") {
|
||||
return [getTier1PlannerModel()];
|
||||
}
|
||||
|
||||
if (options?.role === "critic") {
|
||||
return [getTier2CriticModel()];
|
||||
}
|
||||
|
||||
if (options?.tier === 1 || options?.tier === 2) {
|
||||
return [getTier1ResponderModel()];
|
||||
}
|
||||
|
||||
return [getTier0Model()];
|
||||
const models = getRuntimeLlmModelsForProvider(
|
||||
provider,
|
||||
{
|
||||
role: options?.role,
|
||||
tier: options?.tier,
|
||||
},
|
||||
llamaFallbackModel,
|
||||
);
|
||||
return rotateModelsForProvider(provider, options, models);
|
||||
}
|
||||
|
||||
export function selectMaxTokensForTier(
|
||||
@@ -106,3 +92,28 @@ export function selectProviderOrder(
|
||||
})
|
||||
.map((item) => item.endpoint);
|
||||
}
|
||||
|
||||
function rotateModelsForProvider(
|
||||
provider: ProviderName,
|
||||
options: ChatCompletionOptions | undefined,
|
||||
models: string[],
|
||||
): string[] {
|
||||
if (!getLlmModelRoundRobinEnabled() || models.length <= 1) {
|
||||
return models;
|
||||
}
|
||||
|
||||
const key = [
|
||||
provider,
|
||||
options?.role ?? "none",
|
||||
options?.tier ?? "none",
|
||||
models.join("|"),
|
||||
].join(":");
|
||||
const offset = modelRoundRobinOffsets.get(key) ?? 0;
|
||||
modelRoundRobinOffsets.set(key, offset + 1);
|
||||
const rotation = offset % models.length;
|
||||
return [...models.slice(rotation), ...models.slice(0, rotation)];
|
||||
}
|
||||
|
||||
export function resetLlmRoutingRoundRobinForTests(): void {
|
||||
modelRoundRobinOffsets.clear();
|
||||
}
|
||||
@@ -38,6 +38,48 @@ export const ONBOARDING_INTENT_TABLE = {
|
||||
match: ({ trimmed }) => isRemainingStatusQuestion(trimmed),
|
||||
triggers: ["progress-update", "timeout-warning"],
|
||||
},
|
||||
"discord-verify-question": {
|
||||
confidence: 0.96,
|
||||
keywords: ["디스코드", "확인", "닉네임", "주민신청"],
|
||||
llm: true,
|
||||
match: ({ lower, trimmed }) => {
|
||||
const mentionsDiscord = /(디스코드|디코|discord)/u.test(lower);
|
||||
const asksDiscordVerification =
|
||||
/(확인|조회|찾아|검색|검사|바뀌|변경|닉변|맞(?:아|나요)?|됐(?:나|나요)?|완료됐(?:나|나요)?|승인|반영)/u.test(
|
||||
trimmed,
|
||||
);
|
||||
const reportsDiscordNicknameCompletion =
|
||||
/(닉네임|닉변|프로필\s*닉네임|서버\s*닉네임)/u.test(trimmed) &&
|
||||
/(완료했(?:어|어요|습니다)?|끝냈(?:어|어요|습니다)?|설정했(?:어|어요|습니다)?|바꿨(?:어|어요|습니다)?|변경했(?:어|어요|습니다)?|다\s*했(?:어|어요|습니다)?|해놨(?:어|어요|습니다)?|해뒀(?:어|어요|습니다)?|적용됐(?:어|어요|습니다)?)/u.test(
|
||||
trimmed,
|
||||
);
|
||||
const reportsResidentApplicationCompletion =
|
||||
/(주민\s*신청|주민신청|주민\s*등록|주민등록|프로필\s*메시지|메시지)/u.test(
|
||||
trimmed,
|
||||
) &&
|
||||
/(완료했(?:어|어요|습니다)?|끝냈(?:어|어요|습니다)?|남겼(?:어|어요|습니다)?|썼(?:어|어요|습니다)?|작성했(?:어|어요|습니다)?|다\s*했(?:어|어요|습니다)?|해놨(?:어|어요|습니다)?|해뒀(?:어|어요|습니다)?|했(?:어|어요|습니다)?)/u.test(
|
||||
trimmed,
|
||||
);
|
||||
return (
|
||||
mentionsDiscord &&
|
||||
(asksDiscordVerification ||
|
||||
reportsDiscordNicknameCompletion ||
|
||||
reportsResidentApplicationCompletion)
|
||||
);
|
||||
},
|
||||
},
|
||||
"discord-question": {
|
||||
confidence: 0.95,
|
||||
keywords: ["디스코드", "주민신청", "프로필", "권한", "설정"],
|
||||
llm: true,
|
||||
match: ({ lower }) => /(디스코드|디코|discord)/u.test(lower),
|
||||
regex: [
|
||||
{
|
||||
confidence: 0.94,
|
||||
pattern: /(주민신청\s*채널|주민신청채널|서버\s*프로필)/u,
|
||||
},
|
||||
],
|
||||
},
|
||||
"help-where": {
|
||||
confidence: 0.9,
|
||||
keywords: [
|
||||
@@ -75,39 +117,6 @@ export const ONBOARDING_INTENT_TABLE = {
|
||||
/(가입|등록|절차|도움|도와|뭐해야|뭐 해야|해야 돼|해야해|무엇을 해야)/u,
|
||||
],
|
||||
},
|
||||
"discord-verify-question": {
|
||||
confidence: 0.96,
|
||||
keywords: ["디스코드", "확인", "닉네임"],
|
||||
llm: true,
|
||||
match: ({ lower, trimmed }) => {
|
||||
const mentionsDiscord = /(디스코드|디코|discord)/u.test(lower);
|
||||
const asksDiscordVerification =
|
||||
/(확인|조회|찾아|검색|검사|바뀌|변경|닉변|맞(?:아|나요)?|됐(?:나|나요)?|완료됐(?:나|나요)?)/u.test(
|
||||
trimmed,
|
||||
);
|
||||
const reportsDiscordNicknameCompletion =
|
||||
/(닉네임|닉변|프로필\s*닉네임|서버\s*닉네임)/u.test(trimmed) &&
|
||||
/(완료했(?:어|어요|습니다)?|끝냈(?:어|어요|습니다)?|설정했(?:어|어요|습니다)?|바꿨(?:어|어요|습니다)?|변경했(?:어|어요|습니다)?|다\s*했(?:어|어요|습니다)?|해놨(?:어|어요|습니다)?|해뒀(?:어|어요|습니다)?|적용됐(?:어|어요|습니다)?)/u.test(
|
||||
trimmed,
|
||||
);
|
||||
return (
|
||||
mentionsDiscord &&
|
||||
(asksDiscordVerification || reportsDiscordNicknameCompletion)
|
||||
);
|
||||
},
|
||||
},
|
||||
"discord-question": {
|
||||
confidence: 0.95,
|
||||
keywords: ["디스코드", "주민신청", "프로필", "권한", "설정"],
|
||||
llm: true,
|
||||
match: ({ lower }) => /(디스코드|디코|discord)/u.test(lower),
|
||||
regex: [
|
||||
{
|
||||
confidence: 0.94,
|
||||
pattern: /(주민신청\s*채널|주민신청채널|서버\s*프로필)/u,
|
||||
},
|
||||
],
|
||||
},
|
||||
"age-question": {
|
||||
confidence: 0.93,
|
||||
keywords: ["27", "가입", "가능"],
|
||||
|
||||
@@ -145,12 +145,12 @@ export const ONBOARDING_PROMPT_DATA = {
|
||||
{
|
||||
user: "디스코드 프로필 닉네임 설정 완료했어요",
|
||||
assistant:
|
||||
"디스코드 닉네임을 확인해봤는데 아직 올바른 형식으로 확인되지는 않았어요. 디스코드 서버 닉네임은 이름#태그/출생 연도/티어 형식이에요. 예: 할리#프렌치불독/88/D",
|
||||
"디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '날으는꼴뚜기/D' 으로 확인됩니다. 할리#프렌치불독/88/D 와 같이 닉네임#태그/출생연도/티어로 수정해 주세요.",
|
||||
},
|
||||
{
|
||||
user: "디스코드 닉네임 바꿨어요",
|
||||
assistant:
|
||||
"디스코드 닉네임이 아직 이름#태그/출생 연도/티어 형식으로 확인되지 않아요. 예: 할리#프렌치불독/88/D처럼 태그를 포함해서 설정해 주세요.",
|
||||
"디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '날으는꼴뚜기/D' 으로 확인됩니다. 할리#프렌치불독/88/D 와 같이 닉네임#태그/출생연도/티어로 수정해 주세요.",
|
||||
},
|
||||
],
|
||||
greeting: [
|
||||
@@ -236,10 +236,10 @@ export const ONBOARDING_PROMPT_DATA = {
|
||||
].join("\n"),
|
||||
"discord-question": [
|
||||
"- 디스코드 일반 안내나 규칙 질문이면 lookup-community-rule 도구를 사용해 정확한 규칙만 답해.",
|
||||
"- 사용자가 디스코드 닉네임이나 프로필 설정을 끝냈다고 말하거나 반영 여부를 묻는다면 verify-discord-nickname 도구를 먼저 사용해 직접 확인해.",
|
||||
"- 사용자가 디스코드 닉네임, 프로필 설정, 주민신청 메시지를 끝냈다고 말하거나 반영 여부를 묻는다면 verify-discord-nickname 도구를 먼저 사용해 직접 확인해.",
|
||||
].join("\n"),
|
||||
"discord-verify-question":
|
||||
"- 디스코드 닉네임 확인 요청은 verify-discord-nickname 도구를 먼저 사용하고, 조회 결과만 짧게 답해.",
|
||||
"- 디스코드 닉네임/주민신청 확인 요청은 verify-discord-nickname 도구와 check-onboarding-progress 도구를 먼저 사용하고, 조회 결과만 짧게 답해. 확인 없이 초대 안내만 반복하지 마. 닉네임이 틀렸거나 미설정이면 recommendedNickname/correctionGuide를 우선 사용해 정확히 어떻게 고칠지 안내해.",
|
||||
"help-where":
|
||||
"- 위치 안내가 모호할 때만 get-room-context 도구를 고려하고, 기본적으로는 절차 위치만 답해.",
|
||||
"status-question":
|
||||
|
||||
@@ -4,7 +4,6 @@ import { z } from "zod";
|
||||
import {
|
||||
getCriticMaxTokens,
|
||||
getLlmTemperature,
|
||||
getTier2CriticModel,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
@@ -147,7 +146,6 @@ export class OnboardingCritic {
|
||||
{
|
||||
deadlineMs: options?.deadlineMs,
|
||||
maxTokens: getCriticMaxTokens(),
|
||||
modelOverride: getTier2CriticModel(),
|
||||
role: "critic",
|
||||
temperature: Math.max(0, getLlmTemperature() - 0.15),
|
||||
tier: 2,
|
||||
@@ -205,7 +203,6 @@ export class OnboardingCritic {
|
||||
deadlineMs: options.deadlineMs,
|
||||
jsonMode: true,
|
||||
maxTokens: getCriticMaxTokens(),
|
||||
modelOverride: getTier2CriticModel(),
|
||||
role: "critic",
|
||||
temperature: Math.max(0, getLlmTemperature() - 0.15),
|
||||
tier: 2,
|
||||
|
||||
@@ -207,6 +207,317 @@ describe("OnboardingOrchestrator", () => {
|
||||
expect(calls[1]?.options?.tools).toBeUndefined();
|
||||
});
|
||||
|
||||
it("forces discord verification tools before answering completion reports", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const executedTools: string[] = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "answer-direct",
|
||||
targetPhase: "deadline-near",
|
||||
mustInclude: [],
|
||||
mustAvoid: [],
|
||||
suggestedTools: [],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "잘못된 직접 답변 계획",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "디스코드 닉네임은 확인됐고, 주민신청 메시지도 확인됐어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "확인됐어요." } }] },
|
||||
message: { role: "assistant" as const, content: "확인됐어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
const verifyTool: Tool<Record<string, never>, { status: string }> = {
|
||||
name: "verify-discord-nickname",
|
||||
description: "디스코드 닉네임 확인",
|
||||
argsSchema: z.object({}),
|
||||
resultSchema: z.object({
|
||||
status: z.string(),
|
||||
}),
|
||||
execute: () => {
|
||||
executedTools.push("verify-discord-nickname");
|
||||
return Promise.resolve({ status: "matched-renamed" });
|
||||
},
|
||||
};
|
||||
const progressTool: Tool<
|
||||
{ chatroomId: string; userId: string },
|
||||
{ remainingSteps: string[] }
|
||||
> = {
|
||||
name: "check-onboarding-progress",
|
||||
description: "진행 상태 확인",
|
||||
argsSchema: z.object({
|
||||
chatroomId: z.string(),
|
||||
userId: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
remainingSteps: z.array(z.string()),
|
||||
}),
|
||||
execute: (args) => {
|
||||
executedTools.push(`${args.chatroomId}:${args.userId}`);
|
||||
return Promise.resolve({ remainingSteps: [] });
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({
|
||||
allowShortResponse: false,
|
||||
intent: { kind: "discord-verify-question" },
|
||||
phase: { kind: "deadline-near", remaining: ["discord-profile-message"] },
|
||||
responseMode: "llm-with-tools",
|
||||
tier: 1,
|
||||
}),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{
|
||||
role: "user",
|
||||
content: "저 디코는 어떻게 들어가요? 주민신청까지 해뒀습니다",
|
||||
},
|
||||
],
|
||||
{
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
},
|
||||
toolContext: {},
|
||||
tools: [verifyTool, progressTool],
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toContain("주민신청");
|
||||
expect(executedTools).toEqual([
|
||||
"verify-discord-nickname",
|
||||
"room-1:user-1",
|
||||
]);
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[TOOL_PLAN_RESULTS]");
|
||||
expect(calls[1]?.messages[0]?.content).toContain("주민신청 메시지 확인 상태");
|
||||
expect(calls[1]?.messages[0]?.content).toContain(
|
||||
"디스코드 닉네임 상태를 독립 항목처럼 길게 설명",
|
||||
);
|
||||
expect(calls[1]?.options?.tools).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not expose verification tool again after partial discord tool-plan success", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
let verifyCallCount = 0;
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "validate-then-explain",
|
||||
targetPhase: "deadline-near",
|
||||
mustInclude: [],
|
||||
mustAvoid: [],
|
||||
suggestedTools: ["verify-discord-nickname"],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "닉네임 확인",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "디스코드 닉네임은 확인됐어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "확인됐어요." } }] },
|
||||
message: { role: "assistant" as const, content: "확인됐어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
const verifyTool: Tool<Record<string, never>, { status: string }> = {
|
||||
name: "verify-discord-nickname",
|
||||
description: "디스코드 닉네임 확인",
|
||||
argsSchema: z.object({}),
|
||||
resultSchema: z.object({
|
||||
status: z.string(),
|
||||
}),
|
||||
execute: () => {
|
||||
verifyCallCount += 1;
|
||||
return Promise.resolve({ status: "matched-renamed" });
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({
|
||||
allowShortResponse: false,
|
||||
intent: { kind: "discord-verify-question" },
|
||||
phase: { kind: "deadline-near", remaining: ["discord-profile-nickname"] },
|
||||
responseMode: "llm-with-tools",
|
||||
tier: 1,
|
||||
}),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{
|
||||
role: "user",
|
||||
content: "디스코드 닉네임 변경했어요",
|
||||
},
|
||||
],
|
||||
{
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
},
|
||||
toolContext: {},
|
||||
tools: [verifyTool],
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toContain("닉네임");
|
||||
expect(verifyCallCount).toBe(1);
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[TOOL_PLAN_RESULTS]");
|
||||
expect(calls[1]?.messages[0]?.content).toContain("matched-renamed");
|
||||
expect(calls[1]?.messages[0]?.content).toContain(
|
||||
"Tool not registered for step: check-onboarding-progress",
|
||||
);
|
||||
expect(calls[1]?.options?.tools).toBeUndefined();
|
||||
});
|
||||
|
||||
it("focuses discord nickname answers when only nickname verification was asked", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "validate-then-explain",
|
||||
targetPhase: "deadline-near",
|
||||
mustInclude: [],
|
||||
mustAvoid: [],
|
||||
suggestedTools: [],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "닉네임 확인",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "디스코드 닉네임은 확인됐어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "확인됐어요." } }] },
|
||||
message: { role: "assistant" as const, content: "확인됐어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
const verifyTool: Tool<Record<string, never>, { status: string }> = {
|
||||
name: "verify-discord-nickname",
|
||||
description: "디스코드 닉네임 확인",
|
||||
argsSchema: z.object({}),
|
||||
resultSchema: z.object({
|
||||
status: z.string(),
|
||||
}),
|
||||
execute: () => Promise.resolve({ status: "matched-renamed" }),
|
||||
};
|
||||
const progressTool: Tool<
|
||||
{ chatroomId: string; userId: string },
|
||||
{ remainingSteps: string[] }
|
||||
> = {
|
||||
name: "check-onboarding-progress",
|
||||
description: "진행 상태 확인",
|
||||
argsSchema: z.object({
|
||||
chatroomId: z.string(),
|
||||
userId: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
remainingSteps: z.array(z.string()),
|
||||
}),
|
||||
execute: () => Promise.resolve({ remainingSteps: [] }),
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({
|
||||
allowShortResponse: false,
|
||||
intent: { kind: "discord-verify-question" },
|
||||
phase: { kind: "deadline-near", remaining: ["discord-profile-nickname"] },
|
||||
responseMode: "llm-with-tools",
|
||||
tier: 1,
|
||||
}),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{
|
||||
role: "user",
|
||||
content: "디스코드 닉네임 변경했어요",
|
||||
},
|
||||
],
|
||||
{
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
},
|
||||
toolContext: {},
|
||||
tools: [verifyTool, progressTool],
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toContain("닉네임");
|
||||
expect(calls[1]?.messages[0]?.content).toContain("디스코드 닉네임 조회 결과");
|
||||
expect(calls[1]?.messages[0]?.content).toContain(
|
||||
"주민신청 메시지 상태 언급",
|
||||
);
|
||||
});
|
||||
|
||||
it("injects requested SKILL.md resources before responder when enabled", async () => {
|
||||
process.env["AGENT_SKILL_MD_MODE"] = "true";
|
||||
const calls: Array<{
|
||||
|
||||
@@ -10,11 +10,8 @@ import {
|
||||
getPlannerMaxTokens,
|
||||
getTier0EstimatedCallMs,
|
||||
getTier0MaxTokens,
|
||||
getTier0Model,
|
||||
getTier1EstimatedCallMs,
|
||||
getTier1MaxTokens,
|
||||
getTier1PlannerModel,
|
||||
getTier1ResponderModel,
|
||||
getTier2EstimatedCallMs,
|
||||
getTier2MaxTokens,
|
||||
} from "../../../config.js";
|
||||
@@ -75,6 +72,48 @@ function summarizeMessages(messages: ChatMessage[]): string {
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
type DiscordVerificationFocus =
|
||||
| "both"
|
||||
| "nickname"
|
||||
| "resident-application-message";
|
||||
|
||||
function getLatestUserMessage(messages: ChatMessage[]): string {
|
||||
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
||||
const message = messages[index];
|
||||
if (message?.role === "user") {
|
||||
return message.content;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function getDiscordVerificationFocus(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): DiscordVerificationFocus {
|
||||
if (derived.intent.kind !== "discord-verify-question") {
|
||||
return "both";
|
||||
}
|
||||
|
||||
const latestUserMessage = getLatestUserMessage(messages).toLowerCase();
|
||||
const asksResidentApplication =
|
||||
/(주민\s*신청|주민신청|주민\s*등록|주민등록|프로필\s*메시지|채널\s*메시지|메시지\s*(작성|남김|보냄|썼|완료|했습니다|했어요))/u.test(
|
||||
latestUserMessage,
|
||||
);
|
||||
const asksNickname =
|
||||
/(닉네임|닉변|서버\s*프로필|서버프로필|프로필\s*닉네임|서버\s*표시명|서버표시명)/u.test(
|
||||
latestUserMessage,
|
||||
);
|
||||
|
||||
if (asksResidentApplication && !asksNickname) {
|
||||
return "resident-application-message";
|
||||
}
|
||||
if (asksNickname && !asksResidentApplication) {
|
||||
return "nickname";
|
||||
}
|
||||
return "both";
|
||||
}
|
||||
|
||||
function tryParsePlannerOutput(raw: string): PlannerOutput | null {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
@@ -95,7 +134,24 @@ function tryParsePlannerOutput(raw: string): PlannerOutput | null {
|
||||
}
|
||||
}
|
||||
|
||||
function getDefaultPlan(derived: DerivedState): PlannerOutput {
|
||||
function getDefaultPlan(
|
||||
derived: DerivedState,
|
||||
focus: DiscordVerificationFocus = "both",
|
||||
): PlannerOutput {
|
||||
if (derived.intent.kind === "discord-verify-question") {
|
||||
return enforceRequiredToolPlan(derived, {
|
||||
approach: "validate-then-explain",
|
||||
targetPhase: derived.phase.kind,
|
||||
mustInclude: getDiscordVerificationMustInclude(focus),
|
||||
mustAvoid: getDiscordVerificationMustAvoid(focus),
|
||||
requestResources: [],
|
||||
suggestedTools: [],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 150,
|
||||
rationale: "discord-verification-fallback-plan",
|
||||
}, focus);
|
||||
}
|
||||
|
||||
return {
|
||||
approach:
|
||||
derived.responseMode === "llm-with-tools"
|
||||
@@ -115,6 +171,104 @@ function getDefaultPlan(derived: DerivedState): PlannerOutput {
|
||||
};
|
||||
}
|
||||
|
||||
function hasToolPlanStep(plan: PlannerOutput, toolName: string): boolean {
|
||||
return plan.toolPlan?.steps.some((step) => step.tool === toolName) ?? false;
|
||||
}
|
||||
|
||||
function getDiscordVerificationMustInclude(
|
||||
focus: DiscordVerificationFocus,
|
||||
): string[] {
|
||||
if (focus === "nickname") {
|
||||
return ["디스코드 닉네임 조회 결과"];
|
||||
}
|
||||
if (focus === "resident-application-message") {
|
||||
return [
|
||||
"주민신청 메시지 확인 상태",
|
||||
"메시지 형식이 틀리면 서버프로필 닉네임과 동일하게 다시 작성하라는 안내",
|
||||
];
|
||||
}
|
||||
return [
|
||||
"디스코드 닉네임 조회 결과",
|
||||
"주민신청 메시지 확인 상태",
|
||||
];
|
||||
}
|
||||
|
||||
function getDiscordVerificationMustAvoid(
|
||||
focus: DiscordVerificationFocus,
|
||||
): string[] {
|
||||
const base = ["확인 없이 완료 단정", "초대 안내만 반복"];
|
||||
if (focus === "nickname") {
|
||||
return [...base, "주민신청 메시지 상태 언급"];
|
||||
}
|
||||
if (focus === "resident-application-message") {
|
||||
return [
|
||||
...base,
|
||||
"디스코드 닉네임 상태를 독립 항목처럼 길게 설명",
|
||||
"묻지 않은 남은 절차 나열",
|
||||
];
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
function enforceRequiredToolPlan(
|
||||
derived: DerivedState,
|
||||
plan: PlannerOutput,
|
||||
focus: DiscordVerificationFocus = "both",
|
||||
): PlannerOutput {
|
||||
if (derived.intent.kind !== "discord-verify-question") {
|
||||
return plan;
|
||||
}
|
||||
|
||||
const suggestedTools = [
|
||||
...new Set([
|
||||
...plan.suggestedTools,
|
||||
"verify-discord-nickname" as const,
|
||||
"check-onboarding-progress" as const,
|
||||
]),
|
||||
];
|
||||
const steps = [...(plan.toolPlan?.steps ?? [])];
|
||||
if (!hasToolPlanStep(plan, "verify-discord-nickname")) {
|
||||
steps.unshift({
|
||||
id: "verify-discord",
|
||||
tool: "verify-discord-nickname",
|
||||
on: { error: "fallback-llm" },
|
||||
});
|
||||
}
|
||||
if (!hasToolPlanStep(plan, "check-onboarding-progress")) {
|
||||
steps.push({
|
||||
id: "check-progress",
|
||||
tool: "check-onboarding-progress",
|
||||
argsFromContext: {
|
||||
chatroomId: "$request.chatroomId",
|
||||
userId: "$request.userId",
|
||||
},
|
||||
on: { error: "fallback-llm" },
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...plan,
|
||||
approach: "validate-then-explain",
|
||||
mustAvoid: [
|
||||
...new Set([
|
||||
...plan.mustAvoid,
|
||||
...getDiscordVerificationMustAvoid(focus),
|
||||
]),
|
||||
].slice(0, 6),
|
||||
mustInclude: [
|
||||
...new Set([
|
||||
...plan.mustInclude,
|
||||
...getDiscordVerificationMustInclude(focus),
|
||||
]),
|
||||
].slice(0, 6),
|
||||
suggestedTools: suggestedTools.slice(0, 3),
|
||||
toolPlan: {
|
||||
composeWith: "all-then-llm",
|
||||
steps: steps.slice(0, 4),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function getRemainingDeadlineMs(): number | undefined {
|
||||
const deadlineAt = getContext()?.deadlineAt;
|
||||
if (!deadlineAt) {
|
||||
@@ -224,6 +378,49 @@ function buildToolPlanAddendum(result: ToolPlanResult): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function hasToolPlanResultForTool(
|
||||
plan: PlannerOutput,
|
||||
result: ToolPlanResult,
|
||||
toolName: string,
|
||||
): boolean {
|
||||
return (
|
||||
plan.toolPlan?.steps.some(
|
||||
(step) =>
|
||||
step.tool === toolName &&
|
||||
Object.hasOwn(result.results, step.id),
|
||||
) ?? false
|
||||
);
|
||||
}
|
||||
|
||||
function shouldInjectToolPlanResult(
|
||||
derived: DerivedState,
|
||||
result: ToolPlanResult,
|
||||
): boolean {
|
||||
if (!result.fallbackToLlm) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return (
|
||||
derived.intent.kind === "discord-verify-question" &&
|
||||
Object.keys(result.results).length > 0
|
||||
);
|
||||
}
|
||||
|
||||
function shouldSuppressResponderTools(
|
||||
derived: DerivedState,
|
||||
plan: PlannerOutput,
|
||||
result: ToolPlanResult,
|
||||
): boolean {
|
||||
if (!result.fallbackToLlm) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return (
|
||||
derived.intent.kind === "discord-verify-question" &&
|
||||
hasToolPlanResultForTool(plan, result, "verify-discord-nickname")
|
||||
);
|
||||
}
|
||||
|
||||
function pickToolsForPlan(
|
||||
plan: PlannerOutput,
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
@@ -254,7 +451,6 @@ export class OnboardingOrchestrator {
|
||||
options: OrchestratorRunOptions,
|
||||
responderTools: readonly Tool<unknown, unknown>[],
|
||||
maxTokens: number,
|
||||
modelOverride: string,
|
||||
): Promise<string | null> {
|
||||
const logger = getRequestLogger();
|
||||
const deadlineMs = getRemainingDeadlineMs();
|
||||
@@ -275,7 +471,6 @@ export class OnboardingOrchestrator {
|
||||
llmClient: this.llmClient,
|
||||
maxTokens,
|
||||
messages,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
routingDecision: derived.routingDecision,
|
||||
tier: derived.tier,
|
||||
@@ -288,7 +483,6 @@ export class OnboardingOrchestrator {
|
||||
const response = await this.llmClient.chatCompletion(messages, {
|
||||
deadlineMs,
|
||||
maxTokens,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
routingDecision: derived.routingDecision,
|
||||
temperature: options.temperature ?? getLlmTemperature(),
|
||||
@@ -304,24 +498,29 @@ export class OnboardingOrchestrator {
|
||||
): Promise<PlannerOutput> {
|
||||
const logger = getRequestLogger();
|
||||
const traceStep = getActiveProcessTrace()?.startStep("plan");
|
||||
const verificationFocus = getDiscordVerificationFocus(derived, messages);
|
||||
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
|
||||
const normalizedPlan = enforceRequiredToolPlan(
|
||||
derived,
|
||||
plan,
|
||||
verificationFocus,
|
||||
);
|
||||
traceStep?.finish({
|
||||
approach: plan.approach,
|
||||
mustAvoid: plan.mustAvoid,
|
||||
mustInclude: plan.mustInclude,
|
||||
requestResources: plan.requestResources,
|
||||
suggestedTools: plan.suggestedTools,
|
||||
targetLengthChars: plan.targetLengthChars,
|
||||
toolPlanStepCount: plan.toolPlan?.steps.length,
|
||||
approach: normalizedPlan.approach,
|
||||
mustAvoid: normalizedPlan.mustAvoid,
|
||||
mustInclude: normalizedPlan.mustInclude,
|
||||
requestResources: normalizedPlan.requestResources,
|
||||
suggestedTools: normalizedPlan.suggestedTools,
|
||||
targetLengthChars: normalizedPlan.targetLengthChars,
|
||||
toolPlanStepCount: normalizedPlan.toolPlan?.steps.length,
|
||||
});
|
||||
return plan;
|
||||
return normalizedPlan;
|
||||
};
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
jsonMode: true,
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
routingDecision: derived.routingDecision,
|
||||
temperature: 0,
|
||||
@@ -356,7 +555,6 @@ export class OnboardingOrchestrator {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
jsonMode: true,
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
routingDecision: derived.routingDecision,
|
||||
temperature: 0,
|
||||
@@ -376,7 +574,7 @@ export class OnboardingOrchestrator {
|
||||
return finishTrace(retryPlan);
|
||||
}
|
||||
|
||||
const fallbackPlan = getDefaultPlan(derived);
|
||||
const fallbackPlan = getDefaultPlan(derived, verificationFocus);
|
||||
logger.debug(
|
||||
{
|
||||
intent: derived.intent.kind,
|
||||
@@ -403,7 +601,6 @@ export class OnboardingOrchestrator {
|
||||
options,
|
||||
tools,
|
||||
maxTokens,
|
||||
getTier0Model(),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -428,16 +625,19 @@ export class OnboardingOrchestrator {
|
||||
);
|
||||
}
|
||||
const toolPlanResult = await this.runProgrammaticToolPlan(
|
||||
derived,
|
||||
plan,
|
||||
finalResponderMessages,
|
||||
options,
|
||||
);
|
||||
|
||||
if (toolPlanResult && !toolPlanResult.fallbackToLlm) {
|
||||
if (toolPlanResult && shouldInjectToolPlanResult(derived, toolPlanResult)) {
|
||||
finalResponderMessages = injectSystemAddendum(
|
||||
finalResponderMessages,
|
||||
buildToolPlanAddendum(toolPlanResult),
|
||||
);
|
||||
}
|
||||
if (toolPlanResult && shouldSuppressResponderTools(derived, plan, toolPlanResult)) {
|
||||
responderTools = [];
|
||||
}
|
||||
|
||||
@@ -447,7 +647,6 @@ export class OnboardingOrchestrator {
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier1MaxTokens(),
|
||||
getTier1ResponderModel(),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -472,16 +671,19 @@ export class OnboardingOrchestrator {
|
||||
);
|
||||
}
|
||||
const toolPlanResult = await this.runProgrammaticToolPlan(
|
||||
derived,
|
||||
plan,
|
||||
finalResponderMessages,
|
||||
options,
|
||||
);
|
||||
|
||||
if (toolPlanResult && !toolPlanResult.fallbackToLlm) {
|
||||
if (toolPlanResult && shouldInjectToolPlanResult(derived, toolPlanResult)) {
|
||||
finalResponderMessages = injectSystemAddendum(
|
||||
finalResponderMessages,
|
||||
buildToolPlanAddendum(toolPlanResult),
|
||||
);
|
||||
}
|
||||
if (toolPlanResult && shouldSuppressResponderTools(derived, plan, toolPlanResult)) {
|
||||
responderTools = [];
|
||||
}
|
||||
|
||||
@@ -491,7 +693,6 @@ export class OnboardingOrchestrator {
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier2MaxTokens(),
|
||||
getTier1ResponderModel(),
|
||||
);
|
||||
|
||||
if (!draft) {
|
||||
@@ -517,12 +718,19 @@ export class OnboardingOrchestrator {
|
||||
}
|
||||
|
||||
private async runProgrammaticToolPlan(
|
||||
derived: DerivedState,
|
||||
plan: PlannerOutput,
|
||||
_messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<ToolPlanResult | null> {
|
||||
if (
|
||||
!getAgentProgrammaticToolCalling() ||
|
||||
!getAgentProgrammaticToolCalling() &&
|
||||
derived.intent.kind !== "discord-verify-question"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
!plan.toolPlan ||
|
||||
!options.request ||
|
||||
!options.tools?.length
|
||||
|
||||
@@ -26,7 +26,11 @@ export function selectToolsForDerived(
|
||||
|
||||
if (derived.intent.kind === 'discord-verify-question') {
|
||||
return maybeUseToolSearch(
|
||||
tools.filter((tool) => tool.name === 'verify-discord-nickname'),
|
||||
tools.filter(
|
||||
(tool) =>
|
||||
tool.name === 'verify-discord-nickname' ||
|
||||
tool.name === 'check-onboarding-progress',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,15 @@ describe("OnboardingIntentClassifier", () => {
|
||||
expect(result.source).toBe("regex");
|
||||
});
|
||||
|
||||
it("classifies resident application completion reports as discord verification", async () => {
|
||||
const result = await classifier.classify(
|
||||
"저 디코는 어떻게 들어가요? 주민신청까지 해뒀습니다",
|
||||
"user-message",
|
||||
);
|
||||
expect(result.intent.kind).toBe("discord-verify-question");
|
||||
expect(result.source).toBe("regex");
|
||||
});
|
||||
|
||||
it("classifies identity room questions as help-how", async () => {
|
||||
const result = await classifier.classify(
|
||||
"본인인증 방은 뭐에요?",
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { MemoryService } from "../../memory/memory-types.js";
|
||||
import { lookupRuleTool } from "./lookup-rule.js";
|
||||
import {
|
||||
lookupRuleTool,
|
||||
resolveCommunityRulesPath,
|
||||
} from "./lookup-rule.js";
|
||||
|
||||
const failingMemoryService = {
|
||||
longTerm: {
|
||||
@@ -9,6 +16,26 @@ const failingMemoryService = {
|
||||
} as unknown as MemoryService;
|
||||
|
||||
describe("lookupRuleTool", () => {
|
||||
it("falls back to the source rules resource when imported from dist", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "community-rules-resource-"));
|
||||
const sourceResourceDir = join(
|
||||
root,
|
||||
"src",
|
||||
"skills",
|
||||
"moderation",
|
||||
"resources",
|
||||
);
|
||||
mkdirSync(sourceResourceDir, { recursive: true });
|
||||
const sourceRulesPath = join(sourceResourceDir, "community-rules.md");
|
||||
writeFileSync(sourceRulesPath, "## ads\n광고 금지\n");
|
||||
|
||||
const moduleUrl = pathToFileURL(
|
||||
join(root, "dist", "tools", "moderation", "lookup-rule.js"),
|
||||
).href;
|
||||
|
||||
expect(resolveCommunityRulesPath(moduleUrl, root)).toBe(sourceRulesPath);
|
||||
});
|
||||
|
||||
it("returns the matching community rule section", async () => {
|
||||
const result = await lookupRuleTool.execute({ topic: "ads" }, {});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
@@ -40,10 +41,31 @@ type RuleTopic =
|
||||
| "sanctions"
|
||||
| "spam";
|
||||
|
||||
const RULES_PATH = fileURLToPath(
|
||||
new URL("../../skills/moderation/resources/community-rules.md", import.meta.url),
|
||||
);
|
||||
const RULES_TEXT = readFileSync(RULES_PATH, "utf8");
|
||||
const RULES_RELATIVE_PATH = "skills/moderation/resources/community-rules.md";
|
||||
|
||||
export function resolveCommunityRulesPath(
|
||||
moduleUrl = import.meta.url,
|
||||
cwd = process.cwd(),
|
||||
): string {
|
||||
const moduleDir = dirname(fileURLToPath(moduleUrl));
|
||||
const candidates = [
|
||||
resolve(moduleDir, "../../", RULES_RELATIVE_PATH),
|
||||
resolve(moduleDir, "../../../src", RULES_RELATIVE_PATH),
|
||||
resolve(cwd, "dist", RULES_RELATIVE_PATH),
|
||||
resolve(cwd, "src", RULES_RELATIVE_PATH),
|
||||
];
|
||||
const found = candidates.find((candidate) => existsSync(candidate));
|
||||
|
||||
if (!found) {
|
||||
throw new Error(
|
||||
`Missing moderation community rules resource. Tried: ${candidates.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
const RULES_TEXT = readFileSync(resolveCommunityRulesPath(), "utf8");
|
||||
|
||||
function extractRuleSection(topic: RuleTopic): string {
|
||||
const pattern = new RegExp(`^## ${topic}\\n([\\s\\S]*?)(?=\\n## |$)`, "mu");
|
||||
|
||||
@@ -44,6 +44,8 @@ describe("checkOnboardingProgressTool", () => {
|
||||
|
||||
expect(result.step1Done).toBe(true);
|
||||
expect(result.overallDone).toBe(false);
|
||||
expect(result.discordProfileNicknameDone).toBe(false);
|
||||
expect(result.discordProfileMessageDone).toBe(false);
|
||||
expect(result.remainingSteps).toEqual([
|
||||
"discord-profile-nickname",
|
||||
"discord-profile-message",
|
||||
@@ -104,6 +106,8 @@ describe("checkOnboardingProgressTool", () => {
|
||||
|
||||
expect(result.step1Done).toBe(true);
|
||||
expect(result.overallDone).toBe(true);
|
||||
expect(result.discordProfileNicknameDone).toBe(null);
|
||||
expect(result.discordProfileMessageDone).toBe(null);
|
||||
expect(result.remainingSteps).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -18,6 +18,8 @@ const argsSchema = z.object({
|
||||
});
|
||||
|
||||
const resultSchema = z.object({
|
||||
discordProfileMessageDone: z.boolean().nullable(),
|
||||
discordProfileNicknameDone: z.boolean().nullable(),
|
||||
nicknameDone: z.boolean(),
|
||||
photoDone: z.boolean(),
|
||||
step1Done: z.boolean(),
|
||||
@@ -62,6 +64,8 @@ export const checkOnboardingProgressTool: Tool<
|
||||
: null;
|
||||
|
||||
return {
|
||||
discordProfileMessageDone: session?.discordProfileMessageVerified ?? null,
|
||||
discordProfileNicknameDone: session?.discordProfileNicknameVerified ?? null,
|
||||
nicknameDone: session?.nicknameVerified ?? false,
|
||||
photoDone: session?.photoVerified ?? false,
|
||||
step1Done: isStep1Completed(session),
|
||||
|
||||
@@ -54,6 +54,16 @@ describe("verifyDiscordNicknameTool", () => {
|
||||
username: "priri",
|
||||
},
|
||||
],
|
||||
residentApplicationMessageMatches: [
|
||||
{
|
||||
authorId: "discord-1",
|
||||
channelId: "resident-channel-1",
|
||||
content: "프리리#KR1/95/E",
|
||||
createdAt: "2026-04-17T00:12:00.000Z",
|
||||
matchedBy: "author-id",
|
||||
messageId: "message-1",
|
||||
},
|
||||
],
|
||||
joinedAfterMatches: [],
|
||||
};
|
||||
const request = vi.fn(
|
||||
@@ -81,12 +91,88 @@ describe("verifyDiscordNicknameTool", () => {
|
||||
|
||||
expect(result.status).toBe("matched-renamed");
|
||||
expect(result.message).toContain("올바른 형식으로 설정되어 있어요");
|
||||
expect(result.message).toContain("주민신청 메시지도 확인됐어요");
|
||||
expect(result.discordMemberId).toBe("discord-1");
|
||||
expect(result.residentApplicationMessageStatus).toBe("confirmed");
|
||||
expect(result.residentApplicationMessageId).toBe("message-1");
|
||||
expect(result.residentApplicationMessageContent).toBe("프리리#KR1/95/E");
|
||||
expect(result.residentApplicationMessageExpectedContent).toBe("프리리#KR1/95/E");
|
||||
expect(session.discordMemberId).toBe("discord-1");
|
||||
expect(session.discordProfileNicknameVerified).toBe(true);
|
||||
expect(session.discordProfileMessageVerified).toBe(true);
|
||||
expect(session.discordProfileMessageVerifiedAt).toBe(
|
||||
"2026-04-17T00:12:00.000Z",
|
||||
);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
expect(request.mock.calls[0]?.[1]).toMatchObject({
|
||||
includeResidentApplicationMessages: true,
|
||||
searchName: "프리리",
|
||||
});
|
||||
expect(setJson).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not confirm resident application messages that differ from the server nickname", async () => {
|
||||
const setJson = vi.fn().mockResolvedValue(undefined);
|
||||
const response: DiscordMemberLookupResponse = {
|
||||
status: "ok",
|
||||
nicknameMatches: [
|
||||
{
|
||||
memberId: "discord-1",
|
||||
joinedAt: "2026-04-17T00:10:00.000Z",
|
||||
globalName: "프리리",
|
||||
guildNickname: "프리리#KR1/95/E",
|
||||
matchedBy: "guild-nickname",
|
||||
username: "priri",
|
||||
},
|
||||
],
|
||||
residentApplicationMessageMatches: [
|
||||
{
|
||||
authorId: "discord-1",
|
||||
channelId: "resident-channel-1",
|
||||
content: "프리리/E",
|
||||
createdAt: "2026-04-17T00:12:00.000Z",
|
||||
matchedBy: "author-id",
|
||||
messageId: "message-1",
|
||||
},
|
||||
],
|
||||
joinedAfterMatches: [],
|
||||
};
|
||||
const request = vi.fn(
|
||||
<TRequest = unknown, TResponse = unknown>(
|
||||
_subject: string,
|
||||
_payload: TRequest,
|
||||
_options?: { timeoutMs?: number },
|
||||
): Promise<TResponse> => Promise.resolve(response as TResponse),
|
||||
);
|
||||
const session = makeSession();
|
||||
|
||||
const result = await verifyDiscordNicknameTool.execute(
|
||||
{},
|
||||
{
|
||||
chatroomId: session.chatroomId,
|
||||
garnetClient: {
|
||||
setJson,
|
||||
} as never,
|
||||
natsClient: {
|
||||
request: request as never,
|
||||
},
|
||||
session,
|
||||
userId: session.userId,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe("matched-renamed");
|
||||
expect(result.residentApplicationMessageStatus).toBe("content-mismatch");
|
||||
expect(result.residentApplicationMessageContent).toBe("프리리/E");
|
||||
expect(result.residentApplicationMessageExpectedContent).toBe("프리리#KR1/95/E");
|
||||
expect(result.residentApplicationMessageCorrectionGuide).toContain(
|
||||
"주민신청 메시지는 서버프로필 닉네임과 동일하게 작성해야 해요",
|
||||
);
|
||||
expect(result.message).toContain("현재 메시지는 '프리리/E'");
|
||||
expect(session.discordProfileNicknameVerified).toBe(true);
|
||||
expect(session.discordProfileMessageVerified).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not mark renamed when only the raw kakao nickname matches literally", async () => {
|
||||
const response: DiscordMemberLookupResponse = {
|
||||
status: "ok",
|
||||
@@ -121,7 +207,51 @@ describe("verifyDiscordNicknameTool", () => {
|
||||
);
|
||||
|
||||
expect(result.status).toBe("matched-pending-rename");
|
||||
expect(result.message).toContain("이름#태그/출생연도/티어");
|
||||
expect(result.message).toContain("디스코드 닉네임이 형식과 맞지 않습니다.");
|
||||
expect(result.message).toContain("현재 소환사님의 서버프로필은 '프리리/95/여/E'");
|
||||
expect(result.message).toContain("닉네임#태그/출생연도/티어로 수정해 주세요.");
|
||||
expect(result.recommendedNickname).toBe("프리리#태그/95/E");
|
||||
expect(result.correctionGuide).toBe(result.message);
|
||||
expect(result.guidance).toContain("이름#태그/출생연도/티어");
|
||||
});
|
||||
|
||||
it("preserves an existing discord tag when suggesting a corrected nickname", async () => {
|
||||
const response: DiscordMemberLookupResponse = {
|
||||
status: "ok",
|
||||
nicknameMatches: [
|
||||
{
|
||||
memberId: "discord-1",
|
||||
globalName: "프리리",
|
||||
guildNickname: "프리리#KR1/95/여/E",
|
||||
joinedAt: "2026-04-17T00:10:00.000Z",
|
||||
matchedBy: "global-name",
|
||||
username: "priri",
|
||||
},
|
||||
],
|
||||
joinedAfterMatches: [],
|
||||
};
|
||||
const request = vi.fn(
|
||||
<TRequest = unknown, TResponse = unknown>(
|
||||
_subject: string,
|
||||
_payload: TRequest,
|
||||
_options?: { timeoutMs?: number },
|
||||
): Promise<TResponse> => Promise.resolve(response as TResponse),
|
||||
);
|
||||
|
||||
const result = await verifyDiscordNicknameTool.execute(
|
||||
{},
|
||||
{
|
||||
natsClient: {
|
||||
request: request as never,
|
||||
},
|
||||
session: makeSession(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe("matched-pending-rename");
|
||||
expect(result.recommendedNickname).toBe("프리리#KR1/95/E");
|
||||
expect(result.correctionGuide).toContain("현재 소환사님의 서버프로필은 '프리리#KR1/95/여/E'");
|
||||
expect(result.correctionGuide).toContain("할리#프렌치불독/88/D 와 같이");
|
||||
});
|
||||
|
||||
it("falls back to joined-after candidates and returns not-confirmed when ambiguous", async () => {
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
DiscordMemberCandidate,
|
||||
DiscordMemberLookupRequest,
|
||||
DiscordMemberLookupResponse,
|
||||
DiscordResidentApplicationMessageCandidate,
|
||||
} from '@common/nats-subjects';
|
||||
import { getContext, getRequestLogger } from '@obs/context';
|
||||
import {
|
||||
@@ -20,6 +21,7 @@ import {
|
||||
getKakaotalkNickname,
|
||||
matchesDiscordNicknameRule,
|
||||
parseKakaoNickname,
|
||||
type ParsedKakaoNickname,
|
||||
} from '@skills/onboarding/session';
|
||||
import type { Tool, ToolContext } from '../tool.js';
|
||||
|
||||
@@ -50,6 +52,8 @@ const resultSchema = z.object({
|
||||
searchName: z.string().optional(),
|
||||
discordMemberId: z.string().optional(),
|
||||
currentDisplayName: z.string().optional(),
|
||||
recommendedNickname: z.string().optional(),
|
||||
correctionGuide: z.string().optional(),
|
||||
formatExample: z.string().optional(),
|
||||
guidance: z.string().optional(),
|
||||
issueMessages: z.array(z.string()).optional(),
|
||||
@@ -57,6 +61,19 @@ const resultSchema = z.object({
|
||||
matchSource: z.enum(['joined-after', 'nickname']).optional(),
|
||||
matchedBy: z.enum(['global-name', 'guild-nickname', 'joined-after', 'username']).optional(),
|
||||
nicknameMatchCount: z.number().int().nonnegative().optional(),
|
||||
residentApplicationMessageAuthorId: z.string().optional(),
|
||||
residentApplicationMessageContent: z.string().optional(),
|
||||
residentApplicationMessageCorrectionGuide: z.string().optional(),
|
||||
residentApplicationMessageCreatedAt: z.string().optional(),
|
||||
residentApplicationMessageExpectedContent: z.string().optional(),
|
||||
residentApplicationMessageId: z.string().optional(),
|
||||
residentApplicationMessageMatchCount: z.number().int().nonnegative().optional(),
|
||||
residentApplicationMessageMatchedBy: z
|
||||
.enum(['author-id', 'author-name', 'content', 'joined-after'])
|
||||
.optional(),
|
||||
residentApplicationMessageStatus: z
|
||||
.enum(['ambiguous', 'confirmed', 'content-mismatch', 'lookup-unavailable', 'not-confirmed'])
|
||||
.optional(),
|
||||
});
|
||||
|
||||
function pickDisplayName(candidate: DiscordMemberCandidate): string {
|
||||
@@ -65,34 +82,249 @@ function pickDisplayName(candidate: DiscordMemberCandidate): string {
|
||||
);
|
||||
}
|
||||
|
||||
function resolveTimeoutMs(): number | null {
|
||||
function extractExistingDiscordTag(
|
||||
currentGuildNickname: string | undefined,
|
||||
parsed: ParsedKakaoNickname,
|
||||
): string | null {
|
||||
const normalized = currentGuildNickname?.trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const prefix = `${parsed.baseName}#`;
|
||||
if (!normalized.startsWith(prefix)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const tag = normalized.slice(prefix.length).split('/')[0]?.trim();
|
||||
return tag || null;
|
||||
}
|
||||
|
||||
function buildRecommendedNickname(
|
||||
parsed: ParsedKakaoNickname,
|
||||
currentGuildNickname: string | undefined,
|
||||
): string {
|
||||
const tag = extractExistingDiscordTag(currentGuildNickname, parsed) ?? '태그';
|
||||
return `${parsed.baseName}#${tag}/${parsed.year}/${parsed.tier}`;
|
||||
}
|
||||
|
||||
function buildCorrectionGuide(args: {
|
||||
currentDisplayName: string;
|
||||
currentGuildNickname?: string | undefined;
|
||||
}): string {
|
||||
const formatGuide = `${DISCORD_NICKNAME_FORMAT_EXAMPLE} 와 같이 닉네임#태그/출생연도/티어로 수정해 주세요.`;
|
||||
|
||||
if (args.currentGuildNickname?.trim()) {
|
||||
return `디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '${args.currentGuildNickname.trim()}' 으로 확인됩니다. ${formatGuide}`;
|
||||
}
|
||||
|
||||
return `디스코드 닉네임이 형식과 맞지 않습니다. 현재 소환사님의 서버프로필은 '${args.currentDisplayName}' 으로 확인됩니다. ${formatGuide}`;
|
||||
}
|
||||
|
||||
type ResidentApplicationMessageStatus =
|
||||
| 'ambiguous'
|
||||
| 'confirmed'
|
||||
| 'content-mismatch'
|
||||
| 'lookup-unavailable'
|
||||
| 'not-confirmed';
|
||||
|
||||
interface ResidentApplicationMessageResult {
|
||||
candidate?: DiscordResidentApplicationMessageCandidate | undefined;
|
||||
correctionGuide?: string | undefined;
|
||||
expectedContent?: string | undefined;
|
||||
matchCount?: number | undefined;
|
||||
status: ResidentApplicationMessageStatus;
|
||||
}
|
||||
|
||||
function resolveResidentApplicationMessageResult(
|
||||
matches: DiscordResidentApplicationMessageCandidate[] | undefined,
|
||||
): ResidentApplicationMessageResult {
|
||||
if (!matches) {
|
||||
return { status: 'lookup-unavailable' };
|
||||
}
|
||||
|
||||
if (matches.length === 0) {
|
||||
return { matchCount: 0, status: 'not-confirmed' };
|
||||
}
|
||||
|
||||
if (matches.length > 1) {
|
||||
return { matchCount: matches.length, status: 'ambiguous' };
|
||||
}
|
||||
|
||||
return {
|
||||
candidate: matches[0],
|
||||
matchCount: 1,
|
||||
status: 'confirmed',
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeMessageContent(content: string | undefined): string {
|
||||
return content?.replace(/\s+/gu, ' ').trim() ?? '';
|
||||
}
|
||||
|
||||
function buildResidentApplicationMessageCorrectionGuide(args: {
|
||||
actualContent: string;
|
||||
expectedContent: string;
|
||||
}): string {
|
||||
return `주민신청 메시지는 서버프로필 닉네임과 동일하게 작성해야 해요. 현재 메시지는 '${args.actualContent}' 로 확인됩니다. '${args.expectedContent}' 로 다시 남겨주세요.`;
|
||||
}
|
||||
|
||||
function validateResidentApplicationMessageContent(
|
||||
result: ResidentApplicationMessageResult,
|
||||
expectedContent: string | undefined,
|
||||
): ResidentApplicationMessageResult {
|
||||
if (result.status !== 'confirmed' || !result.candidate || !expectedContent) {
|
||||
return expectedContent ? { ...result, expectedContent } : result;
|
||||
}
|
||||
|
||||
const actualContent = normalizeMessageContent(result.candidate.content);
|
||||
const normalizedExpected = normalizeMessageContent(expectedContent);
|
||||
if (actualContent === normalizedExpected) {
|
||||
return {
|
||||
...result,
|
||||
expectedContent: normalizedExpected,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
correctionGuide: buildResidentApplicationMessageCorrectionGuide({
|
||||
actualContent,
|
||||
expectedContent: normalizedExpected,
|
||||
}),
|
||||
expectedContent: normalizedExpected,
|
||||
status: 'content-mismatch',
|
||||
};
|
||||
}
|
||||
|
||||
function truncateMessageContent(content: string): string {
|
||||
const normalized = content.replace(/\s+/gu, ' ').trim();
|
||||
return normalized.length > 160 ? `${normalized.slice(0, 157)}...` : normalized;
|
||||
}
|
||||
|
||||
function buildResidentApplicationMessageSummary(
|
||||
result: ResidentApplicationMessageResult,
|
||||
): string | null {
|
||||
if (result.status === 'lookup-unavailable') {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (result.status === 'confirmed' && result.candidate) {
|
||||
return `주민신청 메시지도 확인됐어요.`;
|
||||
}
|
||||
|
||||
if (result.status === 'content-mismatch' && result.correctionGuide) {
|
||||
return result.correctionGuide;
|
||||
}
|
||||
|
||||
if (result.status === 'ambiguous') {
|
||||
return '주민신청 메시지 후보가 여러 개라 하나로 확정하지 못했어요.';
|
||||
}
|
||||
|
||||
return '주민신청 채널 메시지는 아직 확인되지 않았어요.';
|
||||
}
|
||||
|
||||
function residentApplicationMessageFields(
|
||||
result: ResidentApplicationMessageResult,
|
||||
): Pick<
|
||||
z.infer<typeof resultSchema>,
|
||||
| 'residentApplicationMessageAuthorId'
|
||||
| 'residentApplicationMessageContent'
|
||||
| 'residentApplicationMessageCorrectionGuide'
|
||||
| 'residentApplicationMessageCreatedAt'
|
||||
| 'residentApplicationMessageExpectedContent'
|
||||
| 'residentApplicationMessageId'
|
||||
| 'residentApplicationMessageMatchCount'
|
||||
| 'residentApplicationMessageMatchedBy'
|
||||
| 'residentApplicationMessageStatus'
|
||||
> {
|
||||
const candidate = result.candidate;
|
||||
return {
|
||||
residentApplicationMessageCorrectionGuide: result.correctionGuide,
|
||||
residentApplicationMessageExpectedContent: result.expectedContent,
|
||||
residentApplicationMessageMatchCount: result.matchCount,
|
||||
residentApplicationMessageStatus: result.status,
|
||||
...(candidate
|
||||
? {
|
||||
residentApplicationMessageAuthorId: candidate.authorId,
|
||||
residentApplicationMessageContent: truncateMessageContent(candidate.content),
|
||||
residentApplicationMessageCreatedAt: candidate.createdAt,
|
||||
residentApplicationMessageId: candidate.messageId,
|
||||
residentApplicationMessageMatchedBy: candidate.matchedBy,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
interface TimeoutBudget {
|
||||
configuredTimeoutMs: number;
|
||||
deadlineAt?: string | undefined;
|
||||
remainingMs?: number | undefined;
|
||||
timeoutMs: number | null;
|
||||
}
|
||||
|
||||
function resolveTimeoutBudget(): TimeoutBudget {
|
||||
const configured = getDiscordVerifyTimeoutMs();
|
||||
const deadlineAt = getContext()?.deadlineAt;
|
||||
|
||||
if (!deadlineAt) {
|
||||
return configured;
|
||||
return {
|
||||
configuredTimeoutMs: configured,
|
||||
timeoutMs: configured,
|
||||
};
|
||||
}
|
||||
|
||||
const remainingMs = Date.parse(deadlineAt) - Date.now() - 100;
|
||||
if (!Number.isFinite(remainingMs) || remainingMs <= 0) {
|
||||
return null;
|
||||
return {
|
||||
configuredTimeoutMs: configured,
|
||||
deadlineAt,
|
||||
remainingMs,
|
||||
timeoutMs: null,
|
||||
};
|
||||
}
|
||||
|
||||
return Math.min(configured, remainingMs);
|
||||
return {
|
||||
configuredTimeoutMs: configured,
|
||||
deadlineAt,
|
||||
remainingMs,
|
||||
timeoutMs: Math.min(configured, remainingMs),
|
||||
};
|
||||
}
|
||||
|
||||
async function persistDiscordMemberId(memberId: string, ctx: ToolContext): Promise<void> {
|
||||
async function persistDiscordVerification(
|
||||
input: {
|
||||
memberId: string;
|
||||
nicknameVerified: boolean;
|
||||
residentApplicationMessageVerifiedAt?: string | undefined;
|
||||
},
|
||||
ctx: ToolContext,
|
||||
): Promise<void> {
|
||||
if (!ctx.garnetClient || !ctx.session) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const nextSession = {
|
||||
...ctx.session,
|
||||
discordMemberId: memberId,
|
||||
discordMemberId: input.memberId,
|
||||
...(input.nicknameVerified
|
||||
? {
|
||||
discordProfileNicknameVerified: true,
|
||||
discordProfileNicknameVerifiedAt:
|
||||
ctx.session.discordProfileNicknameVerifiedAt ?? now,
|
||||
}
|
||||
: {}),
|
||||
...(input.residentApplicationMessageVerifiedAt
|
||||
? {
|
||||
discordProfileMessageVerified: true,
|
||||
discordProfileMessageVerifiedAt: input.residentApplicationMessageVerifiedAt,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const key = `${getOnboardingSessionKeyPrefix()}:onboarding:${nextSession.chatroomId}:${nextSession.userId}`;
|
||||
await ctx.garnetClient.setJson(key, nextSession);
|
||||
ctx.session.discordMemberId = memberId;
|
||||
Object.assign(ctx.session, nextSession);
|
||||
}
|
||||
|
||||
export const verifyDiscordNicknameTool: Tool<
|
||||
@@ -146,8 +378,7 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
}
|
||||
|
||||
const parsedKakaoNickname = parseKakaoNickname(expectedNickname);
|
||||
const searchName = parsedKakaoNickname?.baseName ?? '';
|
||||
if (!searchName) {
|
||||
if (!parsedKakaoNickname?.baseName) {
|
||||
logger.debug(
|
||||
{
|
||||
expectedNickname,
|
||||
@@ -161,6 +392,7 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
expectedNickname,
|
||||
};
|
||||
}
|
||||
const searchName = parsedKakaoNickname.baseName;
|
||||
|
||||
if (!ctx.natsClient) {
|
||||
logger.debug(
|
||||
@@ -179,11 +411,15 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
};
|
||||
}
|
||||
|
||||
const timeoutMs = resolveTimeoutMs();
|
||||
const timeoutBudget = resolveTimeoutBudget();
|
||||
const timeoutMs = timeoutBudget.timeoutMs;
|
||||
if (!timeoutMs) {
|
||||
logger.debug(
|
||||
{
|
||||
configuredTimeoutMs: timeoutBudget.configuredTimeoutMs,
|
||||
deadlineAt: timeoutBudget.deadlineAt,
|
||||
expectedNickname,
|
||||
remainingMs: timeoutBudget.remainingMs,
|
||||
searchName,
|
||||
},
|
||||
'[VerifyDiscordNickname] No timeout budget left for lookup',
|
||||
@@ -201,6 +437,7 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
try {
|
||||
const payload: DiscordMemberLookupRequest = {
|
||||
chatroomId: ctx.chatroomId ?? ctx.session.chatroomId,
|
||||
includeResidentApplicationMessages: true,
|
||||
joinedAfter: ctx.session.joinedAt,
|
||||
searchName,
|
||||
userId: ctx.userId ?? ctx.session.userId,
|
||||
@@ -209,6 +446,9 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
logger.debug(
|
||||
{
|
||||
payload,
|
||||
configuredTimeoutMs: timeoutBudget.configuredTimeoutMs,
|
||||
deadlineAt: timeoutBudget.deadlineAt,
|
||||
remainingMs: timeoutBudget.remainingMs,
|
||||
subject: getDiscordMemberLookupSubject(),
|
||||
timeoutMs,
|
||||
},
|
||||
@@ -257,11 +497,18 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
|
||||
const nicknameMatches = lookupResult.nicknameMatches;
|
||||
const joinedAfterMatches = lookupResult.joinedAfterMatches;
|
||||
const unvalidatedResidentApplicationMessageResult = resolveResidentApplicationMessageResult(
|
||||
lookupResult.residentApplicationMessageMatches,
|
||||
);
|
||||
logger.debug(
|
||||
{
|
||||
expectedNickname,
|
||||
joinedAfterMatches: joinedAfterMatches.length,
|
||||
nicknameMatches: nicknameMatches.length,
|
||||
residentApplicationMessageMatchCount:
|
||||
unvalidatedResidentApplicationMessageResult.matchCount,
|
||||
residentApplicationMessageStatus:
|
||||
unvalidatedResidentApplicationMessageResult.status,
|
||||
searchName,
|
||||
},
|
||||
'[VerifyDiscordNickname] Lookup response received',
|
||||
@@ -293,6 +540,10 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
expectedNickname,
|
||||
joinedAfterMatchCount: joinedAfterMatches.length,
|
||||
nicknameMatchCount: nicknameMatches.length,
|
||||
residentApplicationMessageMatchCount:
|
||||
unvalidatedResidentApplicationMessageResult.matchCount,
|
||||
residentApplicationMessageStatus:
|
||||
unvalidatedResidentApplicationMessageResult.status,
|
||||
reason:
|
||||
reason ??
|
||||
(nicknameMatches.length === 0 ? 'nickname-search-none' : 'nickname-search-ambiguous'),
|
||||
@@ -310,29 +561,63 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
searchName,
|
||||
joinedAfterMatchCount: joinedAfterMatches.length,
|
||||
nicknameMatchCount: nicknameMatches.length,
|
||||
...residentApplicationMessageFields(unvalidatedResidentApplicationMessageResult),
|
||||
};
|
||||
}
|
||||
|
||||
await persistDiscordMemberId(candidate.memberId, ctx);
|
||||
|
||||
const currentDisplayName = pickDisplayName(candidate);
|
||||
const currentGuildNickname = candidate.guildNickname?.trim();
|
||||
const isRenamed = matchesDiscordNicknameRule(currentGuildNickname, expectedNickname);
|
||||
const status = isRenamed ? 'matched-renamed' : 'matched-pending-rename';
|
||||
const recommendedNickname = buildRecommendedNickname(parsedKakaoNickname, currentGuildNickname);
|
||||
const residentApplicationMessageExpectedContent = isRenamed
|
||||
? currentGuildNickname
|
||||
: recommendedNickname.includes('#태그/')
|
||||
? undefined
|
||||
: recommendedNickname;
|
||||
const residentApplicationMessageResult = validateResidentApplicationMessageContent(
|
||||
unvalidatedResidentApplicationMessageResult,
|
||||
residentApplicationMessageExpectedContent,
|
||||
);
|
||||
await persistDiscordVerification(
|
||||
{
|
||||
memberId: candidate.memberId,
|
||||
nicknameVerified: isRenamed,
|
||||
residentApplicationMessageVerifiedAt:
|
||||
residentApplicationMessageResult.status === 'confirmed'
|
||||
? residentApplicationMessageResult.candidate?.createdAt
|
||||
: undefined,
|
||||
},
|
||||
ctx,
|
||||
);
|
||||
|
||||
const correctionGuide = buildCorrectionGuide({
|
||||
currentDisplayName,
|
||||
currentGuildNickname,
|
||||
});
|
||||
const discordFormatGuide = `디스코드 서버 닉네임은 이름#태그/출생연도/티어 형식이에요. 카카오톡 형식과 달리 성별은 빼고 태그(#)가 들어가요. 예: ${DISCORD_NICKNAME_FORMAT_EXAMPLE}`;
|
||||
const residentApplicationMessageSummary = buildResidentApplicationMessageSummary(
|
||||
residentApplicationMessageResult,
|
||||
);
|
||||
const message = isRenamed
|
||||
? '디스코드 서버 닉네임이 올바른 형식으로 설정되어 있어요. 확인 완료!'
|
||||
: currentGuildNickname
|
||||
? `디스코드에서 유저를 찾았지만 서버 닉네임이 아직 올바른 형식으로 확인되지는 않았어요. 현재 서버 닉네임은 "${currentGuildNickname}"으로 보여요. ${discordFormatGuide}`
|
||||
: `디스코드에서 유저를 찾았지만 서버 닉네임이 아직 설정되지 않았거나 확인되지 않았어요. 현재 표시 이름은 "${currentDisplayName}"으로 보여요. ${discordFormatGuide}`;
|
||||
: correctionGuide;
|
||||
const finalMessage = [message, residentApplicationMessageSummary]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
currentDisplayName,
|
||||
currentGuildNickname,
|
||||
discordMemberId: candidate.memberId,
|
||||
expectedNickname,
|
||||
matchSource,
|
||||
matchedBy: candidate.matchedBy,
|
||||
recommendedNickname,
|
||||
residentApplicationMessageMatchCount:
|
||||
residentApplicationMessageResult.matchCount,
|
||||
residentApplicationMessageStatus: residentApplicationMessageResult.status,
|
||||
status,
|
||||
},
|
||||
'[VerifyDiscordNickname] Verification resolved',
|
||||
@@ -340,12 +625,16 @@ export const verifyDiscordNicknameTool: Tool<
|
||||
|
||||
return {
|
||||
status,
|
||||
message,
|
||||
message: finalMessage,
|
||||
currentDisplayName,
|
||||
correctionGuide: isRenamed ? undefined : correctionGuide,
|
||||
discordMemberId: candidate.memberId,
|
||||
expectedNickname,
|
||||
guidance: isRenamed ? undefined : discordFormatGuide,
|
||||
matchSource,
|
||||
matchedBy: candidate.matchedBy,
|
||||
recommendedNickname: isRenamed ? undefined : recommendedNickname,
|
||||
...residentApplicationMessageFields(residentApplicationMessageResult),
|
||||
searchName,
|
||||
};
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user