diff --git a/package.json b/package.json index 4c822eb..4394043 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", - "eval": "node --import tsx src/eval/cli.ts run --set=golden", + "eval": "node --import tsx src/eval/cli.ts run --set=onboarding/cases", + "eval:moderation": "node --import tsx src/eval/cli.ts run --set=moderation/cases", "eval:capture": "node --import tsx src/eval/cli.ts capture", "eval:baseline": "node --import tsx src/eval/cli.ts baseline", "eval:report": "node --import tsx src/eval/cli.ts report", diff --git a/src/agent/agent.test.ts b/src/agent/agent.test.ts new file mode 100644 index 0000000..f80dea5 --- /dev/null +++ b/src/agent/agent.test.ts @@ -0,0 +1,53 @@ +import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js"; +import { describe, expect, it } from "vitest"; +import type { Skill } from "@skills/skill"; +import { selectSkillForRequest } from "./agent.js"; + +function makeRequest(skill = "shared"): AgentRequest { + return { + chatroomId: "room-1", + nickname: "tester", + requestId: "request-1", + skill, + trigger: "user-message", + userId: "user-1", + }; +} + +function makeSkill(name: string, priority: number, canHandle = true): Skill { + return { + name, + priority, + canHandle: () => canHandle, + handle: ( + request: AgentRequest, + _conversationHistory: ConversationMessage[], + ): Promise => + Promise.resolve({ + confidence: 1, + requestId: request.requestId, + source: "template", + text: name, + }), + }; +} + +describe("selectSkillForRequest", () => { + it("selects the highest-priority matching skill", () => { + const request = makeRequest(); + const selected = selectSkillForRequest( + [ + makeSkill("low", 10), + makeSkill("high", 90), + makeSkill("disabled", 100, false), + ], + request, + ); + + expect(selected?.name).toBe("high"); + }); + + it("returns null when no skill can handle the request", () => { + expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull(); + }); +}); diff --git a/src/agent/agent.ts b/src/agent/agent.ts index c482c09..e52dd65 100644 --- a/src/agent/agent.ts +++ b/src/agent/agent.ts @@ -25,6 +25,17 @@ function isValidRequest(data: unknown): data is AgentRequest { ); } +export function selectSkillForRequest( + skills: readonly Skill[], + request: AgentRequest, +): Skill | null { + return ( + skills + .filter((candidate) => candidate.canHandle(request)) + .sort((left, right) => right.priority - left.priority)[0] ?? null + ); +} + export class Agent { private readonly skills: Skill[] = []; private readonly logger = getLogger(); @@ -96,7 +107,7 @@ export class Agent { "[Agent] Request received", ); - const skill = this.skills.find((candidate) => candidate.canHandle(request)); + const skill = selectSkillForRequest(this.skills, request); if (!skill) { logger.info("[Agent] No skill found for request"); const response: AgentResponse = { diff --git a/src/eval/capture.ts b/src/eval/capture.ts index 06d7cf3..8316a30 100644 --- a/src/eval/capture.ts +++ b/src/eval/capture.ts @@ -14,6 +14,7 @@ import { interface CaptureOptions { limit?: number | undefined; outputPath?: string | undefined; + skill?: string | undefined; } function asRecord(value: unknown): Record { @@ -122,7 +123,8 @@ export async function captureGoldenSetScaffold( const outputPath = resolve( process.cwd(), - options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl", + options?.outputPath ?? + `src/eval/golden/${options?.skill ?? "onboarding"}/capture.jsonl`, ); await mkdir(dirname(outputPath), { recursive: true }); await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8"); diff --git a/src/eval/cli.ts b/src/eval/cli.ts index 655ab77..2e27d50 100644 --- a/src/eval/cli.ts +++ b/src/eval/cli.ts @@ -319,39 +319,46 @@ async function main(): Promise { parsePositiveInteger(getFlagString(flags, "limit")) ?? parsePositiveInteger(positionals[0]), outputPath: getFlagString(flags, "output") ?? positionals[1], + skill: getFlagString(flags, "skill"), }); logger.info({ outputPath }, '[eval] "capture" completed'); return; } if (command === "run") { - const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import( + const { loadEvalCases, resolveEvalSetPath, runEval } = await import( "./runner.js" ); const { publishEvalReportIfPossible } = await import( "./runner.js" ); - const setName = getFlagString(flags, "set") ?? "golden"; - const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName); + const setName = getFlagString(flags, "set") ?? "onboarding/cases"; + const resolvedSet = resolveEvalSetPath(setName); + const skill = getFlagString(flags, "skill") ?? resolvedSet.skill; + const inputPath = getFlagString(flags, "input") ?? resolvedSet.path; const limit = parsePositiveInteger(getFlagString(flags, "limit")); if (hasFlag(flags, "dry-run")) { const cases = await loadEvalCases(inputPath, { limit }); - console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`); + console.info( + `[eval] dry-run skill=${skill} parsed ${cases.length} cases from ${inputPath}`, + ); return; } - const { report, reportPath } = await runOnboardingEval({ + const { report, reportPath } = await runEval({ enableLlmJudge: !hasFlag(flags, "skip-llm-judge"), inputPath, limit, outputPath: getFlagString(flags, "output"), setName, + skill, }); const publishedReport = hasFlag(flags, "no-publish-report") ? false : await publishEvalReportIfPossible(report); printRunSummary(report, reportPath); + console.info(`[eval] skill=${skill}`); console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`); return; } @@ -414,6 +421,7 @@ async function main(): Promise { const result = await appendCandidatesToJsonl( candidates, getFlagString(flags, "output"), + getFlagString(flags, "skill"), ); console.info( `[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`, diff --git a/src/eval/golden/moderation/candidate.jsonl b/src/eval/golden/moderation/candidate.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/src/eval/golden/moderation/capture.jsonl b/src/eval/golden/moderation/capture.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/src/eval/golden/moderation/cases.jsonl b/src/eval/golden/moderation/cases.jsonl new file mode 100644 index 0000000..3c6550b --- /dev/null +++ b/src/eval/golden/moderation/cases.jsonl @@ -0,0 +1,5 @@ +{"id":"moderation_rule_ads","name":"moderation:rule-question:ads","input":{"request":{"requestId":"mod_req_ads","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_1","nickname":"문의자","trigger":"user-message","userMessage":"광고 올려도 돼요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["광고","홍보","허가","제한"],"mustNotInclude":["무조건 가능","바로 올려도"]}} +{"id":"moderation_rule_profanity","name":"moderation:rule-question:profanity","input":{"request":{"requestId":"mod_req_profanity","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_2","nickname":"문의자","trigger":"user-message","userMessage":"욕설하면 어떻게 돼요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["욕설","금지","운영진"],"mustNotInclude":["괜찮아요","상관없"]}} +{"id":"moderation_rule_spam","name":"moderation:rule-question:spam","input":{"request":{"requestId":"mod_req_spam","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_3","nickname":"문의자","trigger":"user-message","userMessage":"도배 기준이 뭐예요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["도배","반복","제재"],"mustNotInclude":["기준 없음","상관없"]}} +{"id":"moderation_report_profanity","name":"moderation:violation-report:profanity","input":{"request":{"requestId":"mod_req_report","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_4","nickname":"제보자","trigger":"user-message","userMessage":"어떤 분이 욕했어요 신고하고 싶어요"}},"expected":{"intent":"violation-report","mustIncludeAny":["관련 메시지","상황","운영진"],"mustNotInclude":["제가 처리","제재 확정"]}} +{"id":"moderation_appeal_sanction","name":"moderation:appeal:sanction","input":{"request":{"requestId":"mod_req_appeal","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_5","nickname":"소명자","trigger":"user-message","userMessage":"정지당했는데 억울해요"}},"expected":{"intent":"appeal","mustIncludeAny":["이의 제기","소명","운영진"],"mustNotInclude":["해제됐","풀렸"]}} diff --git a/src/eval/golden/onboarding.candidate.jsonl b/src/eval/golden/onboarding/candidate.jsonl similarity index 100% rename from src/eval/golden/onboarding.candidate.jsonl rename to src/eval/golden/onboarding/candidate.jsonl diff --git a/src/eval/golden/onboarding.capture.jsonl b/src/eval/golden/onboarding/capture.jsonl similarity index 100% rename from src/eval/golden/onboarding.capture.jsonl rename to src/eval/golden/onboarding/capture.jsonl diff --git a/src/eval/golden/onboarding.jsonl b/src/eval/golden/onboarding/cases.jsonl similarity index 100% rename from src/eval/golden/onboarding.jsonl rename to src/eval/golden/onboarding/cases.jsonl diff --git a/src/eval/judges/rule-judge.ts b/src/eval/judges/rule-judge.ts index 0baecfe..72e6e07 100644 --- a/src/eval/judges/rule-judge.ts +++ b/src/eval/judges/rule-judge.ts @@ -10,6 +10,113 @@ function round(value: number): number { return Number(value.toFixed(3)); } +export function judgeRuleCase( + expected: EvalCaseExpected, + responseText: string, + actual: { + intent: string; + responseMode?: string | undefined; + }, +): RuleJudgeResult { + const checks: RuleJudgeCheck[] = []; + + checks.push({ + name: "validator", + passed: true, + skipped: true, + detail: "not configured for this skill", + }); + + const expectedIntent = expected.intent?.trim(); + if (expectedIntent) { + checks.push({ + name: "intent", + passed: normalizeText(expectedIntent) === normalizeText(actual.intent), + skipped: false, + detail: `expected=${expectedIntent}, actual=${actual.intent}`, + }); + } else { + checks.push({ + name: "intent", + passed: true, + skipped: true, + }); + } + + const expectedResponseMode = expected.responseMode?.trim(); + if (expectedResponseMode && actual.responseMode) { + checks.push({ + name: "responseMode", + passed: + normalizeText(expectedResponseMode) === + normalizeText(actual.responseMode), + skipped: false, + detail: `expected=${expectedResponseMode}, actual=${actual.responseMode}`, + }); + } else { + checks.push({ + name: "responseMode", + passed: true, + skipped: true, + }); + } + + const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? []; + if (mustIncludeAny.length > 0) { + const normalizedResponse = normalizeText(responseText); + const matched = mustIncludeAny.find((candidate) => + normalizedResponse.includes(normalizeText(candidate)), + ); + + checks.push({ + name: "mustIncludeAny", + passed: Boolean(matched), + skipped: false, + detail: matched + ? `matched=${matched}` + : `expected one of: ${mustIncludeAny.join(", ")}`, + }); + } else { + checks.push({ + name: "mustIncludeAny", + passed: true, + skipped: true, + }); + } + + const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? []; + if (mustNotInclude.length > 0) { + const normalizedResponse = normalizeText(responseText); + const matched = mustNotInclude.find((candidate) => + normalizedResponse.includes(normalizeText(candidate)), + ); + + checks.push({ + name: "mustNotInclude", + passed: !matched, + skipped: false, + detail: matched ? `forbidden=${matched}` : undefined, + }); + } else { + checks.push({ + name: "mustNotInclude", + passed: true, + skipped: true, + }); + } + + const activeChecks = checks.filter((check) => !check.skipped); + const passedChecks = activeChecks.filter((check) => check.passed).length; + + return { + passed: activeChecks.every((check) => check.passed), + score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length), + checks, + validatorStatus: "pass", + validatorReasons: [], + }; +} + export function judgeOnboardingRuleCase( expected: EvalCaseExpected, responseText: string, diff --git a/src/eval/promote.ts b/src/eval/promote.ts index 296514a..7047a21 100644 --- a/src/eval/promote.ts +++ b/src/eval/promote.ts @@ -20,7 +20,6 @@ import { getEvalArchiveCollection } from "./archive.js"; import { queryArchivedReviewLabels } from "./review/label-archive.js"; import type { ReviewLabel } from "./review/queue.js"; -const DEFAULT_CANDIDATE_PATH = "src/eval/golden/onboarding.candidate.jsonl"; const DAY_MS = 24 * 60 * 60 * 1000; export interface PromotionCandidate { @@ -47,6 +46,10 @@ function stableShortHash(value: string): string { return createHash("sha256").update(value).digest("hex").slice(0, 8); } +function defaultCandidatePathFor(skill: string): string { + return `src/eval/golden/${skill}/candidate.jsonl`; +} + function getLlmOverall(entry: EvalArchiveEntry): number | null { if (entry.llmJudge.skipped) { return null; @@ -326,9 +329,10 @@ function extractExistingCaseIds(raw: string): Set { export async function appendCandidatesToJsonl( candidates: PromotionCandidate[], - outputPath = DEFAULT_CANDIDATE_PATH, + outputPath?: string, + skill = "onboarding", ): Promise<{ appendedCount: number; outputPath: string }> { - const resolved = resolve(process.cwd(), outputPath); + const resolved = resolve(process.cwd(), outputPath ?? defaultCandidatePathFor(skill)); await mkdir(dirname(resolved), { recursive: true }); const existingRaw = existsSync(resolved) ? await readFile(resolved, "utf8") diff --git a/src/eval/runner.test.ts b/src/eval/runner.test.ts index 28cc16f..d910a99 100644 --- a/src/eval/runner.test.ts +++ b/src/eval/runner.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects"; -import { publishEvalReportIfPossible } from "./runner.js"; +import { publishEvalReportIfPossible, resolveEvalSetPath } from "./runner.js"; import type { EvalReport } from "./types.js"; const natsMocks = vi.hoisted(() => ({ @@ -54,3 +54,29 @@ describe("publishEvalReportIfPossible", () => { expect(natsMocks.close).toHaveBeenCalledTimes(1); }); }); + +describe("resolveEvalSetPath", () => { + it("resolves skill/subset eval sets", () => { + const resolved = resolveEvalSetPath("moderation/cases"); + + expect(resolved).toMatchObject({ + skill: "moderation", + subset: "cases", + }); + expect(resolved.path).toContain("src/eval/golden/moderation/cases.jsonl"); + }); + + it("maps legacy golden set to onboarding cases", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const resolved = resolveEvalSetPath("golden"); + + expect(resolved.skill).toBe("onboarding"); + expect(resolved.subset).toBe("cases"); + expect(warn).toHaveBeenCalledWith( + "[eval] --set=golden is deprecated; use --set=onboarding/cases", + ); + + warn.mockRestore(); + }); +}); diff --git a/src/eval/runner.ts b/src/eval/runner.ts index 943434e..df8c400 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -13,6 +13,9 @@ import { runWithContext } from "@obs/context"; import { getLogger } from "@obs/logger"; import type { RoomMessage } from "@room/types"; import { RoomContextService } from "@room/room-context-service"; +import type { Skill } from "@skills/skill"; +import { ModerationSkill } from "@skills/moderation/index"; +import { classifyModerationIntent } from "@skills/moderation/intents"; import { OnboardingSkill } from "@skills/onboarding/index"; import { ProcessTraceCollector, @@ -29,7 +32,10 @@ import { noopShadowEvalPublisher } from "./shadow-bus.js"; import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js"; import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; import { judgeProcessRule } from "./judges/process-judge.js"; -import { judgeOnboardingRuleCase } from "./judges/rule-judge.js"; +import { + judgeOnboardingRuleCase, + judgeRuleCase, +} from "./judges/rule-judge.js"; import type { EvalBaselineFile, EvalCaseDefinition, @@ -44,11 +50,32 @@ import type { } from "./types.js"; const DEFAULT_REPORTS_DIR = "eval-reports"; -const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`; -const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl"; -const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl"; +const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`; const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline"; +export interface ResolvedEvalSetPath { + path: string; + skill: string; + subset: string; +} + +interface SkillRunnerDeps { + garnet: GarnetClient; + llm: LlmClient; + room?: RoomContextService | undefined; +} + +const SKILL_RUNNER_FACTORIES: Record Skill> = { + moderation: (deps) => new ModerationSkill(deps.llm), + onboarding: (deps) => + new OnboardingSkill( + deps.garnet, + deps.llm, + deps.room, + noopShadowEvalPublisher, + ), +}; + class InMemoryGarnetClient extends GarnetClient { private readonly hashStore = new Map>(); private readonly stringStore = new Map(); @@ -127,6 +154,7 @@ export interface EvalRunOptions { limit?: number | undefined; outputPath?: string | undefined; setName?: string | undefined; + skill?: string | undefined; } function round(value: number): number { @@ -580,30 +608,72 @@ async function writeReport( return resolved; } -async function loadBaselineFile(): Promise { - const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE); - if (!existsSync(baselinePath)) { +function baselinePathFor(skill: string): string { + return `${DEFAULT_REPORTS_DIR}/baseline.${skill}.json`; +} + +async function loadBaselineFile(skill: string): Promise { + const baselinePath = resolve(process.cwd(), baselinePathFor(skill)); + const legacyBaselinePath = resolve(process.cwd(), LEGACY_BASELINE_FILE); + const path = existsSync(baselinePath) + ? baselinePath + : skill === "onboarding" && existsSync(legacyBaselinePath) + ? legacyBaselinePath + : null; + if (!path) { return null; } - const raw = await readFile(baselinePath, "utf8"); + const raw = await readFile(path, "utf8"); return JSON.parse(raw) as EvalBaselineFile; } -export function resolveEvalSetPath(setNameOrPath = "golden"): string { +export function resolveEvalSetPath( + setNameOrPath = "onboarding/cases", +): ResolvedEvalSetPath { if (setNameOrPath === "golden") { - return resolve(process.cwd(), GOLDEN_SET_PATH); + console.warn( + '[eval] --set=golden is deprecated; use --set=onboarding/cases', + ); + return { + path: resolve(process.cwd(), "src/eval/golden/onboarding/cases.jsonl"), + skill: "onboarding", + subset: "cases", + }; } if (setNameOrPath === "capture") { - return resolve(process.cwd(), CAPTURE_SET_PATH); + console.warn( + '[eval] --set=capture is deprecated; use --set=onboarding/capture', + ); + return { + path: resolve(process.cwd(), "src/eval/golden/onboarding/capture.jsonl"), + skill: "onboarding", + subset: "capture", + }; } - if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) { - return resolve(process.cwd(), setNameOrPath); + if (setNameOrPath.endsWith(".jsonl")) { + return { + path: resolve(process.cwd(), setNameOrPath), + skill: "onboarding", + subset: basename(setNameOrPath, ".jsonl"), + }; } - return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`); + const [skill, subset = "cases"] = setNameOrPath.includes("/") + ? setNameOrPath.split("/", 2) + : [setNameOrPath, "cases"]; + + if (!skill?.trim() || !subset.trim()) { + throw new Error(`Invalid eval set name: ${setNameOrPath}`); + } + + return { + path: resolve(process.cwd(), "src/eval/golden", skill, `${subset}.jsonl`), + skill, + subset, + }; } export async function loadEvalCases( @@ -632,6 +702,7 @@ async function evaluateCase( evalCase: EvalCaseDefinition, llmClient: LlmClient, enableLlmJudge: boolean, + skillName: string, ): Promise { const input = evalCase.input; const inMemoryGarnet = new InMemoryGarnetClient(); @@ -641,14 +712,16 @@ async function evaluateCase( input.roomContext && input.roomContext.length > 0 ? new InMemoryRoomContextService(toRoomMessages(input.roomContext)) : undefined; - const skill = new OnboardingSkill( - inMemoryGarnet, - llmClient, - roomContextService, - noopShadowEvalPublisher, - ); - const classifier = new OnboardingIntentClassifier(llmClient); - const processTrace = new ProcessTraceCollector(); + const effectiveSkill = input.request.skill || skillName; + const factory = SKILL_RUNNER_FACTORIES[effectiveSkill]; + if (!factory) { + throw new Error(`No eval runner factory for skill: ${effectiveSkill}`); + } + const skill = factory({ + garnet: inMemoryGarnet, + llm: llmClient, + room: roomContextService, + }); return runWithContext( { @@ -660,109 +733,219 @@ async function evaluateCase( trigger: input.request.trigger, userId: input.request.userId, }, - () => runWithProcessTrace(processTrace, async () => { - const classified = await classifier.classify( - input.request.userMessage, - input.request.trigger, - ); - const derived = deriveState( - input.session ?? null, - input.request.trigger, - classified.intent, - ); - - const startedAt = Date.now(); - try { - const response = await skill.handle( - input.request, - input.conversationHistory ?? [], - ); - const latencyMs = Date.now() - startedAt; - const derivedSnapshot = toDerivedSnapshot(derived); - const ruleJudge = judgeOnboardingRuleCase( - evalCase.expected, - response.text, - derived, - ); - const llmJudge = enableLlmJudge - ? await judgeOnboardingWithLlm( - llmClient, - evalCase, - response.text, - derivedSnapshot, - ) - : { - skipped: true as const, - reason: "disabled", - }; - const processTraceSnapshot = processTrace.snapshot(); - const processJudge = judgeProcessRule( - evalCase.expected, - processTraceSnapshot, - ); - - return { - id: evalCase.id, - lineNumber: evalCase.lineNumber, - name: evalCase.name, - latencyMs, - classified: toClassifiedSnapshot(classified), - derived: derivedSnapshot, - request: input.request, - response, - ruleJudge, - llmJudge, - processJudge, - processTrace: processTraceSnapshot, - }; - } catch (error) { - const latencyMs = Date.now() - startedAt; - const errorMessage = - error instanceof Error ? error.message : String(error); - const processTraceSnapshot = processTrace.snapshot(); - - return { - id: evalCase.id, - lineNumber: evalCase.lineNumber, - name: evalCase.name, - latencyMs, - classified: toClassifiedSnapshot(classified), - derived: toDerivedSnapshot(derived), - request: input.request, - response: { - requestId: input.request.requestId, - text: "", - confidence: 0, - source: "template", - metadata: { - error: errorMessage, - }, - }, - ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived), - llmJudge: { - skipped: true, - reason: "skill-error", - }, - processJudge: judgeProcessRule( - evalCase.expected, - processTraceSnapshot, - ), - processTrace: processTraceSnapshot, - error: errorMessage, - }; - } - }), + () => + effectiveSkill === "onboarding" + ? evaluateOnboardingCase( + evalCase, + input, + skill, + llmClient, + enableLlmJudge, + ) + : evaluateGenericSkillCase(evalCase, input, skill), ); } -export async function runOnboardingEval( +async function evaluateOnboardingCase( + evalCase: EvalCaseDefinition, + input: EvalCaseInput, + skill: Skill, + llmClient: LlmClient, + enableLlmJudge: boolean, +): Promise { + const classifier = new OnboardingIntentClassifier(llmClient); + const processTrace = new ProcessTraceCollector(); + + return runWithProcessTrace(processTrace, async () => { + const classified = await classifier.classify( + input.request.userMessage, + input.request.trigger, + ); + const derived = deriveState( + input.session ?? null, + input.request.trigger, + classified.intent, + ); + + const startedAt = Date.now(); + try { + const response = await skill.handle( + input.request, + input.conversationHistory ?? [], + ); + const latencyMs = Date.now() - startedAt; + const derivedSnapshot = toDerivedSnapshot(derived); + const ruleJudge = judgeOnboardingRuleCase( + evalCase.expected, + response.text, + derived, + ); + const llmJudge = enableLlmJudge + ? await judgeOnboardingWithLlm( + llmClient, + evalCase, + response.text, + derivedSnapshot, + ) + : { + skipped: true as const, + reason: "disabled", + }; + const processTraceSnapshot = processTrace.snapshot(); + const processJudge = judgeProcessRule( + evalCase.expected, + processTraceSnapshot, + ); + + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: toClassifiedSnapshot(classified), + derived: derivedSnapshot, + request: input.request, + response, + ruleJudge, + llmJudge, + processJudge, + processTrace: processTraceSnapshot, + }; + } catch (error) { + const latencyMs = Date.now() - startedAt; + const errorMessage = + error instanceof Error ? error.message : String(error); + const processTraceSnapshot = processTrace.snapshot(); + + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: toClassifiedSnapshot(classified), + derived: toDerivedSnapshot(derived), + request: input.request, + response: { + requestId: input.request.requestId, + text: "", + confidence: 0, + source: "template", + metadata: { + error: errorMessage, + }, + }, + ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived), + llmJudge: { + skipped: true, + reason: "skill-error", + }, + processJudge: judgeProcessRule( + evalCase.expected, + processTraceSnapshot, + ), + processTrace: processTraceSnapshot, + error: errorMessage, + }; + } + }); +} + +async function evaluateGenericSkillCase( + evalCase: EvalCaseDefinition, + input: EvalCaseInput, + skill: Skill, +): Promise { + const classified = classifyModerationIntent(input.request.userMessage ?? ""); + const derived: EvalDerivedSnapshot = { + allowShortResponse: true, + intent: classified.intent.kind, + phase: skill.name, + responseMode: classified.intent.kind === "off-topic" ? "template" : "llm", + shouldRespond: classified.intent.kind !== "off-topic", + suggestedMaxTokens: 192, + tier: 0, + }; + const startedAt = Date.now(); + + try { + const response = await skill.handle( + input.request, + input.conversationHistory ?? [], + ); + const latencyMs = Date.now() - startedAt; + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: { + confidence: classified.confidence, + intent: classified.intent.kind, + source: "regex", + }, + derived, + request: input.request, + response, + ruleJudge: judgeRuleCase(evalCase.expected, response.text, { + intent: classified.intent.kind, + responseMode: derived.responseMode, + }), + llmJudge: { + skipped: true, + reason: "unsupported-skill", + }, + }; + } catch (error) { + const latencyMs = Date.now() - startedAt; + const errorMessage = error instanceof Error ? error.message : String(error); + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: { + confidence: classified.confidence, + intent: classified.intent.kind, + source: "regex", + }, + derived, + request: input.request, + response: { + requestId: input.request.requestId, + text: "", + confidence: 0, + source: "template", + metadata: { + error: errorMessage, + }, + }, + ruleJudge: judgeRuleCase(evalCase.expected, "", { + intent: classified.intent.kind, + responseMode: derived.responseMode, + }), + llmJudge: { + skipped: true, + reason: "skill-error", + }, + error: errorMessage, + }; + } +} + +export async function runEval( options?: EvalRunOptions, ): Promise<{ report: EvalReport; reportPath: string }> { const logger = getLogger(); - const setName = options?.setName ?? "golden"; - const inputPath = options?.inputPath - ? resolve(process.cwd(), options.inputPath) + const setName = options?.setName ?? "onboarding/cases"; + const resolvedSet = options?.inputPath + ? { + path: resolve(process.cwd(), options.inputPath), + skill: options.skill ?? "onboarding", + subset: basename(options.inputPath, ".jsonl"), + } : resolveEvalSetPath(setName); + const skill = options?.skill ?? resolvedSet.skill; + const inputPath = resolvedSet.path; const evalCases = await loadEvalCases(inputPath, { limit: options?.limit, }); @@ -772,11 +955,11 @@ export async function runOnboardingEval( const caseResults: EvalCaseResult[] = []; for (const evalCase of evalCases) { - caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge)); + caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge, skill)); } const summary = createSummary(caseResults); - const baseline = await loadBaselineFile(); + const baseline = await loadBaselineFile(skill); const report: EvalReport = { generatedAt: new Date().toISOString(), inputPath, @@ -798,6 +981,7 @@ export async function runOnboardingEval( cases: summary.cases, passRate: summary.passRate, reportPath, + skill, }, "[Eval] run completed", ); @@ -805,6 +989,12 @@ export async function runOnboardingEval( return { report, reportPath }; } +export async function runOnboardingEval( + options?: EvalRunOptions, +): Promise<{ report: EvalReport; reportPath: string }> { + return runEval({ ...options, skill: options?.skill ?? "onboarding" }); +} + export async function findLatestEvalReportPath(): Promise { const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR); if (!existsSync(reportsDir)) { @@ -815,7 +1005,8 @@ export async function findLatestEvalReportPath(): Promise { const candidates = entries .filter((entry) => entry.isFile()) .map((entry) => entry.name) - .filter((name) => name.endsWith(".json") && name !== "baseline.json") + .filter((name) => name.endsWith(".json") && !name.startsWith("baseline.")) + .filter((name) => name !== "baseline.json") .sort((left, right) => right.localeCompare(left)); if (candidates.length === 0) { @@ -845,6 +1036,7 @@ export async function updateEvalBaseline( const raw = await readFile(sourcePath, "utf8"); const report = JSON.parse(raw) as EvalReport; + const skill = resolveEvalSetPath(report.setName).skill; const baseline: EvalBaselineFile = { reportPath: sourcePath, setName: report.setName, @@ -852,7 +1044,7 @@ export async function updateEvalBaseline( updatedAt: new Date().toISOString(), }; - const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE); + const baselinePath = resolve(process.cwd(), baselinePathFor(skill)); await mkdir(dirname(baselinePath), { recursive: true }); await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8"); diff --git a/src/eval/shadow-worker.ts b/src/eval/shadow-worker.ts index ee00c68..83d1c31 100644 --- a/src/eval/shadow-worker.ts +++ b/src/eval/shadow-worker.ts @@ -80,7 +80,10 @@ const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq"; const BACKGROUND_GROUP = "bg-worker"; const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline"; const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results"; -const EVAL_BASELINE_FILE = "eval-reports/baseline.json"; +const EVAL_BASELINE_FILES = [ + "eval-reports/baseline.onboarding.json", + "eval-reports/baseline.json", +]; const ALERT_WINDOW_MS = 5 * 60_000; const ALERT_WINDOW_LABEL = "5m"; const ALERT_TICK_MS = 60_000; @@ -1123,8 +1126,10 @@ export class ShadowWorker { return redisParsed; } - const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE); - if (!existsSync(filePath)) { + const filePath = EVAL_BASELINE_FILES + .map((candidate) => resolve(process.cwd(), candidate)) + .find((candidate) => existsSync(candidate)); + if (!filePath) { return null; } diff --git a/src/main.ts b/src/main.ts index 73b6118..8957623 100644 --- a/src/main.ts +++ b/src/main.ts @@ -228,7 +228,17 @@ async function bootstrap(): Promise { moderation: createModerationSkill, onboarding: createOnboardingSkill, }; + const seenSkills = new Set(); for (const manifest of skillRegistry.list()) { + if (seenSkills.has(manifest.name)) { + logger.warn( + { skill: manifest.name }, + "[Skill] Duplicate manifest ignored", + ); + continue; + } + seenSkills.add(manifest.name); + const factory = factories[manifest.name]; if (!factory) { logger.warn( diff --git a/src/skills/moderation/SKILL.md b/src/skills/moderation/SKILL.md index fd2e7ba..7266ec6 100644 --- a/src/skills/moderation/SKILL.md +++ b/src/skills/moderation/SKILL.md @@ -2,10 +2,12 @@ name: moderation description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill. version: 1 +priority: 50 intents: [greeting, rule-question, violation-report, appeal, off-topic] triggers: [user-message] -tools: [] -resources: {} +tools: [lookup-rule] +resources: + community-rules: ./resources/community-rules.md trustTier: T4 --- diff --git a/src/skills/moderation/intents.ts b/src/skills/moderation/intents.ts index 5f758d7..7ecd080 100644 --- a/src/skills/moderation/intents.ts +++ b/src/skills/moderation/intents.ts @@ -6,19 +6,21 @@ interface ModerationIntentDefinition { export const MODERATION_INTENT_TABLE = { greeting: { confidence: 0.9, - regex: [/^(안녕|하이|반갑|문의드립니다)/u], + regex: [/^(안녕|하이|반갑|문의드립|운영진|모더|관리자)/u], }, "rule-question": { confidence: 0.92, - regex: [/(규칙|룰|금지|가능|해도\s*돼|하면\s*안|제재|경고)/u], + regex: [ + /(규칙|룰|금지|가능|해도\s*돼|하면\s*안|제재|경고|광고|홍보|도배|링크|욕설|혐오|차별)/u, + ], }, "violation-report": { confidence: 0.94, - regex: [/(신고|제보|위반|욕설|분쟁|비매너|문제\s*있|처리해)/u], + regex: [/(신고|제보|위반|욕설|분쟁|비매너|문제\s*있|처리해|누가|사람이|유저가|이용자)/u], }, appeal: { confidence: 0.94, - regex: [/(이의|소명|억울|해제|풀어|재검토|항소|정지)/u], + regex: [/(이의|소명|억울|해제|풀어|재검토|항소|정지|벤|밴|차단\s*풀|차단)/u], }, "off-topic": { confidence: 0, diff --git a/src/skills/moderation/prompt-data.ts b/src/skills/moderation/prompt-data.ts index ce45df5..31d5a84 100644 --- a/src/skills/moderation/prompt-data.ts +++ b/src/skills/moderation/prompt-data.ts @@ -8,6 +8,7 @@ export const MODERATION_PROMPT_DATA = { policy: { base: [ "- 답변은 1~3문장으로 짧게 한다.", + "- 참고 규칙이 제공되면 그 범위 안에서만 답한다.", "- 처벌, 제재, 신고 처리 결과를 확정해서 말하지 않는다.", "- 개인정보, 계정 정보, 특정 이용자 평가를 요구하면 운영진 확인으로 돌린다.", ], diff --git a/src/skills/moderation/resources/community-rules.md b/src/skills/moderation/resources/community-rules.md new file mode 100644 index 0000000..311510b --- /dev/null +++ b/src/skills/moderation/resources/community-rules.md @@ -0,0 +1,35 @@ +## ads +광고, 홍보, 외부 링크 반복 게시물은 운영진 허가가 없는 경우 제한될 수 있다. +개인 방송, 거래, 모집 홍보도 사전 확인이 필요하다. + +## spam +같거나 유사한 메시지를 짧은 시간에 반복하는 도배는 제재 대상이 될 수 있다. +대화 흐름을 방해하는 과도한 멘션이나 이모지도 도배로 볼 수 있다. + +## profanity +욕설, 혐오 표현, 차별적 표현, 특정 이용자 비하 발언은 금지된다. +분쟁이 생기면 맞대응하지 말고 운영진에게 상황을 전달한다. + +## report +위반 제보는 문제 메시지, 시간, 관련 이용자, 맥락을 함께 정리해 운영진에게 전달한다. +처리 결과와 제재 여부는 운영진 확인 뒤 결정된다. + +## sanctions +제재는 경고, 메시지 삭제, 임시 제한, 강퇴 등 단계적으로 적용될 수 있다. +사안의 반복성, 고의성, 피해 정도에 따라 운영진이 판단한다. + +## appeal +정지나 차단이 억울하면 당시 상황, 근거, 반성 또는 소명 내용을 정리해 운영진에게 이의 제기한다. +assistant는 해제 여부를 확정하지 않는다. + +## minor-safety +미성년자 보호를 위해 부적절한 대화, 사적 연락 강요, 성적 발언은 엄격히 제한된다. +불편하거나 위험한 상황은 즉시 운영진에게 알린다. + +## privacy +전화번호, 주소, 계정 정보, 실명 등 개인정보 공유나 요구는 금지된다. +타인의 개인정보를 발견하면 확산하지 말고 운영진에게 제보한다. + +## off-topic +규칙과 무관한 대화는 해당 채팅방의 목적에 맞는 곳으로 옮기도록 안내한다. +반복적으로 흐름을 방해하면 운영진 확인이 필요할 수 있다. diff --git a/src/skills/moderation/skill.ts b/src/skills/moderation/skill.ts index 0d74818..a224a3d 100644 --- a/src/skills/moderation/skill.ts +++ b/src/skills/moderation/skill.ts @@ -6,10 +6,22 @@ import type { import type { LlmClient } from "@llm/llm-client"; import type { ChatMessage } from "@llm/types"; import type { Skill } from "@skills/skill"; +import { lookupRuleTool } from "@tools/moderation/lookup-rule"; import { classifyModerationIntent, type ModerationIntent } from "./intents.js"; import { MODERATION_PROMPT_DATA } from "./prompt-data.js"; import { createModerationToolRegistry } from "./tools.js"; +type RuleTopic = + | "ads" + | "appeal" + | "minor-safety" + | "off-topic" + | "privacy" + | "profanity" + | "report" + | "sanctions" + | "spam"; + function formatHistory(history: ConversationMessage[]): string { if (history.length === 0) { return "(none)"; @@ -32,11 +44,15 @@ function buildMessages( request: AgentRequest, history: ConversationMessage[], intent: ModerationIntent, + referenceText?: string, ): ChatMessage[] { const policies = [ ...MODERATION_PROMPT_DATA.policy.base, ...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind], ]; + const referenceBlock = referenceText?.trim() + ? ["", "[reference]", referenceText.trim()] + : []; return [ { @@ -52,6 +68,7 @@ function buildMessages( "", "[examples]", formatExamples(intent), + ...referenceBlock, ].join("\n"), }, { @@ -67,6 +84,52 @@ function buildMessages( ]; } +function resolveRuleTopic(message: string): RuleTopic | null { + if (/(광고|홍보|링크)/u.test(message)) { + return "ads"; + } + if (/(도배|반복|스팸)/u.test(message)) { + return "spam"; + } + if (/(욕설|욕했|혐오|차별|비하)/u.test(message)) { + return "profanity"; + } + if (/(신고|제보|위반)/u.test(message)) { + return "report"; + } + if (/(경고|제재|강퇴|처벌)/u.test(message)) { + return "sanctions"; + } + if (/(이의|소명|억울|정지|벤|밴|차단)/u.test(message)) { + return "appeal"; + } + if (/(미성년|미자|성적|보호)/u.test(message)) { + return "minor-safety"; + } + if (/(개인정보|전화번호|주소|실명|계정)/u.test(message)) { + return "privacy"; + } + if (/(잡담|주제|오프토픽|off-topic)/iu.test(message)) { + return "off-topic"; + } + return null; +} + +async function lookupReferenceText( + intent: ModerationIntent, + message: string, +): Promise { + if (intent.kind !== "rule-question") { + return undefined; + } + const topic = resolveRuleTopic(message); + if (!topic) { + return undefined; + } + const result = await lookupRuleTool.execute({ topic }, {}); + return result.text; +} + function templateForIntent(intent: ModerationIntent): string { switch (intent.kind) { case "appeal": @@ -84,6 +147,7 @@ function templateForIntent(intent: ModerationIntent): string { export class ModerationSkill implements Skill { readonly name = "moderation"; + readonly priority = 50; private readonly toolRegistry = createModerationToolRegistry(); constructor(private readonly llmClient: LlmClient) {} @@ -97,6 +161,10 @@ export class ModerationSkill implements Skill { conversationHistory: ConversationMessage[], ): Promise { const classified = classifyModerationIntent(request.userMessage ?? ""); + const referenceText = await lookupReferenceText( + classified.intent, + request.userMessage ?? "", + ); const metadata = { intent: classified.intent.kind, toolCount: this.toolRegistry.list().length, @@ -113,7 +181,7 @@ export class ModerationSkill implements Skill { } const response = await this.llmClient.chatCompletion( - buildMessages(request, conversationHistory, classified.intent), + buildMessages(request, conversationHistory, classified.intent, referenceText), { maxTokens: 192, temperature: 0.2, @@ -126,7 +194,10 @@ export class ModerationSkill implements Skill { metadata, requestId: request.requestId, source: response ? "llm" : "template", - text: response?.content.trim() || templateForIntent(classified.intent), + text: + response?.content.trim() || + referenceText || + templateForIntent(classified.intent), }; } } diff --git a/src/skills/moderation/tools.ts b/src/skills/moderation/tools.ts index 7ac8400..811dd04 100644 --- a/src/skills/moderation/tools.ts +++ b/src/skills/moderation/tools.ts @@ -1,5 +1,6 @@ import { ToolRegistry } from "@tools/registry"; +import { lookupRuleTool } from "@tools/moderation/lookup-rule"; export function createModerationToolRegistry(): ToolRegistry { - return new ToolRegistry(); + return new ToolRegistry().register(lookupRuleTool); } diff --git a/src/skills/onboarding/SKILL.md b/src/skills/onboarding/SKILL.md index 195dc93..39439d2 100644 --- a/src/skills/onboarding/SKILL.md +++ b/src/skills/onboarding/SKILL.md @@ -2,6 +2,7 @@ name: onboarding description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할. version: 1 +priority: 70 intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question] triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed] tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname] diff --git a/src/skills/onboarding/runtime/fallback-text-resolver.ts b/src/skills/onboarding/runtime/fallback-text-resolver.ts new file mode 100644 index 0000000..b22bb1c --- /dev/null +++ b/src/skills/onboarding/runtime/fallback-text-resolver.ts @@ -0,0 +1,78 @@ +import { + getAcknowledgementMessage, + getAgeRequirementMessage, + getDiscordMessage, + getGenericHelpMessage, + getGreetingMessage, + getHeartGuideMessage, + getHeartLocationMessage, + getNicknameInvalidMessage, + getNicknameFormatMessage, + getPhotoFailedMessage, + getRemainingStatusMessage, +} from '../onboarding-templates.js'; +import type { DerivedState, Intent } from '../state/onboarding-state.js'; +import type { OnboardingSession } from '../types.js'; + +export function getIntentFallbackText( + nickname: string, + session: OnboardingSession | null, + intent: Intent, + derived: DerivedState, +): string { + switch (intent.kind) { + case 'greeting': + return getGreetingMessage(); + case 'acknowledgement': + return getAcknowledgementMessage(); + case 'status-question': + return getRemainingStatusMessage(session); + case 'nickname-question': + return derived.phase.kind === 'nickname-rejected' + ? getNicknameInvalidMessage(nickname, session) + : getNicknameFormatMessage(); + case 'photo-question': + return derived.phase.kind === 'photo-rejected' + ? getPhotoFailedMessage(nickname) + : getHeartGuideMessage(); + case 'help-where': + if ( + derived.phase.kind === 'photo-pending' || + derived.phase.kind === 'photo-rejected' || + (derived.phase.kind === 'deadline-near' && + derived.phase.remaining.includes('photo')) + ) { + return getHeartLocationMessage(); + } + return getGenericHelpMessage(nickname, session); + case 'help-how': + if ( + derived.phase.kind === 'nickname-pending' || + derived.phase.kind === 'nickname-rejected' || + (derived.phase.kind === 'deadline-near' && + derived.phase.remaining.includes('nickname')) + ) { + return getNicknameFormatMessage(); + } + if ( + derived.phase.kind === 'photo-pending' || + derived.phase.kind === 'photo-rejected' || + (derived.phase.kind === 'deadline-near' && + derived.phase.remaining.includes('photo')) + ) { + return getHeartGuideMessage(); + } + return getGenericHelpMessage(nickname, session); + case 'discord-question': + return getDiscordMessage(nickname, session); + case 'discord-verify-question': + return getDiscordMessage(nickname, session); + case 'age-question': + return getAgeRequirementMessage(); + case 'chitchat': + case 'off-topic-to-other': + return ''; + case 'unknown': + return getGenericHelpMessage(nickname, session); + } +} diff --git a/src/skills/onboarding/runtime/message-builder.ts b/src/skills/onboarding/runtime/message-builder.ts new file mode 100644 index 0000000..065e3b8 --- /dev/null +++ b/src/skills/onboarding/runtime/message-builder.ts @@ -0,0 +1,125 @@ +import type { AgentRequest, ConversationMessage } from '@agent/types'; +import type { GarnetClient } from '@common/garnet'; +import { getConversationSummaryKey } from '@conv/conversation-store'; +import type { ChatMessage } from '@llm/types'; +import { getRequestLogger } from '@obs/context'; +import type { RoomContextService } from '@room/room-context-service'; +import { meetsTrustTier } from '../../../eval/governance/trust.js'; +import { + getAgentTrustMinTier, + getAgentUserMemoryEnabled, + getPromptVariantId, +} from '../../../config.js'; +import { + loadPromptVariantManifest, + PromptBuilder, +} from '../prompts/builder.js'; +import type { OnboardingBlackboardStore } from '../state/blackboard.js'; +import type { UserMemoryStore } from '../state/user-memory.js'; +import type { DerivedState } from '../state/onboarding-state.js'; +import type { OnboardingSession, PromptContext } from '../types.js'; + +interface MessageBuilderDeps { + blackboardStore: OnboardingBlackboardStore; + garnetClient: GarnetClient; + roomContextService?: RoomContextService | undefined; + userMemoryStore: UserMemoryStore; +} + +interface MessageBuilderInput { + conversationHistory: ConversationMessage[]; + derived: DerivedState; + request: AgentRequest; + session: OnboardingSession | null; +} + +function buildPromptUserMessage( + request: AgentRequest, + derived: DerivedState, +): string { + switch (request.trigger) { + case 'nickname-invalid': + return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`; + case 'photo-failed': + return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.'; + case 'progress-update': + return '현재 새내기 등록 진행상황만 짧게 알려줘.'; + case 'timeout-warning': + return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.'; + case 'completed': + return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.'; + case 'user-message': + if (derived.intent.kind === 'status-question') { + return '완료된 것은 빼고 남은 절차만 짧게 말해줘.'; + } + return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.'; + default: + return '지금 필요한 온보딩 안내만 짧게 도와줘.'; + } +} + +async function loadConversationSummary( + garnetClient: GarnetClient, + chatroomId: string, + userId: string, +): Promise { + const key = getConversationSummaryKey(chatroomId, userId); + return (await garnetClient.getString(key)) ?? undefined; +} + +export async function buildLlmMessages( + deps: MessageBuilderDeps, + input: MessageBuilderInput, +): Promise { + const logger = getRequestLogger(); + const { conversationHistory, derived, request, session } = input; + const roomContextPromise = + request.trigger === 'user-message' && deps.roomContextService + ? deps.roomContextService + .getRecentMessages(request.chatroomId) + .catch((error) => { + logger.warn( + { err: error }, + '[OnboardingSkill] Failed to fetch room context', + ); + return undefined; + }) + : Promise.resolve(undefined); + const userProfilePromise = getAgentUserMemoryEnabled() + ? deps.userMemoryStore.load(request.userId).catch((error) => { + logger.warn( + { err: error }, + '[OnboardingSkill] Failed to load user memory', + ); + return null; + }) + : Promise.resolve(null); + const [blackboard, summary, roomContext, userProfile] = await Promise.all([ + deps.blackboardStore.load(request.chatroomId, request.userId), + loadConversationSummary(deps.garnetClient, request.chatroomId, request.userId), + roomContextPromise, + userProfilePromise, + ]); + + const context: PromptContext = { + blackboard, + conversationHistory, + roomContext, + summary, + userMessage: buildPromptUserMessage(request, derived), + userProfile, + }; + + const builder = PromptBuilder.forDerived(derived, session, context); + const promptVariantId = getPromptVariantId(); + if (!promptVariantId) { + return builder.build(); + } + + const manifest = loadPromptVariantManifest(promptVariantId); + if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) { + return builder.build(); + } + + return builder.applyVariant(manifest).build(); +} diff --git a/src/skills/onboarding/runtime/state-updater.ts b/src/skills/onboarding/runtime/state-updater.ts new file mode 100644 index 0000000..ecec19f --- /dev/null +++ b/src/skills/onboarding/runtime/state-updater.ts @@ -0,0 +1,63 @@ +import type { AgentRequest } from '@agent/types'; +import type { DerivedState } from '../state/onboarding-state.js'; +import type { OnboardingBlackboardStore } from '../state/blackboard.js'; + +export async function rememberTurn( + blackboardStore: OnboardingBlackboardStore, + request: AgentRequest, + derived: DerivedState, +): Promise { + const now = new Date().toISOString(); + const userMessage = request.userMessage?.trim(); + + await blackboardStore.update(request.chatroomId, request.userId, (current) => { + const sameIntentStreak = + current.lastIntent === derived.intent.kind + ? (current.sameIntentStreak ?? 1) + 1 + : 1; + const openQuestions = + userMessage && + (derived.intent.kind === 'help-how' || + derived.intent.kind === 'help-where' || + derived.intent.kind === 'nickname-question' || + derived.intent.kind === 'photo-question' || + derived.intent.kind === 'discord-question' || + derived.intent.kind === 'discord-verify-question') + ? [...current.openQuestions, userMessage].slice(-3) + : current.openQuestions; + + const recentNicknameAttempts = + request.trigger === 'nickname-invalid' || + derived.intent.kind === 'nickname-question' + ? Array.from( + new Set([...current.recentNicknameAttempts, request.nickname]), + ).slice(-5) + : current.recentNicknameAttempts; + + return { + ...current, + flaggedIssues: + derived.phase.kind === 'nickname-rejected' + ? derived.phase.issues.map((issue) => issue.message) + : derived.phase.kind === 'completed' + ? [] + : current.flaggedIssues, + lastIntent: derived.intent.kind, + lastIntentAt: now, + openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions, + recentNicknameAttempts, + sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak, + toneMemory: { + ...current.toneMemory, + greetedAt: + derived.intent.kind === 'greeting' + ? now + : current.toneMemory.greetedAt, + thankedAt: + derived.intent.kind === 'acknowledgement' + ? now + : current.toneMemory.thankedAt, + }, + }; + }); +} diff --git a/src/skills/onboarding/runtime/text-generator.ts b/src/skills/onboarding/runtime/text-generator.ts new file mode 100644 index 0000000..fe8911b --- /dev/null +++ b/src/skills/onboarding/runtime/text-generator.ts @@ -0,0 +1,101 @@ +import type { AgentRequest } from '@agent/types'; +import type { ChatMessage } from '@llm/types'; +import type { Tool, ToolContext } from '@tools/tool'; +import { getLlmTemperature } from '../../../config.js'; +import type { DerivedState } from '../state/onboarding-state.js'; +import type { OnboardingOrchestrator } from './orchestrator.js'; +import { validateOnboardingResponse } from './validator.js'; + +export type LlmTextResult = + | { status: 'success'; text: string } + | { status: 'no_response' } + | { status: 'failed' }; + +export async function generateLlmText( + orchestrator: OnboardingOrchestrator, + messages: ChatMessage[], + derived: DerivedState, + options?: { + allowShortResponse?: boolean; + maxTokens?: number; + request: AgentRequest; + retryOnLowQuality?: boolean; + temperature?: number; + toolContext?: ToolContext | undefined; + tools?: readonly Tool[] | undefined; + }, +): Promise { + const attempts: Array<{ + maxTokens?: number; + temperature?: number; + }> = []; + + const firstAttempt: { + maxTokens?: number; + temperature?: number; + } = {}; + if (options?.maxTokens !== undefined) { + firstAttempt.maxTokens = options.maxTokens; + } + if (options?.temperature !== undefined) { + firstAttempt.temperature = options.temperature; + } + attempts.push(firstAttempt); + + if (options?.retryOnLowQuality) { + const retryAttempt: { + maxTokens?: number; + temperature?: number; + } = { + temperature: getLlmTemperature() + 0.1, + }; + if (options.maxTokens !== undefined) { + retryAttempt.maxTokens = options.maxTokens; + } + attempts.push(retryAttempt); + } + + for (const attempt of attempts) { + const requestOptions: { + maxTokens?: number; + temperature?: number; + } = {}; + if (attempt.maxTokens !== undefined) { + requestOptions.maxTokens = attempt.maxTokens; + } + if (attempt.temperature !== undefined) { + requestOptions.temperature = attempt.temperature; + } + + const toolContext = options?.toolContext ?? {}; + const llmText = await orchestrator.run(derived, messages, { + maxTokens: requestOptions.maxTokens, + request: options?.request, + temperature: requestOptions.temperature, + toolContext, + tools: options?.tools, + }); + + if (!llmText) { + continue; + } + + const validation = validateOnboardingResponse(llmText, derived, { + allowShortResponse: options?.allowShortResponse, + }); + + if (validation.text.includes('[NO_RESPONSE]')) { + return { status: 'no_response' }; + } + if (validation.status === 'reject') { + continue; + } + + return { + status: 'success', + text: validation.text, + }; + } + + return { status: 'failed' }; +} diff --git a/src/skills/onboarding/runtime/tool-selector.ts b/src/skills/onboarding/runtime/tool-selector.ts new file mode 100644 index 0000000..a6cc3a7 --- /dev/null +++ b/src/skills/onboarding/runtime/tool-selector.ts @@ -0,0 +1,61 @@ +import type { Tool } from '@tools/tool'; +import { createToolSearchTool } from '@tools/tool-search'; +import type { ToolRegistry } from '@tools/registry'; +import { + getAgentToolSearchEnabled, + getAgentToolSearchThreshold, +} from '../../../config.js'; +import type { DerivedState } from '../state/onboarding-state.js'; + +export function selectToolsForDerived( + derived: DerivedState, + tools: readonly Tool[], + toolRegistry: ToolRegistry, +): readonly Tool[] { + const maybeUseToolSearch = ( + selectedTools: readonly Tool[], + ): readonly Tool[] => { + if ( + getAgentToolSearchEnabled() && + selectedTools.length >= getAgentToolSearchThreshold() + ) { + return [createToolSearchTool(toolRegistry)]; + } + return selectedTools; + }; + + if (derived.intent.kind === 'discord-verify-question') { + return maybeUseToolSearch( + tools.filter((tool) => tool.name === 'verify-discord-nickname'), + ); + } + + if (derived.phase.kind === 'nickname-rejected') { + return maybeUseToolSearch( + tools.filter( + (tool) => + tool.name === 'validate-nickname' || + tool.name === 'lookup-community-rule' || + tool.name === 'check-onboarding-progress', + ), + ); + } + + if (derived.intent.kind === 'status-question') { + return maybeUseToolSearch( + tools.filter((tool) => tool.name === 'check-onboarding-progress'), + ); + } + + if (derived.intent.kind === 'discord-question') { + return maybeUseToolSearch( + tools.filter( + (tool) => + tool.name === 'lookup-community-rule' || + tool.name === 'verify-discord-nickname', + ), + ); + } + + return maybeUseToolSearch(tools); +} diff --git a/src/skills/onboarding/skill.ts b/src/skills/onboarding/skill.ts index 25136d8..63f9713 100644 --- a/src/skills/onboarding/skill.ts +++ b/src/skills/onboarding/skill.ts @@ -1,38 +1,22 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types'; -import { getConversationSummaryKey } from '@conv/conversation-store'; import type { GarnetClient } from '@common/garnet'; import type { NatsClient } from '@common/nats'; import { getContext, getRequestLogger } from '@obs/context'; import type { LlmClient } from '@llm/llm-client'; -import type { ChatMessage } from '@llm/types'; import type { RoomContextService } from '@room/room-context-service'; import type { Skill } from '@skills/skill'; import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus'; import { anonymizeProcessTrace } from '@eval/anonymize'; -import { meetsTrustTier } from '../../eval/governance/trust.js'; import { createOnboardingToolRegistry } from './tools.js'; import type { ToolRegistry } from '@tools/registry'; import type { Tool, ToolContext } from '@tools/tool'; -import { createToolSearchTool } from '@tools/tool-search'; import { - loadPromptVariantManifest, - PromptBuilder, -} from './prompts/builder.js'; -import { - getAcknowledgementMessage, - getAgeRequirementMessage, getCompletedMessage, - getDiscordMessage, - getGreetingMessage, getGenericHelpMessage, - getHeartGuideMessage, - getHeartLocationMessage, getNicknameInvalidMessage, - getNicknameFormatMessage, getOnboardingTemplate, getPhotoFailedMessage, getProgressMessage, - getRemainingStatusMessage, getTimeoutWarningMessage, } from './onboarding-templates.js'; import { @@ -41,13 +25,8 @@ import { getAgentAbstentionThreshold, getAgentRouterMode, getAgentRouterWeightsPath, - getAgentTrustMinTier, - getAgentToolSearchEnabled, - getAgentToolSearchThreshold, getAgentUserMemoryEnabled, getOnboardingSessionKeyPrefix, - getLlmTemperature, - getPromptVariantId, } from '../../config.js'; import { OnboardingBlackboardStore } from './state/blackboard.js'; import { UserMemoryStore } from './state/user-memory.js'; @@ -74,21 +53,24 @@ import { runWithProcessTrace, } from './runtime/trace.js'; import { validateOnboardingResponse } from './runtime/validator.js'; +import { getIntentFallbackText } from './runtime/fallback-text-resolver.js'; +import { buildLlmMessages } from './runtime/message-builder.js'; +import { + generateLlmText, + type LlmTextResult, +} from './runtime/text-generator.js'; +import { selectToolsForDerived } from './runtime/tool-selector.js'; +import { rememberTurn } from './runtime/state-updater.js'; import { normalizeOnboardingSession } from './session.js'; import type { Blackboard, OnboardingSession, - PromptContext, StoredOnboardingSession, } from './types.js'; -type LlmTextResult = - | { status: 'success'; text: string } - | { status: 'no_response' } - | { status: 'failed' }; - export class OnboardingSkill implements Skill { readonly name = 'onboarding'; + readonly priority = 70; private readonly classifier: OnboardingIntentClassifier; private readonly blackboardStore: OnboardingBlackboardStore; private readonly orchestrator: OnboardingOrchestrator; @@ -357,9 +339,26 @@ export class OnboardingSkill implements Skill { conversationHistory: ConversationMessage[], derived: DerivedState, ): Promise { - const messages = await this.buildLlmMessages(request, session, conversationHistory, derived); + if (!this.llmClient.hasEndpoints()) { + return { status: 'failed' }; + } - return this.generateLlmText(messages, derived, { + const messages = await buildLlmMessages( + { + blackboardStore: this.blackboardStore, + garnetClient: this.garnetClient, + roomContextService: this.roomContextService, + userMemoryStore: this.userMemoryStore, + }, + { + conversationHistory, + derived, + request, + session, + }, + ); + + return generateLlmText(this.orchestrator, messages, derived, { allowShortResponse: derived.allowShortResponse, maxTokens: derived.suggestedMaxTokens, request, @@ -368,135 +367,12 @@ export class OnboardingSkill implements Skill { !derived.allowShortResponse && derived.responseMode !== 'llm-with-tools', toolContext: this.buildToolContext(request, session), - tools: this.selectToolsForDerived(derived), + tools: selectToolsForDerived(derived, this.tools, this.toolRegistry), }); } - private async buildLlmMessages( - request: AgentRequest, - session: OnboardingSession | null, - conversationHistory: ConversationMessage[], - derived: DerivedState, - ): Promise { - const logger = getRequestLogger(); - const roomContextPromise = - request.trigger === 'user-message' && this.roomContextService - ? this.roomContextService.getRecentMessages(request.chatroomId).catch((error) => { - logger.warn({ err: error }, '[OnboardingSkill] Failed to fetch room context'); - return undefined; - }) - : Promise.resolve(undefined); - const userProfilePromise = getAgentUserMemoryEnabled() - ? this.userMemoryStore.load(request.userId).catch((error) => { - logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory'); - return null; - }) - : Promise.resolve(null); - const [blackboard, summary, roomContext, userProfile] = await Promise.all([ - this.blackboardStore.load(request.chatroomId, request.userId), - this.loadConversationSummary(request.chatroomId, request.userId), - roomContextPromise, - userProfilePromise, - ]); - - const context: PromptContext = { - blackboard, - conversationHistory, - roomContext, - summary, - userMessage: this.buildPromptUserMessage(request, derived), - userProfile, - }; - - const builder = PromptBuilder.forDerived(derived, session, context); - const promptVariantId = getPromptVariantId(); - if (!promptVariantId) { - return builder.build(); - } - - const manifest = loadPromptVariantManifest(promptVariantId); - if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) { - return builder.build(); - } - - return builder.applyVariant(manifest).build(); - } - - private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string { - switch (request.trigger) { - case 'nickname-invalid': - return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`; - case 'photo-failed': - return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.'; - case 'progress-update': - return '현재 새내기 등록 진행상황만 짧게 알려줘.'; - case 'timeout-warning': - return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.'; - case 'completed': - return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.'; - case 'user-message': - if (derived.intent.kind === 'status-question') { - return '완료된 것은 빼고 남은 절차만 짧게 말해줘.'; - } - return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.'; - default: - return '지금 필요한 온보딩 안내만 짧게 도와줘.'; - } - } - - private async loadConversationSummary( - chatroomId: string, - userId: string, - ): Promise { - const key = getConversationSummaryKey(chatroomId, userId); - return (await this.garnetClient.getString(key)) ?? undefined; - } - private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise { - const now = new Date().toISOString(); - const userMessage = request.userMessage?.trim(); - - await this.blackboardStore.update(request.chatroomId, request.userId, (current) => { - const sameIntentStreak = - current.lastIntent === derived.intent.kind - ? (current.sameIntentStreak ?? 1) + 1 - : 1; - const openQuestions = - userMessage && - (derived.intent.kind === 'help-how' || - derived.intent.kind === 'help-where' || - derived.intent.kind === 'nickname-question' || - derived.intent.kind === 'photo-question' || - derived.intent.kind === 'discord-question' || - derived.intent.kind === 'discord-verify-question') - ? [...current.openQuestions, userMessage].slice(-3) - : current.openQuestions; - - const recentNicknameAttempts = - request.trigger === 'nickname-invalid' || derived.intent.kind === 'nickname-question' - ? Array.from(new Set([...current.recentNicknameAttempts, request.nickname])).slice(-5) - : current.recentNicknameAttempts; - - return { - ...current, - flaggedIssues: - derived.phase.kind === 'nickname-rejected' - ? derived.phase.issues.map((issue) => issue.message) - : derived.phase.kind === 'completed' - ? [] - : current.flaggedIssues, - lastIntent: derived.intent.kind, - lastIntentAt: now, - openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions, - recentNicknameAttempts, - sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak, - toneMemory: { - ...current.toneMemory, - greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt, - thankedAt: derived.intent.kind === 'acknowledgement' ? now : current.toneMemory.thankedAt, - }, - }; - }); + return rememberTurn(this.blackboardStore, request, derived); } private async observeUserMemory( @@ -552,163 +428,11 @@ export class OnboardingSkill implements Skill { default: return this.makeTemplateResponse( request.requestId, - this.getIntentFallbackText(request.nickname, session, intent, derived), + getIntentFallbackText(request.nickname, session, intent, derived), ); } } - private getIntentFallbackText( - nickname: string, - session: OnboardingSession | null, - intent: Intent, - derived: DerivedState, - ): string { - switch (intent.kind) { - case 'greeting': - return getGreetingMessage(); - case 'acknowledgement': - return getAcknowledgementMessage(); - case 'status-question': - return getRemainingStatusMessage(session); - case 'nickname-question': - return derived.phase.kind === 'nickname-rejected' - ? getNicknameInvalidMessage(nickname, session) - : getNicknameFormatMessage(); - case 'photo-question': - return derived.phase.kind === 'photo-rejected' - ? getPhotoFailedMessage(nickname) - : getHeartGuideMessage(); - case 'help-where': - if ( - derived.phase.kind === 'photo-pending' || - derived.phase.kind === 'photo-rejected' || - (derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo')) - ) { - return getHeartLocationMessage(); - } - return getGenericHelpMessage(nickname, session); - case 'help-how': - if ( - derived.phase.kind === 'nickname-pending' || - derived.phase.kind === 'nickname-rejected' || - (derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('nickname')) - ) { - return getNicknameFormatMessage(); - } - if ( - derived.phase.kind === 'photo-pending' || - derived.phase.kind === 'photo-rejected' || - (derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo')) - ) { - return getHeartGuideMessage(); - } - return getGenericHelpMessage(nickname, session); - case 'discord-question': - return getDiscordMessage(nickname, session); - case 'discord-verify-question': - return getDiscordMessage(nickname, session); - case 'age-question': - return getAgeRequirementMessage(); - case 'chitchat': - case 'off-topic-to-other': - return ''; - case 'unknown': - return getGenericHelpMessage(nickname, session); - } - } - - private async generateLlmText( - messages: ChatMessage[], - derived: DerivedState, - options?: { - allowShortResponse?: boolean; - maxTokens?: number; - request: AgentRequest; - retryOnLowQuality?: boolean; - temperature?: number; - toolContext?: ToolContext | undefined; - tools?: readonly Tool[] | undefined; - }, - ): Promise { - if (!this.llmClient.hasEndpoints()) { - return { status: 'failed' }; - } - - const attempts: Array<{ - maxTokens?: number; - temperature?: number; - }> = []; - - const firstAttempt: { - maxTokens?: number; - temperature?: number; - } = {}; - if (options?.maxTokens !== undefined) { - firstAttempt.maxTokens = options.maxTokens; - } - if (options?.temperature !== undefined) { - firstAttempt.temperature = options.temperature; - } - attempts.push(firstAttempt); - - if (options?.retryOnLowQuality) { - const retryAttempt: { - maxTokens?: number; - temperature?: number; - } = { - temperature: getLlmTemperature() + 0.1, - }; - if (options.maxTokens !== undefined) { - retryAttempt.maxTokens = options.maxTokens; - } - attempts.push(retryAttempt); - } - - for (const attempt of attempts) { - const requestOptions: { - maxTokens?: number; - temperature?: number; - } = {}; - if (attempt.maxTokens !== undefined) { - requestOptions.maxTokens = attempt.maxTokens; - } - if (attempt.temperature !== undefined) { - requestOptions.temperature = attempt.temperature; - } - - const toolContext = options?.toolContext ?? {}; - const llmText = await this.orchestrator.run(derived, messages, { - maxTokens: requestOptions.maxTokens, - request: options?.request, - temperature: requestOptions.temperature, - toolContext, - tools: options?.tools, - }); - - if (!llmText) { - continue; - } - - const validation = validateOnboardingResponse(llmText, derived, { - allowShortResponse: options?.allowShortResponse, - }); - - if (validation.text.includes('[NO_RESPONSE]')) { - return { status: 'no_response' }; - } - if (validation.status === 'reject') { - continue; - } - - return { - status: 'success', - text: validation.text, - }; - } - - return { status: 'failed' }; - } - private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext { return { chatroomId: request.chatroomId, @@ -721,53 +445,6 @@ export class OnboardingSkill implements Skill { }; } - private selectToolsForDerived(derived: DerivedState): readonly Tool[] { - const maybeUseToolSearch = ( - tools: readonly Tool[], - ): readonly Tool[] => { - if ( - getAgentToolSearchEnabled() && - tools.length >= getAgentToolSearchThreshold() - ) { - return [createToolSearchTool(this.toolRegistry)]; - } - return tools; - }; - - if (derived.intent.kind === 'discord-verify-question') { - return maybeUseToolSearch( - this.tools.filter((tool) => tool.name === 'verify-discord-nickname'), - ); - } - - if (derived.phase.kind === 'nickname-rejected') { - return maybeUseToolSearch( - this.tools.filter( - (tool) => - tool.name === 'validate-nickname' || - tool.name === 'lookup-community-rule' || - tool.name === 'check-onboarding-progress', - ), - ); - } - - if (derived.intent.kind === 'status-question') { - return maybeUseToolSearch( - this.tools.filter((tool) => tool.name === 'check-onboarding-progress'), - ); - } - - if (derived.intent.kind === 'discord-question') { - return maybeUseToolSearch( - this.tools.filter( - (tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname', - ), - ); - } - - return maybeUseToolSearch(this.tools); - } - private makeTemplateResponse(requestId: string, text: string): AgentResponse { return { requestId, diff --git a/src/skills/skill-loader.test.ts b/src/skills/skill-loader.test.ts index 96ef751..9d75bcf 100644 --- a/src/skills/skill-loader.test.ts +++ b/src/skills/skill-loader.test.ts @@ -18,6 +18,7 @@ function makeTempSkillRoot(): string { "name: demo", "description: Demo skill.", "version: 1", + "priority: 65", "intents: [hello, help]", "triggers: [user-message]", "tools: [demo-tool]", @@ -41,6 +42,7 @@ describe("SkillRegistry", () => { name: "demo", description: "Demo skill.", intents: ["hello", "help"], + priority: 65, tools: ["demo-tool"], trustTier: "T2", version: 1, @@ -49,6 +51,27 @@ describe("SkillRegistry", () => { expect(registry.loadResourceSync("demo", "guide")).toBe("guide body"); }); + it("defaults priority to 50 when omitted", () => { + const root = mkdtempSync(join(tmpdir(), "lolympus-skills-priority-")); + const skillDir = join(root, "default-priority"); + mkdirSync(skillDir, { recursive: true }); + writeFileSync( + join(skillDir, "SKILL.md"), + [ + "---", + "name: default-priority", + "description: Uses default priority.", + "version: 1", + "trustTier: T2", + "---", + "", + "body", + ].join("\n"), + ); + + expect(SkillRegistry.load(root).get("default-priority")?.priority).toBe(50); + }); + it("ignores invalid frontmatter", () => { const root = mkdtempSync(join(tmpdir(), "lolympus-skills-invalid-")); const skillDir = join(root, "bad"); diff --git a/src/skills/skill-loader.ts b/src/skills/skill-loader.ts index 3609c69..70d3863 100644 --- a/src/skills/skill-loader.ts +++ b/src/skills/skill-loader.ts @@ -1,5 +1,6 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; +import { getLogger } from "@obs/logger"; export type TrustTier = "T1" | "T2" | "T3" | "T4"; @@ -8,6 +9,7 @@ export interface SkillManifest { description: string; intents: string[]; name: string; + priority: number; resources: Record; tools: string[]; triggers: string[]; @@ -123,6 +125,15 @@ function isTrustTier(value: unknown): value is TrustTier { return value === "T1" || value === "T2" || value === "T3" || value === "T4"; } +function isPriority(value: unknown): value is number { + return ( + typeof value === "number" && + Number.isInteger(value) && + value >= 0 && + value <= 100 + ); +} + function parseSkillFile(path: string): ParsedSkillFile | null { const raw = readFileSync(path, "utf8"); const parsed = parseFrontmatter(raw); @@ -133,6 +144,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null { const skillDir = dirname(path); const name = parsed.fields["name"]; const description = parsed.fields["description"]; + const priority = parsed.fields["priority"] ?? 50; const version = parsed.fields["version"]; const trustTier = parsed.fields["trustTier"]; @@ -141,6 +153,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null { !name.trim() || typeof description !== "string" || !description.trim() || + !isPriority(priority) || typeof version !== "number" || !isTrustTier(trustTier) ) { @@ -154,6 +167,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null { description, intents: readStringArray(parsed.fields, "intents"), name, + priority, resources: readResources(parsed.fields, skillDir), tools: readStringArray(parsed.fields, "tools"), triggers: readStringArray(parsed.fields, "triggers"), @@ -200,6 +214,13 @@ export class SkillRegistry { for (const item of parsed) { if (seen.has(item.manifest.name)) { + getLogger().warn( + { + path: item.manifest.bodyPath, + skill: item.manifest.name, + }, + "[SkillRegistry] Duplicate skill manifest ignored", + ); continue; } seen.add(item.manifest.name); diff --git a/src/skills/skill.ts b/src/skills/skill.ts index f33a39b..f1123ce 100644 --- a/src/skills/skill.ts +++ b/src/skills/skill.ts @@ -6,6 +6,7 @@ import type { export interface Skill { readonly name: string; + readonly priority: number; canHandle(request: AgentRequest): boolean; handle( request: AgentRequest, diff --git a/src/tools/moderation/lookup-rule.test.ts b/src/tools/moderation/lookup-rule.test.ts new file mode 100644 index 0000000..488b78c --- /dev/null +++ b/src/tools/moderation/lookup-rule.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { lookupRuleTool } from "./lookup-rule.js"; + +describe("lookupRuleTool", () => { + it("returns the matching community rule section", async () => { + const result = await lookupRuleTool.execute({ topic: "ads" }, {}); + + expect(result.source).toBe("community-rules"); + expect(result.text).toContain("광고"); + expect(result.text).toContain("홍보"); + }); + + it("validates topic enum values", () => { + expect(() => + lookupRuleTool.argsSchema.parse({ topic: "unknown" }), + ).toThrow(); + }); +}); diff --git a/src/tools/moderation/lookup-rule.ts b/src/tools/moderation/lookup-rule.ts new file mode 100644 index 0000000..3e7a92d --- /dev/null +++ b/src/tools/moderation/lookup-rule.ts @@ -0,0 +1,55 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; +import type { Tool } from "../tool.js"; + +const argsSchema = z.object({ + topic: z.enum([ + "ads", + "appeal", + "minor-safety", + "off-topic", + "privacy", + "profanity", + "report", + "sanctions", + "spam", + ]), +}); + +const resultSchema = z.object({ + topic: argsSchema.shape.topic, + text: z.string(), + source: z.literal("community-rules"), +}); + +type RuleTopic = z.infer["topic"]; + +const RULES_PATH = fileURLToPath( + new URL("../../skills/moderation/resources/community-rules.md", import.meta.url), +); +const RULES_TEXT = readFileSync(RULES_PATH, "utf8"); + +function extractRuleSection(topic: RuleTopic): string { + const pattern = new RegExp(`^## ${topic}\\n([\\s\\S]*?)(?=\\n## |$)`, "mu"); + const match = RULES_TEXT.match(pattern); + return match?.[1]?.trim() || ""; +} + +export const lookupRuleTool: Tool< + z.infer, + z.infer +> = { + name: "lookup-rule", + description: + "광고, 도배, 욕설, 신고, 제재, 이의 제기, 개인정보 같은 moderation 규칙을 조회한다.", + argsSchema, + resultSchema, + execute(args) { + return Promise.resolve({ + topic: args.topic, + text: extractRuleSection(args.topic), + source: "community-rules", + }); + }, +};