Iterate skill SSOT routing and eval support

Add skill priority routing and duplicate manifest handling.

Split eval golden sets by skill and add moderation rule lookup coverage.

Move onboarding runtime helpers out of the main skill class.
This commit is contained in:
dahyeon committed 2026-05-18 09:19:17 -07:00
1 parent fb189330da
commit 12e731a428
35 files changed
+1260 -500

No files matched your search

+53
View File
@@ -0,0 +1,53 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
import { describe, expect, it } from "vitest";
import type { Skill } from "@skills/skill";
import { selectSkillForRequest } from "./agent.js";
function makeRequest(skill = "shared"): AgentRequest {
return {
chatroomId: "room-1",
nickname: "tester",
requestId: "request-1",
skill,
trigger: "user-message",
userId: "user-1",
};
}
function makeSkill(name: string, priority: number, canHandle = true): Skill {
return {
name,
priority,
canHandle: () => canHandle,
handle: (
request: AgentRequest,
_conversationHistory: ConversationMessage[],
): Promise<AgentResponse> =>
Promise.resolve({
confidence: 1,
requestId: request.requestId,
source: "template",
text: name,
}),
};
}
describe("selectSkillForRequest", () => {
it("selects the highest-priority matching skill", () => {
const request = makeRequest();
const selected = selectSkillForRequest(
[
makeSkill("low", 10),
makeSkill("high", 90),
makeSkill("disabled", 100, false),
],
request,
);
expect(selected?.name).toBe("high");
});
it("returns null when no skill can handle the request", () => {
expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull();
});
});
+12 -1
View File
@@ -25,6 +25,17 @@ function isValidRequest(data: unknown): data is AgentRequest {
);
}
export function selectSkillForRequest(
skills: readonly Skill[],
request: AgentRequest,
): Skill | null {
return (
skills
.filter((candidate) => candidate.canHandle(request))
.sort((left, right) => right.priority - left.priority)[0] ?? null
);
}
export class Agent {
private readonly skills: Skill[] = [];
private readonly logger = getLogger();
@@ -96,7 +107,7 @@ export class Agent {
"[Agent] Request received",
);
const skill = this.skills.find((candidate) => candidate.canHandle(request));
const skill = selectSkillForRequest(this.skills, request);
if (!skill) {
logger.info("[Agent] No skill found for request");
const response: AgentResponse = {
+3 -1
View File
@@ -14,6 +14,7 @@ import {
interface CaptureOptions {
limit?: number | undefined;
outputPath?: string | undefined;
skill?: string | undefined;
}
function asRecord(value: unknown): Record<string, unknown> {
@@ -122,7 +123,8 @@ export async function captureGoldenSetScaffold(
const outputPath = resolve(
process.cwd(),
options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl",
options?.outputPath ??
`src/eval/golden/${options?.skill ?? "onboarding"}/capture.jsonl`,
);
await mkdir(dirname(outputPath), { recursive: true });
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
+13 -5
View File
@@ -319,39 +319,46 @@ async function main(): Promise<void> {
parsePositiveInteger(getFlagString(flags, "limit")) ??
parsePositiveInteger(positionals[0]),
outputPath: getFlagString(flags, "output") ?? positionals[1],
skill: getFlagString(flags, "skill"),
});
logger.info({ outputPath }, '[eval] "capture" completed');
return;
}
if (command === "run") {
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
const { loadEvalCases, resolveEvalSetPath, runEval } = await import(
"./runner.js"
);
const { publishEvalReportIfPossible } = await import(
"./runner.js"
);
const setName = getFlagString(flags, "set") ?? "golden";
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
const setName = getFlagString(flags, "set") ?? "onboarding/cases";
const resolvedSet = resolveEvalSetPath(setName);
const skill = getFlagString(flags, "skill") ?? resolvedSet.skill;
const inputPath = getFlagString(flags, "input") ?? resolvedSet.path;
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
if (hasFlag(flags, "dry-run")) {
const cases = await loadEvalCases(inputPath, { limit });
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
console.info(
`[eval] dry-run skill=${skill} parsed ${cases.length} cases from ${inputPath}`,
);
return;
}
const { report, reportPath } = await runOnboardingEval({
const { report, reportPath } = await runEval({
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
inputPath,
limit,
outputPath: getFlagString(flags, "output"),
setName,
skill,
});
const publishedReport = hasFlag(flags, "no-publish-report")
? false
: await publishEvalReportIfPossible(report);
printRunSummary(report, reportPath);
console.info(`[eval] skill=${skill}`);
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
return;
}
@@ -414,6 +421,7 @@ async function main(): Promise<void> {
const result = await appendCandidatesToJsonl(
candidates,
getFlagString(flags, "output"),
getFlagString(flags, "skill"),
);
console.info(
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
Whitespace-only changes.
Whitespace-only changes.
+5
View File
@@ -0,0 +1,5 @@
{"id":"moderation_rule_ads","name":"moderation:rule-question:ads","input":{"request":{"requestId":"mod_req_ads","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_1","nickname":"문의자","trigger":"user-message","userMessage":"광고 올려도 돼요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["광고","홍보","허가","제한"],"mustNotInclude":["무조건 가능","바로 올려도"]}}
{"id":"moderation_rule_profanity","name":"moderation:rule-question:profanity","input":{"request":{"requestId":"mod_req_profanity","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_2","nickname":"문의자","trigger":"user-message","userMessage":"욕설하면 어떻게 돼요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["욕설","금지","운영진"],"mustNotInclude":["괜찮아요","상관없"]}}
{"id":"moderation_rule_spam","name":"moderation:rule-question:spam","input":{"request":{"requestId":"mod_req_spam","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_3","nickname":"문의자","trigger":"user-message","userMessage":"도배 기준이 뭐예요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["도배","반복","제재"],"mustNotInclude":["기준 없음","상관없"]}}
{"id":"moderation_report_profanity","name":"moderation:violation-report:profanity","input":{"request":{"requestId":"mod_req_report","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_4","nickname":"제보자","trigger":"user-message","userMessage":"어떤 분이 욕했어요 신고하고 싶어요"}},"expected":{"intent":"violation-report","mustIncludeAny":["관련 메시지","상황","운영진"],"mustNotInclude":["제가 처리","제재 확정"]}}
{"id":"moderation_appeal_sanction","name":"moderation:appeal:sanction","input":{"request":{"requestId":"mod_req_appeal","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_5","nickname":"소명자","trigger":"user-message","userMessage":"정지당했는데 억울해요"}},"expected":{"intent":"appeal","mustIncludeAny":["이의 제기","소명","운영진"],"mustNotInclude":["해제됐","풀렸"]}}
File renamed without changes.
+107
View File
@@ -10,6 +10,113 @@ function round(value: number): number {
return Number(value.toFixed(3));
}
export function judgeRuleCase(
expected: EvalCaseExpected,
responseText: string,
actual: {
intent: string;
responseMode?: string | undefined;
},
): RuleJudgeResult {
const checks: RuleJudgeCheck[] = [];
checks.push({
name: "validator",
passed: true,
skipped: true,
detail: "not configured for this skill",
});
const expectedIntent = expected.intent?.trim();
if (expectedIntent) {
checks.push({
name: "intent",
passed: normalizeText(expectedIntent) === normalizeText(actual.intent),
skipped: false,
detail: `expected=${expectedIntent}, actual=${actual.intent}`,
});
} else {
checks.push({
name: "intent",
passed: true,
skipped: true,
});
}
const expectedResponseMode = expected.responseMode?.trim();
if (expectedResponseMode && actual.responseMode) {
checks.push({
name: "responseMode",
passed:
normalizeText(expectedResponseMode) ===
normalizeText(actual.responseMode),
skipped: false,
detail: `expected=${expectedResponseMode}, actual=${actual.responseMode}`,
});
} else {
checks.push({
name: "responseMode",
passed: true,
skipped: true,
});
}
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
if (mustIncludeAny.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustIncludeAny.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustIncludeAny",
passed: Boolean(matched),
skipped: false,
detail: matched
? `matched=${matched}`
: `expected one of: ${mustIncludeAny.join(", ")}`,
});
} else {
checks.push({
name: "mustIncludeAny",
passed: true,
skipped: true,
});
}
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
if (mustNotInclude.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustNotInclude.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustNotInclude",
passed: !matched,
skipped: false,
detail: matched ? `forbidden=${matched}` : undefined,
});
} else {
checks.push({
name: "mustNotInclude",
passed: true,
skipped: true,
});
}
const activeChecks = checks.filter((check) => !check.skipped);
const passedChecks = activeChecks.filter((check) => check.passed).length;
return {
passed: activeChecks.every((check) => check.passed),
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
checks,
validatorStatus: "pass",
validatorReasons: [],
};
}
export function judgeOnboardingRuleCase(
expected: EvalCaseExpected,
responseText: string,
+7 -3
View File
@@ -20,7 +20,6 @@ import { getEvalArchiveCollection } from "./archive.js";
import { queryArchivedReviewLabels } from "./review/label-archive.js";
import type { ReviewLabel } from "./review/queue.js";
const DEFAULT_CANDIDATE_PATH = "src/eval/golden/onboarding.candidate.jsonl";
const DAY_MS = 24 * 60 * 60 * 1000;
export interface PromotionCandidate {
@@ -47,6 +46,10 @@ function stableShortHash(value: string): string {
return createHash("sha256").update(value).digest("hex").slice(0, 8);
}
function defaultCandidatePathFor(skill: string): string {
return `src/eval/golden/${skill}/candidate.jsonl`;
}
function getLlmOverall(entry: EvalArchiveEntry): number | null {
if (entry.llmJudge.skipped) {
return null;
@@ -326,9 +329,10 @@ function extractExistingCaseIds(raw: string): Set<string> {
export async function appendCandidatesToJsonl(
candidates: PromotionCandidate[],
outputPath = DEFAULT_CANDIDATE_PATH,
outputPath?: string,
skill = "onboarding",
): Promise<{ appendedCount: number; outputPath: string }> {
const resolved = resolve(process.cwd(), outputPath);
const resolved = resolve(process.cwd(), outputPath ?? defaultCandidatePathFor(skill));
await mkdir(dirname(resolved), { recursive: true });
const existingRaw = existsSync(resolved)
? await readFile(resolved, "utf8")
+27 -1
View File
@@ -1,7 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { publishEvalReportIfPossible } from "./runner.js";
import { publishEvalReportIfPossible, resolveEvalSetPath } from "./runner.js";
import type { EvalReport } from "./types.js";
const natsMocks = vi.hoisted(() => ({
@@ -54,3 +54,29 @@ describe("publishEvalReportIfPossible", () => {
expect(natsMocks.close).toHaveBeenCalledTimes(1);
});
});
describe("resolveEvalSetPath", () => {
it("resolves skill/subset eval sets", () => {
const resolved = resolveEvalSetPath("moderation/cases");
expect(resolved).toMatchObject({
skill: "moderation",
subset: "cases",
});
expect(resolved.path).toContain("src/eval/golden/moderation/cases.jsonl");
});
it("maps legacy golden set to onboarding cases", () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
const resolved = resolveEvalSetPath("golden");
expect(resolved.skill).toBe("onboarding");
expect(resolved.subset).toBe("cases");
expect(warn).toHaveBeenCalledWith(
"[eval] --set=golden is deprecated; use --set=onboarding/cases",
);
warn.mockRestore();
});
});
+314 -122
View File
@@ -13,6 +13,9 @@ import { runWithContext } from "@obs/context";
import { getLogger } from "@obs/logger";
import type { RoomMessage } from "@room/types";
import { RoomContextService } from "@room/room-context-service";
import type { Skill } from "@skills/skill";
import { ModerationSkill } from "@skills/moderation/index";
import { classifyModerationIntent } from "@skills/moderation/intents";
import { OnboardingSkill } from "@skills/onboarding/index";
import {
ProcessTraceCollector,
@@ -29,7 +32,10 @@ import { noopShadowEvalPublisher } from "./shadow-bus.js";
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import { judgeProcessRule } from "./judges/process-judge.js";
import { judgeOnboardingRuleCase } from "./judges/rule-judge.js";
import {
judgeOnboardingRuleCase,
judgeRuleCase,
} from "./judges/rule-judge.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
@@ -44,11 +50,32 @@ import type {
} from "./types.js";
const DEFAULT_REPORTS_DIR = "eval-reports";
const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl";
const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl";
const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
export interface ResolvedEvalSetPath {
path: string;
skill: string;
subset: string;
}
interface SkillRunnerDeps {
garnet: GarnetClient;
llm: LlmClient;
room?: RoomContextService | undefined;
}
const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> = {
moderation: (deps) => new ModerationSkill(deps.llm),
onboarding: (deps) =>
new OnboardingSkill(
deps.garnet,
deps.llm,
deps.room,
noopShadowEvalPublisher,
),
};
class InMemoryGarnetClient extends GarnetClient {
private readonly hashStore = new Map<string, Record<string, string>>();
private readonly stringStore = new Map<string, string>();
@@ -127,6 +154,7 @@ export interface EvalRunOptions {
limit?: number | undefined;
outputPath?: string | undefined;
setName?: string | undefined;
skill?: string | undefined;
}
function round(value: number): number {
@@ -580,30 +608,72 @@ async function writeReport(
return resolved;
}
async function loadBaselineFile(): Promise<EvalBaselineFile | null> {
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
if (!existsSync(baselinePath)) {
function baselinePathFor(skill: string): string {
return `${DEFAULT_REPORTS_DIR}/baseline.${skill}.json`;
}
async function loadBaselineFile(skill: string): Promise<EvalBaselineFile | null> {
const baselinePath = resolve(process.cwd(), baselinePathFor(skill));
const legacyBaselinePath = resolve(process.cwd(), LEGACY_BASELINE_FILE);
const path = existsSync(baselinePath)
? baselinePath
: skill === "onboarding" && existsSync(legacyBaselinePath)
? legacyBaselinePath
: null;
if (!path) {
return null;
}
const raw = await readFile(baselinePath, "utf8");
const raw = await readFile(path, "utf8");
return JSON.parse(raw) as EvalBaselineFile;
}
export function resolveEvalSetPath(setNameOrPath = "golden"): string {
export function resolveEvalSetPath(
setNameOrPath = "onboarding/cases",
): ResolvedEvalSetPath {
if (setNameOrPath === "golden") {
return resolve(process.cwd(), GOLDEN_SET_PATH);
console.warn(
'[eval] --set=golden is deprecated; use --set=onboarding/cases',
);
return {
path: resolve(process.cwd(), "src/eval/golden/onboarding/cases.jsonl"),
skill: "onboarding",
subset: "cases",
};
}
if (setNameOrPath === "capture") {
return resolve(process.cwd(), CAPTURE_SET_PATH);
console.warn(
'[eval] --set=capture is deprecated; use --set=onboarding/capture',
);
return {
path: resolve(process.cwd(), "src/eval/golden/onboarding/capture.jsonl"),
skill: "onboarding",
subset: "capture",
};
}
if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) {
return resolve(process.cwd(), setNameOrPath);
if (setNameOrPath.endsWith(".jsonl")) {
return {
path: resolve(process.cwd(), setNameOrPath),
skill: "onboarding",
subset: basename(setNameOrPath, ".jsonl"),
};
}
return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`);
const [skill, subset = "cases"] = setNameOrPath.includes("/")
? setNameOrPath.split("/", 2)
: [setNameOrPath, "cases"];
if (!skill?.trim() || !subset.trim()) {
throw new Error(`Invalid eval set name: ${setNameOrPath}`);
}
return {
path: resolve(process.cwd(), "src/eval/golden", skill, `${subset}.jsonl`),
skill,
subset,
};
}
export async function loadEvalCases(
@@ -632,6 +702,7 @@ async function evaluateCase(
evalCase: EvalCaseDefinition,
llmClient: LlmClient,
enableLlmJudge: boolean,
skillName: string,
): Promise<EvalCaseResult> {
const input = evalCase.input;
const inMemoryGarnet = new InMemoryGarnetClient();
@@ -641,14 +712,16 @@ async function evaluateCase(
input.roomContext && input.roomContext.length > 0
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
: undefined;
const skill = new OnboardingSkill(
inMemoryGarnet,
llmClient,
roomContextService,
noopShadowEvalPublisher,
);
const classifier = new OnboardingIntentClassifier(llmClient);
const processTrace = new ProcessTraceCollector();
const effectiveSkill = input.request.skill || skillName;
const factory = SKILL_RUNNER_FACTORIES[effectiveSkill];
if (!factory) {
throw new Error(`No eval runner factory for skill: ${effectiveSkill}`);
}
const skill = factory({
garnet: inMemoryGarnet,
llm: llmClient,
room: roomContextService,
});
return runWithContext(
{
@@ -660,109 +733,219 @@ async function evaluateCase(
trigger: input.request.trigger,
userId: input.request.userId,
},
() => runWithProcessTrace(processTrace, async () => {
const classified = await classifier.classify(
input.request.userMessage,
input.request.trigger,
);
const derived = deriveState(
input.session ?? null,
input.request.trigger,
classified.intent,
);
const startedAt = Date.now();
try {
const response = await skill.handle(
input.request,
input.conversationHistory ?? [],
);
const latencyMs = Date.now() - startedAt;
const derivedSnapshot = toDerivedSnapshot(derived);
const ruleJudge = judgeOnboardingRuleCase(
evalCase.expected,
response.text,
derived,
);
const llmJudge = enableLlmJudge
? await judgeOnboardingWithLlm(
llmClient,
evalCase,
response.text,
derivedSnapshot,
)
: {
skipped: true as const,
reason: "disabled",
};
const processTraceSnapshot = processTrace.snapshot();
const processJudge = judgeProcessRule(
evalCase.expected,
processTraceSnapshot,
);
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: derivedSnapshot,
request: input.request,
response,
ruleJudge,
llmJudge,
processJudge,
processTrace: processTraceSnapshot,
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
const errorMessage =
error instanceof Error ? error.message : String(error);
const processTraceSnapshot = processTrace.snapshot();
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: toDerivedSnapshot(derived),
request: input.request,
response: {
requestId: input.request.requestId,
text: "",
confidence: 0,
source: "template",
metadata: {
error: errorMessage,
},
},
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
llmJudge: {
skipped: true,
reason: "skill-error",
},
processJudge: judgeProcessRule(
evalCase.expected,
processTraceSnapshot,
),
processTrace: processTraceSnapshot,
error: errorMessage,
};
}
}),
() =>
effectiveSkill === "onboarding"
? evaluateOnboardingCase(
evalCase,
input,
skill,
llmClient,
enableLlmJudge,
)
: evaluateGenericSkillCase(evalCase, input, skill),
);
}
export async function runOnboardingEval(
async function evaluateOnboardingCase(
evalCase: EvalCaseDefinition,
input: EvalCaseInput,
skill: Skill,
llmClient: LlmClient,
enableLlmJudge: boolean,
): Promise<EvalCaseResult> {
const classifier = new OnboardingIntentClassifier(llmClient);
const processTrace = new ProcessTraceCollector();
return runWithProcessTrace(processTrace, async () => {
const classified = await classifier.classify(
input.request.userMessage,
input.request.trigger,
);
const derived = deriveState(
input.session ?? null,
input.request.trigger,
classified.intent,
);
const startedAt = Date.now();
try {
const response = await skill.handle(
input.request,
input.conversationHistory ?? [],
);
const latencyMs = Date.now() - startedAt;
const derivedSnapshot = toDerivedSnapshot(derived);
const ruleJudge = judgeOnboardingRuleCase(
evalCase.expected,
response.text,
derived,
);
const llmJudge = enableLlmJudge
? await judgeOnboardingWithLlm(
llmClient,
evalCase,
response.text,
derivedSnapshot,
)
: {
skipped: true as const,
reason: "disabled",
};
const processTraceSnapshot = processTrace.snapshot();
const processJudge = judgeProcessRule(
evalCase.expected,
processTraceSnapshot,
);
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: derivedSnapshot,
request: input.request,
response,
ruleJudge,
llmJudge,
processJudge,
processTrace: processTraceSnapshot,
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
const errorMessage =
error instanceof Error ? error.message : String(error);
const processTraceSnapshot = processTrace.snapshot();
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: toDerivedSnapshot(derived),
request: input.request,
response: {
requestId: input.request.requestId,
text: "",
confidence: 0,
source: "template",
metadata: {
error: errorMessage,
},
},
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
llmJudge: {
skipped: true,
reason: "skill-error",
},
processJudge: judgeProcessRule(
evalCase.expected,
processTraceSnapshot,
),
processTrace: processTraceSnapshot,
error: errorMessage,
};
}
});
}
async function evaluateGenericSkillCase(
evalCase: EvalCaseDefinition,
input: EvalCaseInput,
skill: Skill,
): Promise<EvalCaseResult> {
const classified = classifyModerationIntent(input.request.userMessage ?? "");
const derived: EvalDerivedSnapshot = {
allowShortResponse: true,
intent: classified.intent.kind,
phase: skill.name,
responseMode: classified.intent.kind === "off-topic" ? "template" : "llm",
shouldRespond: classified.intent.kind !== "off-topic",
suggestedMaxTokens: 192,
tier: 0,
};
const startedAt = Date.now();
try {
const response = await skill.handle(
input.request,
input.conversationHistory ?? [],
);
const latencyMs = Date.now() - startedAt;
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: {
confidence: classified.confidence,
intent: classified.intent.kind,
source: "regex",
},
derived,
request: input.request,
response,
ruleJudge: judgeRuleCase(evalCase.expected, response.text, {
intent: classified.intent.kind,
responseMode: derived.responseMode,
}),
llmJudge: {
skipped: true,
reason: "unsupported-skill",
},
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
const errorMessage = error instanceof Error ? error.message : String(error);
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: {
confidence: classified.confidence,
intent: classified.intent.kind,
source: "regex",
},
derived,
request: input.request,
response: {
requestId: input.request.requestId,
text: "",
confidence: 0,
source: "template",
metadata: {
error: errorMessage,
},
},
ruleJudge: judgeRuleCase(evalCase.expected, "", {
intent: classified.intent.kind,
responseMode: derived.responseMode,
}),
llmJudge: {
skipped: true,
reason: "skill-error",
},
error: errorMessage,
};
}
}
export async function runEval(
options?: EvalRunOptions,
): Promise<{ report: EvalReport; reportPath: string }> {
const logger = getLogger();
const setName = options?.setName ?? "golden";
const inputPath = options?.inputPath
? resolve(process.cwd(), options.inputPath)
const setName = options?.setName ?? "onboarding/cases";
const resolvedSet = options?.inputPath
? {
path: resolve(process.cwd(), options.inputPath),
skill: options.skill ?? "onboarding",
subset: basename(options.inputPath, ".jsonl"),
}
: resolveEvalSetPath(setName);
const skill = options?.skill ?? resolvedSet.skill;
const inputPath = resolvedSet.path;
const evalCases = await loadEvalCases(inputPath, {
limit: options?.limit,
});
@@ -772,11 +955,11 @@ export async function runOnboardingEval(
const caseResults: EvalCaseResult[] = [];
for (const evalCase of evalCases) {
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge));
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge, skill));
}
const summary = createSummary(caseResults);
const baseline = await loadBaselineFile();
const baseline = await loadBaselineFile(skill);
const report: EvalReport = {
generatedAt: new Date().toISOString(),
inputPath,
@@ -798,6 +981,7 @@ export async function runOnboardingEval(
cases: summary.cases,
passRate: summary.passRate,
reportPath,
skill,
},
"[Eval] run completed",
);
@@ -805,6 +989,12 @@ export async function runOnboardingEval(
return { report, reportPath };
}
export async function runOnboardingEval(
options?: EvalRunOptions,
): Promise<{ report: EvalReport; reportPath: string }> {
return runEval({ ...options, skill: options?.skill ?? "onboarding" });
}
export async function findLatestEvalReportPath(): Promise<string | null> {
const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR);
if (!existsSync(reportsDir)) {
@@ -815,7 +1005,8 @@ export async function findLatestEvalReportPath(): Promise<string | null> {
const candidates = entries
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
.filter((name) => name.endsWith(".json") && name !== "baseline.json")
.filter((name) => name.endsWith(".json") && !name.startsWith("baseline."))
.filter((name) => name !== "baseline.json")
.sort((left, right) => right.localeCompare(left));
if (candidates.length === 0) {
@@ -845,6 +1036,7 @@ export async function updateEvalBaseline(
const raw = await readFile(sourcePath, "utf8");
const report = JSON.parse(raw) as EvalReport;
const skill = resolveEvalSetPath(report.setName).skill;
const baseline: EvalBaselineFile = {
reportPath: sourcePath,
setName: report.setName,
@@ -852,7 +1044,7 @@ export async function updateEvalBaseline(
updatedAt: new Date().toISOString(),
};
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
const baselinePath = resolve(process.cwd(), baselinePathFor(skill));
await mkdir(dirname(baselinePath), { recursive: true });
await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
+8 -3
View File
@@ -80,7 +80,10 @@ const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
const BACKGROUND_GROUP = "bg-worker";
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
const EVAL_BASELINE_FILES = [
"eval-reports/baseline.onboarding.json",
"eval-reports/baseline.json",
];
const ALERT_WINDOW_MS = 5 * 60_000;
const ALERT_WINDOW_LABEL = "5m";
const ALERT_TICK_MS = 60_000;
@@ -1123,8 +1126,10 @@ export class ShadowWorker {
return redisParsed;
}
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
if (!existsSync(filePath)) {
const filePath = EVAL_BASELINE_FILES
.map((candidate) => resolve(process.cwd(), candidate))
.find((candidate) => existsSync(candidate));
if (!filePath) {
return null;
}
+10
View File
@@ -228,7 +228,17 @@ async function bootstrap(): Promise<void> {
moderation: createModerationSkill,
onboarding: createOnboardingSkill,
};
const seenSkills = new Set<string>();
for (const manifest of skillRegistry.list()) {
if (seenSkills.has(manifest.name)) {
logger.warn(
{ skill: manifest.name },
"[Skill] Duplicate manifest ignored",
);
continue;
}
seenSkills.add(manifest.name);
const factory = factories[manifest.name];
if (!factory) {
logger.warn(
+4 -2
View File
@@ -2,10 +2,12 @@
name: moderation
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
version: 1
priority: 50
intents: [greeting, rule-question, violation-report, appeal, off-topic]
triggers: [user-message]
tools: []
resources: {}
tools: [lookup-rule]
resources:
community-rules: ./resources/community-rules.md
trustTier: T4
---
+6 -4
View File
@@ -6,19 +6,21 @@ interface ModerationIntentDefinition {
export const MODERATION_INTENT_TABLE = {
greeting: {
confidence: 0.9,
regex: [/^(안녕|하이|반갑|문의드립니다)/u],
regex: [/^(안녕|하이|반갑|문의드립|운영진|모더|관리자)/u],
},
"rule-question": {
confidence: 0.92,
regex: [/(규칙|룰|금지|가능|해도\s*돼|하면\s*안|제재|경고)/u],
regex: [
/(규칙|룰|금지|가능|해도\s*돼|하면\s*안|제재|경고|광고|홍보|도배|링크|욕설|혐오|차별)/u,
],
},
"violation-report": {
confidence: 0.94,
regex: [/(신고|제보|위반|욕설|분쟁|비매너|문제\s*있|처리해)/u],
regex: [/(신고|제보|위반|욕설|분쟁|비매너|문제\s*있|처리해|누가|사람이|유저가|이용자)/u],
},
appeal: {
confidence: 0.94,
regex: [/(이의|소명|억울|해제|풀어|재검토|항소|정지)/u],
regex: [/(이의|소명|억울|해제|풀어|재검토|항소|정지|벤|밴|차단\s*풀|차단)/u],
},
"off-topic": {
confidence: 0,
+1
View File
@@ -8,6 +8,7 @@ export const MODERATION_PROMPT_DATA = {
policy: {
base: [
"- 답변은 1~3문장으로 짧게 한다.",
"- 참고 규칙이 제공되면 그 범위 안에서만 답한다.",
"- 처벌, 제재, 신고 처리 결과를 확정해서 말하지 않는다.",
"- 개인정보, 계정 정보, 특정 이용자 평가를 요구하면 운영진 확인으로 돌린다.",
],
@@ -0,0 +1,35 @@
## ads
광고, 홍보, 외부 링크 반복 게시물은 운영진 허가가 없는 경우 제한될 수 있다.
개인 방송, 거래, 모집 홍보도 사전 확인이 필요하다.
## spam
같거나 유사한 메시지를 짧은 시간에 반복하는 도배는 제재 대상이 될 수 있다.
대화 흐름을 방해하는 과도한 멘션이나 이모지도 도배로 볼 수 있다.
## profanity
욕설, 혐오 표현, 차별적 표현, 특정 이용자 비하 발언은 금지된다.
분쟁이 생기면 맞대응하지 말고 운영진에게 상황을 전달한다.
## report
위반 제보는 문제 메시지, 시간, 관련 이용자, 맥락을 함께 정리해 운영진에게 전달한다.
처리 결과와 제재 여부는 운영진 확인 뒤 결정된다.
## sanctions
제재는 경고, 메시지 삭제, 임시 제한, 강퇴 등 단계적으로 적용될 수 있다.
사안의 반복성, 고의성, 피해 정도에 따라 운영진이 판단한다.
## appeal
정지나 차단이 억울하면 당시 상황, 근거, 반성 또는 소명 내용을 정리해 운영진에게 이의 제기한다.
assistant는 해제 여부를 확정하지 않는다.
## minor-safety
미성년자 보호를 위해 부적절한 대화, 사적 연락 강요, 성적 발언은 엄격히 제한된다.
불편하거나 위험한 상황은 즉시 운영진에게 알린다.
## privacy
전화번호, 주소, 계정 정보, 실명 등 개인정보 공유나 요구는 금지된다.
타인의 개인정보를 발견하면 확산하지 말고 운영진에게 제보한다.
## off-topic
규칙과 무관한 대화는 해당 채팅방의 목적에 맞는 곳으로 옮기도록 안내한다.
반복적으로 흐름을 방해하면 운영진 확인이 필요할 수 있다.
+73 -2
View File
@@ -6,10 +6,22 @@ import type {
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { Skill } from "@skills/skill";
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
import { createModerationToolRegistry } from "./tools.js";
type RuleTopic =
| "ads"
| "appeal"
| "minor-safety"
| "off-topic"
| "privacy"
| "profanity"
| "report"
| "sanctions"
| "spam";
function formatHistory(history: ConversationMessage[]): string {
if (history.length === 0) {
return "(none)";
@@ -32,11 +44,15 @@ function buildMessages(
request: AgentRequest,
history: ConversationMessage[],
intent: ModerationIntent,
referenceText?: string,
): ChatMessage[] {
const policies = [
...MODERATION_PROMPT_DATA.policy.base,
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
];
const referenceBlock = referenceText?.trim()
? ["", "[reference]", referenceText.trim()]
: [];
return [
{
@@ -52,6 +68,7 @@ function buildMessages(
"",
"[examples]",
formatExamples(intent),
...referenceBlock,
].join("\n"),
},
{
@@ -67,6 +84,52 @@ function buildMessages(
];
}
function resolveRuleTopic(message: string): RuleTopic | null {
if (/(광고|홍보|링크)/u.test(message)) {
return "ads";
}
if (/(도배|반복|스팸)/u.test(message)) {
return "spam";
}
if (/(욕설|욕했|혐오|차별|비하)/u.test(message)) {
return "profanity";
}
if (/(신고|제보|위반)/u.test(message)) {
return "report";
}
if (/(경고|제재|강퇴|처벌)/u.test(message)) {
return "sanctions";
}
if (/(이의|소명|억울|정지|벤|밴|차단)/u.test(message)) {
return "appeal";
}
if (/(미성년|미자|성적|보호)/u.test(message)) {
return "minor-safety";
}
if (/(개인정보|전화번호|주소|실명|계정)/u.test(message)) {
return "privacy";
}
if (/(잡담|주제|오프토픽|off-topic)/iu.test(message)) {
return "off-topic";
}
return null;
}
async function lookupReferenceText(
intent: ModerationIntent,
message: string,
): Promise<string | undefined> {
if (intent.kind !== "rule-question") {
return undefined;
}
const topic = resolveRuleTopic(message);
if (!topic) {
return undefined;
}
const result = await lookupRuleTool.execute({ topic }, {});
return result.text;
}
function templateForIntent(intent: ModerationIntent): string {
switch (intent.kind) {
case "appeal":
@@ -84,6 +147,7 @@ function templateForIntent(intent: ModerationIntent): string {
export class ModerationSkill implements Skill {
readonly name = "moderation";
readonly priority = 50;
private readonly toolRegistry = createModerationToolRegistry();
constructor(private readonly llmClient: LlmClient) {}
@@ -97,6 +161,10 @@ export class ModerationSkill implements Skill {
conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const classified = classifyModerationIntent(request.userMessage ?? "");
const referenceText = await lookupReferenceText(
classified.intent,
request.userMessage ?? "",
);
const metadata = {
intent: classified.intent.kind,
toolCount: this.toolRegistry.list().length,
@@ -113,7 +181,7 @@ export class ModerationSkill implements Skill {
}
const response = await this.llmClient.chatCompletion(
buildMessages(request, conversationHistory, classified.intent),
buildMessages(request, conversationHistory, classified.intent, referenceText),
{
maxTokens: 192,
temperature: 0.2,
@@ -126,7 +194,10 @@ export class ModerationSkill implements Skill {
metadata,
requestId: request.requestId,
source: response ? "llm" : "template",
text: response?.content.trim() || templateForIntent(classified.intent),
text:
response?.content.trim() ||
referenceText ||
templateForIntent(classified.intent),
};
}
}
+2 -1
View File
@@ -1,5 +1,6 @@
import { ToolRegistry } from "@tools/registry";
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
export function createModerationToolRegistry(): ToolRegistry {
return new ToolRegistry();
return new ToolRegistry().register(lookupRuleTool);
}
+1
View File
@@ -2,6 +2,7 @@
name: onboarding
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
version: 1
priority: 70
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
@@ -0,0 +1,78 @@
import {
getAcknowledgementMessage,
getAgeRequirementMessage,
getDiscordMessage,
getGenericHelpMessage,
getGreetingMessage,
getHeartGuideMessage,
getHeartLocationMessage,
getNicknameInvalidMessage,
getNicknameFormatMessage,
getPhotoFailedMessage,
getRemainingStatusMessage,
} from '../onboarding-templates.js';
import type { DerivedState, Intent } from '../state/onboarding-state.js';
import type { OnboardingSession } from '../types.js';
export function getIntentFallbackText(
nickname: string,
session: OnboardingSession | null,
intent: Intent,
derived: DerivedState,
): string {
switch (intent.kind) {
case 'greeting':
return getGreetingMessage();
case 'acknowledgement':
return getAcknowledgementMessage();
case 'status-question':
return getRemainingStatusMessage(session);
case 'nickname-question':
return derived.phase.kind === 'nickname-rejected'
? getNicknameInvalidMessage(nickname, session)
: getNicknameFormatMessage();
case 'photo-question':
return derived.phase.kind === 'photo-rejected'
? getPhotoFailedMessage(nickname)
: getHeartGuideMessage();
case 'help-where':
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' &&
derived.phase.remaining.includes('photo'))
) {
return getHeartLocationMessage();
}
return getGenericHelpMessage(nickname, session);
case 'help-how':
if (
derived.phase.kind === 'nickname-pending' ||
derived.phase.kind === 'nickname-rejected' ||
(derived.phase.kind === 'deadline-near' &&
derived.phase.remaining.includes('nickname'))
) {
return getNicknameFormatMessage();
}
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' &&
derived.phase.remaining.includes('photo'))
) {
return getHeartGuideMessage();
}
return getGenericHelpMessage(nickname, session);
case 'discord-question':
return getDiscordMessage(nickname, session);
case 'discord-verify-question':
return getDiscordMessage(nickname, session);
case 'age-question':
return getAgeRequirementMessage();
case 'chitchat':
case 'off-topic-to-other':
return '';
case 'unknown':
return getGenericHelpMessage(nickname, session);
}
}
@@ -0,0 +1,125 @@
import type { AgentRequest, ConversationMessage } from '@agent/types';
import type { GarnetClient } from '@common/garnet';
import { getConversationSummaryKey } from '@conv/conversation-store';
import type { ChatMessage } from '@llm/types';
import { getRequestLogger } from '@obs/context';
import type { RoomContextService } from '@room/room-context-service';
import { meetsTrustTier } from '../../../eval/governance/trust.js';
import {
getAgentTrustMinTier,
getAgentUserMemoryEnabled,
getPromptVariantId,
} from '../../../config.js';
import {
loadPromptVariantManifest,
PromptBuilder,
} from '../prompts/builder.js';
import type { OnboardingBlackboardStore } from '../state/blackboard.js';
import type { UserMemoryStore } from '../state/user-memory.js';
import type { DerivedState } from '../state/onboarding-state.js';
import type { OnboardingSession, PromptContext } from '../types.js';
interface MessageBuilderDeps {
blackboardStore: OnboardingBlackboardStore;
garnetClient: GarnetClient;
roomContextService?: RoomContextService | undefined;
userMemoryStore: UserMemoryStore;
}
interface MessageBuilderInput {
conversationHistory: ConversationMessage[];
derived: DerivedState;
request: AgentRequest;
session: OnboardingSession | null;
}
function buildPromptUserMessage(
request: AgentRequest,
derived: DerivedState,
): string {
switch (request.trigger) {
case 'nickname-invalid':
return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`;
case 'photo-failed':
return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.';
case 'progress-update':
return '현재 새내기 등록 진행상황만 짧게 알려줘.';
case 'timeout-warning':
return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.';
case 'completed':
return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.';
case 'user-message':
if (derived.intent.kind === 'status-question') {
return '완료된 것은 빼고 남은 절차만 짧게 말해줘.';
}
return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.';
default:
return '지금 필요한 온보딩 안내만 짧게 도와줘.';
}
}
async function loadConversationSummary(
garnetClient: GarnetClient,
chatroomId: string,
userId: string,
): Promise<string | undefined> {
const key = getConversationSummaryKey(chatroomId, userId);
return (await garnetClient.getString(key)) ?? undefined;
}
export async function buildLlmMessages(
deps: MessageBuilderDeps,
input: MessageBuilderInput,
): Promise<ChatMessage[]> {
const logger = getRequestLogger();
const { conversationHistory, derived, request, session } = input;
const roomContextPromise =
request.trigger === 'user-message' && deps.roomContextService
? deps.roomContextService
.getRecentMessages(request.chatroomId)
.catch((error) => {
logger.warn(
{ err: error },
'[OnboardingSkill] Failed to fetch room context',
);
return undefined;
})
: Promise.resolve(undefined);
const userProfilePromise = getAgentUserMemoryEnabled()
? deps.userMemoryStore.load(request.userId).catch((error) => {
logger.warn(
{ err: error },
'[OnboardingSkill] Failed to load user memory',
);
return null;
})
: Promise.resolve(null);
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
deps.blackboardStore.load(request.chatroomId, request.userId),
loadConversationSummary(deps.garnetClient, request.chatroomId, request.userId),
roomContextPromise,
userProfilePromise,
]);
const context: PromptContext = {
blackboard,
conversationHistory,
roomContext,
summary,
userMessage: buildPromptUserMessage(request, derived),
userProfile,
};
const builder = PromptBuilder.forDerived(derived, session, context);
const promptVariantId = getPromptVariantId();
if (!promptVariantId) {
return builder.build();
}
const manifest = loadPromptVariantManifest(promptVariantId);
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
return builder.build();
}
return builder.applyVariant(manifest).build();
}
@@ -0,0 +1,63 @@
import type { AgentRequest } from '@agent/types';
import type { DerivedState } from '../state/onboarding-state.js';
import type { OnboardingBlackboardStore } from '../state/blackboard.js';
export async function rememberTurn(
blackboardStore: OnboardingBlackboardStore,
request: AgentRequest,
derived: DerivedState,
): Promise<void> {
const now = new Date().toISOString();
const userMessage = request.userMessage?.trim();
await blackboardStore.update(request.chatroomId, request.userId, (current) => {
const sameIntentStreak =
current.lastIntent === derived.intent.kind
? (current.sameIntentStreak ?? 1) + 1
: 1;
const openQuestions =
userMessage &&
(derived.intent.kind === 'help-how' ||
derived.intent.kind === 'help-where' ||
derived.intent.kind === 'nickname-question' ||
derived.intent.kind === 'photo-question' ||
derived.intent.kind === 'discord-question' ||
derived.intent.kind === 'discord-verify-question')
? [...current.openQuestions, userMessage].slice(-3)
: current.openQuestions;
const recentNicknameAttempts =
request.trigger === 'nickname-invalid' ||
derived.intent.kind === 'nickname-question'
? Array.from(
new Set([...current.recentNicknameAttempts, request.nickname]),
).slice(-5)
: current.recentNicknameAttempts;
return {
...current,
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: derived.phase.kind === 'completed'
? []
: current.flaggedIssues,
lastIntent: derived.intent.kind,
lastIntentAt: now,
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
recentNicknameAttempts,
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
toneMemory: {
...current.toneMemory,
greetedAt:
derived.intent.kind === 'greeting'
? now
: current.toneMemory.greetedAt,
thankedAt:
derived.intent.kind === 'acknowledgement'
? now
: current.toneMemory.thankedAt,
},
};
});
}
@@ -0,0 +1,101 @@
import type { AgentRequest } from '@agent/types';
import type { ChatMessage } from '@llm/types';
import type { Tool, ToolContext } from '@tools/tool';
import { getLlmTemperature } from '../../../config.js';
import type { DerivedState } from '../state/onboarding-state.js';
import type { OnboardingOrchestrator } from './orchestrator.js';
import { validateOnboardingResponse } from './validator.js';
export type LlmTextResult =
| { status: 'success'; text: string }
| { status: 'no_response' }
| { status: 'failed' };
export async function generateLlmText(
orchestrator: OnboardingOrchestrator,
messages: ChatMessage[],
derived: DerivedState,
options?: {
allowShortResponse?: boolean;
maxTokens?: number;
request: AgentRequest;
retryOnLowQuality?: boolean;
temperature?: number;
toolContext?: ToolContext | undefined;
tools?: readonly Tool<unknown, unknown>[] | undefined;
},
): Promise<LlmTextResult> {
const attempts: Array<{
maxTokens?: number;
temperature?: number;
}> = [];
const firstAttempt: {
maxTokens?: number;
temperature?: number;
} = {};
if (options?.maxTokens !== undefined) {
firstAttempt.maxTokens = options.maxTokens;
}
if (options?.temperature !== undefined) {
firstAttempt.temperature = options.temperature;
}
attempts.push(firstAttempt);
if (options?.retryOnLowQuality) {
const retryAttempt: {
maxTokens?: number;
temperature?: number;
} = {
temperature: getLlmTemperature() + 0.1,
};
if (options.maxTokens !== undefined) {
retryAttempt.maxTokens = options.maxTokens;
}
attempts.push(retryAttempt);
}
for (const attempt of attempts) {
const requestOptions: {
maxTokens?: number;
temperature?: number;
} = {};
if (attempt.maxTokens !== undefined) {
requestOptions.maxTokens = attempt.maxTokens;
}
if (attempt.temperature !== undefined) {
requestOptions.temperature = attempt.temperature;
}
const toolContext = options?.toolContext ?? {};
const llmText = await orchestrator.run(derived, messages, {
maxTokens: requestOptions.maxTokens,
request: options?.request,
temperature: requestOptions.temperature,
toolContext,
tools: options?.tools,
});
if (!llmText) {
continue;
}
const validation = validateOnboardingResponse(llmText, derived, {
allowShortResponse: options?.allowShortResponse,
});
if (validation.text.includes('[NO_RESPONSE]')) {
return { status: 'no_response' };
}
if (validation.status === 'reject') {
continue;
}
return {
status: 'success',
text: validation.text,
};
}
return { status: 'failed' };
}
@@ -0,0 +1,61 @@
import type { Tool } from '@tools/tool';
import { createToolSearchTool } from '@tools/tool-search';
import type { ToolRegistry } from '@tools/registry';
import {
getAgentToolSearchEnabled,
getAgentToolSearchThreshold,
} from '../../../config.js';
import type { DerivedState } from '../state/onboarding-state.js';
export function selectToolsForDerived(
derived: DerivedState,
tools: readonly Tool<unknown, unknown>[],
toolRegistry: ToolRegistry,
): readonly Tool<unknown, unknown>[] {
const maybeUseToolSearch = (
selectedTools: readonly Tool<unknown, unknown>[],
): readonly Tool<unknown, unknown>[] => {
if (
getAgentToolSearchEnabled() &&
selectedTools.length >= getAgentToolSearchThreshold()
) {
return [createToolSearchTool(toolRegistry)];
}
return selectedTools;
};
if (derived.intent.kind === 'discord-verify-question') {
return maybeUseToolSearch(
tools.filter((tool) => tool.name === 'verify-discord-nickname'),
);
}
if (derived.phase.kind === 'nickname-rejected') {
return maybeUseToolSearch(
tools.filter(
(tool) =>
tool.name === 'validate-nickname' ||
tool.name === 'lookup-community-rule' ||
tool.name === 'check-onboarding-progress',
),
);
}
if (derived.intent.kind === 'status-question') {
return maybeUseToolSearch(
tools.filter((tool) => tool.name === 'check-onboarding-progress'),
);
}
if (derived.intent.kind === 'discord-question') {
return maybeUseToolSearch(
tools.filter(
(tool) =>
tool.name === 'lookup-community-rule' ||
tool.name === 'verify-discord-nickname',
),
);
}
return maybeUseToolSearch(tools);
}
+31 -354
View File
@@ -1,38 +1,22 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
import { getConversationSummaryKey } from '@conv/conversation-store';
import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats';
import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { ChatMessage } from '@llm/types';
import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { meetsTrustTier } from '../../eval/governance/trust.js';
import { createOnboardingToolRegistry } from './tools.js';
import type { ToolRegistry } from '@tools/registry';
import type { Tool, ToolContext } from '@tools/tool';
import { createToolSearchTool } from '@tools/tool-search';
import {
loadPromptVariantManifest,
PromptBuilder,
} from './prompts/builder.js';
import {
getAcknowledgementMessage,
getAgeRequirementMessage,
getCompletedMessage,
getDiscordMessage,
getGreetingMessage,
getGenericHelpMessage,
getHeartGuideMessage,
getHeartLocationMessage,
getNicknameInvalidMessage,
getNicknameFormatMessage,
getOnboardingTemplate,
getPhotoFailedMessage,
getProgressMessage,
getRemainingStatusMessage,
getTimeoutWarningMessage,
} from './onboarding-templates.js';
import {
@@ -41,13 +25,8 @@ import {
getAgentAbstentionThreshold,
getAgentRouterMode,
getAgentRouterWeightsPath,
getAgentTrustMinTier,
getAgentToolSearchEnabled,
getAgentToolSearchThreshold,
getAgentUserMemoryEnabled,
getOnboardingSessionKeyPrefix,
getLlmTemperature,
getPromptVariantId,
} from '../../config.js';
import { OnboardingBlackboardStore } from './state/blackboard.js';
import { UserMemoryStore } from './state/user-memory.js';
@@ -74,21 +53,24 @@ import {
runWithProcessTrace,
} from './runtime/trace.js';
import { validateOnboardingResponse } from './runtime/validator.js';
import { getIntentFallbackText } from './runtime/fallback-text-resolver.js';
import { buildLlmMessages } from './runtime/message-builder.js';
import {
generateLlmText,
type LlmTextResult,
} from './runtime/text-generator.js';
import { selectToolsForDerived } from './runtime/tool-selector.js';
import { rememberTurn } from './runtime/state-updater.js';
import { normalizeOnboardingSession } from './session.js';
import type {
Blackboard,
OnboardingSession,
PromptContext,
StoredOnboardingSession,
} from './types.js';
type LlmTextResult =
| { status: 'success'; text: string }
| { status: 'no_response' }
| { status: 'failed' };
export class OnboardingSkill implements Skill {
readonly name = 'onboarding';
readonly priority = 70;
private readonly classifier: OnboardingIntentClassifier;
private readonly blackboardStore: OnboardingBlackboardStore;
private readonly orchestrator: OnboardingOrchestrator;
@@ -357,9 +339,26 @@ export class OnboardingSkill implements Skill {
conversationHistory: ConversationMessage[],
derived: DerivedState,
): Promise<LlmTextResult> {
const messages = await this.buildLlmMessages(request, session, conversationHistory, derived);
if (!this.llmClient.hasEndpoints()) {
return { status: 'failed' };
}
return this.generateLlmText(messages, derived, {
const messages = await buildLlmMessages(
{
blackboardStore: this.blackboardStore,
garnetClient: this.garnetClient,
roomContextService: this.roomContextService,
userMemoryStore: this.userMemoryStore,
},
{
conversationHistory,
derived,
request,
session,
},
);
return generateLlmText(this.orchestrator, messages, derived, {
allowShortResponse: derived.allowShortResponse,
maxTokens: derived.suggestedMaxTokens,
request,
@@ -368,135 +367,12 @@ export class OnboardingSkill implements Skill {
!derived.allowShortResponse &&
derived.responseMode !== 'llm-with-tools',
toolContext: this.buildToolContext(request, session),
tools: this.selectToolsForDerived(derived),
tools: selectToolsForDerived(derived, this.tools, this.toolRegistry),
});
}
private async buildLlmMessages(
request: AgentRequest,
session: OnboardingSession | null,
conversationHistory: ConversationMessage[],
derived: DerivedState,
): Promise<ChatMessage[]> {
const logger = getRequestLogger();
const roomContextPromise =
request.trigger === 'user-message' && this.roomContextService
? this.roomContextService.getRecentMessages(request.chatroomId).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to fetch room context');
return undefined;
})
: Promise.resolve(undefined);
const userProfilePromise = getAgentUserMemoryEnabled()
? this.userMemoryStore.load(request.userId).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory');
return null;
})
: Promise.resolve(null);
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
this.blackboardStore.load(request.chatroomId, request.userId),
this.loadConversationSummary(request.chatroomId, request.userId),
roomContextPromise,
userProfilePromise,
]);
const context: PromptContext = {
blackboard,
conversationHistory,
roomContext,
summary,
userMessage: this.buildPromptUserMessage(request, derived),
userProfile,
};
const builder = PromptBuilder.forDerived(derived, session, context);
const promptVariantId = getPromptVariantId();
if (!promptVariantId) {
return builder.build();
}
const manifest = loadPromptVariantManifest(promptVariantId);
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
return builder.build();
}
return builder.applyVariant(manifest).build();
}
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
switch (request.trigger) {
case 'nickname-invalid':
return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`;
case 'photo-failed':
return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.';
case 'progress-update':
return '현재 새내기 등록 진행상황만 짧게 알려줘.';
case 'timeout-warning':
return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.';
case 'completed':
return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.';
case 'user-message':
if (derived.intent.kind === 'status-question') {
return '완료된 것은 빼고 남은 절차만 짧게 말해줘.';
}
return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.';
default:
return '지금 필요한 온보딩 안내만 짧게 도와줘.';
}
}
private async loadConversationSummary(
chatroomId: string,
userId: string,
): Promise<string | undefined> {
const key = getConversationSummaryKey(chatroomId, userId);
return (await this.garnetClient.getString(key)) ?? undefined;
}
private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise<void> {
const now = new Date().toISOString();
const userMessage = request.userMessage?.trim();
await this.blackboardStore.update(request.chatroomId, request.userId, (current) => {
const sameIntentStreak =
current.lastIntent === derived.intent.kind
? (current.sameIntentStreak ?? 1) + 1
: 1;
const openQuestions =
userMessage &&
(derived.intent.kind === 'help-how' ||
derived.intent.kind === 'help-where' ||
derived.intent.kind === 'nickname-question' ||
derived.intent.kind === 'photo-question' ||
derived.intent.kind === 'discord-question' ||
derived.intent.kind === 'discord-verify-question')
? [...current.openQuestions, userMessage].slice(-3)
: current.openQuestions;
const recentNicknameAttempts =
request.trigger === 'nickname-invalid' || derived.intent.kind === 'nickname-question'
? Array.from(new Set([...current.recentNicknameAttempts, request.nickname])).slice(-5)
: current.recentNicknameAttempts;
return {
...current,
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: derived.phase.kind === 'completed'
? []
: current.flaggedIssues,
lastIntent: derived.intent.kind,
lastIntentAt: now,
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
recentNicknameAttempts,
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
toneMemory: {
...current.toneMemory,
greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt,
thankedAt: derived.intent.kind === 'acknowledgement' ? now : current.toneMemory.thankedAt,
},
};
});
return rememberTurn(this.blackboardStore, request, derived);
}
private async observeUserMemory(
@@ -552,163 +428,11 @@ export class OnboardingSkill implements Skill {
default:
return this.makeTemplateResponse(
request.requestId,
this.getIntentFallbackText(request.nickname, session, intent, derived),
getIntentFallbackText(request.nickname, session, intent, derived),
);
}
}
private getIntentFallbackText(
nickname: string,
session: OnboardingSession | null,
intent: Intent,
derived: DerivedState,
): string {
switch (intent.kind) {
case 'greeting':
return getGreetingMessage();
case 'acknowledgement':
return getAcknowledgementMessage();
case 'status-question':
return getRemainingStatusMessage(session);
case 'nickname-question':
return derived.phase.kind === 'nickname-rejected'
? getNicknameInvalidMessage(nickname, session)
: getNicknameFormatMessage();
case 'photo-question':
return derived.phase.kind === 'photo-rejected'
? getPhotoFailedMessage(nickname)
: getHeartGuideMessage();
case 'help-where':
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
) {
return getHeartLocationMessage();
}
return getGenericHelpMessage(nickname, session);
case 'help-how':
if (
derived.phase.kind === 'nickname-pending' ||
derived.phase.kind === 'nickname-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('nickname'))
) {
return getNicknameFormatMessage();
}
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
) {
return getHeartGuideMessage();
}
return getGenericHelpMessage(nickname, session);
case 'discord-question':
return getDiscordMessage(nickname, session);
case 'discord-verify-question':
return getDiscordMessage(nickname, session);
case 'age-question':
return getAgeRequirementMessage();
case 'chitchat':
case 'off-topic-to-other':
return '';
case 'unknown':
return getGenericHelpMessage(nickname, session);
}
}
private async generateLlmText(
messages: ChatMessage[],
derived: DerivedState,
options?: {
allowShortResponse?: boolean;
maxTokens?: number;
request: AgentRequest;
retryOnLowQuality?: boolean;
temperature?: number;
toolContext?: ToolContext | undefined;
tools?: readonly Tool<unknown, unknown>[] | undefined;
},
): Promise<LlmTextResult> {
if (!this.llmClient.hasEndpoints()) {
return { status: 'failed' };
}
const attempts: Array<{
maxTokens?: number;
temperature?: number;
}> = [];
const firstAttempt: {
maxTokens?: number;
temperature?: number;
} = {};
if (options?.maxTokens !== undefined) {
firstAttempt.maxTokens = options.maxTokens;
}
if (options?.temperature !== undefined) {
firstAttempt.temperature = options.temperature;
}
attempts.push(firstAttempt);
if (options?.retryOnLowQuality) {
const retryAttempt: {
maxTokens?: number;
temperature?: number;
} = {
temperature: getLlmTemperature() + 0.1,
};
if (options.maxTokens !== undefined) {
retryAttempt.maxTokens = options.maxTokens;
}
attempts.push(retryAttempt);
}
for (const attempt of attempts) {
const requestOptions: {
maxTokens?: number;
temperature?: number;
} = {};
if (attempt.maxTokens !== undefined) {
requestOptions.maxTokens = attempt.maxTokens;
}
if (attempt.temperature !== undefined) {
requestOptions.temperature = attempt.temperature;
}
const toolContext = options?.toolContext ?? {};
const llmText = await this.orchestrator.run(derived, messages, {
maxTokens: requestOptions.maxTokens,
request: options?.request,
temperature: requestOptions.temperature,
toolContext,
tools: options?.tools,
});
if (!llmText) {
continue;
}
const validation = validateOnboardingResponse(llmText, derived, {
allowShortResponse: options?.allowShortResponse,
});
if (validation.text.includes('[NO_RESPONSE]')) {
return { status: 'no_response' };
}
if (validation.status === 'reject') {
continue;
}
return {
status: 'success',
text: validation.text,
};
}
return { status: 'failed' };
}
private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext {
return {
chatroomId: request.chatroomId,
@@ -721,53 +445,6 @@ export class OnboardingSkill implements Skill {
};
}
private selectToolsForDerived(derived: DerivedState): readonly Tool<unknown, unknown>[] {
const maybeUseToolSearch = (
tools: readonly Tool<unknown, unknown>[],
): readonly Tool<unknown, unknown>[] => {
if (
getAgentToolSearchEnabled() &&
tools.length >= getAgentToolSearchThreshold()
) {
return [createToolSearchTool(this.toolRegistry)];
}
return tools;
};
if (derived.intent.kind === 'discord-verify-question') {
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'verify-discord-nickname'),
);
}
if (derived.phase.kind === 'nickname-rejected') {
return maybeUseToolSearch(
this.tools.filter(
(tool) =>
tool.name === 'validate-nickname' ||
tool.name === 'lookup-community-rule' ||
tool.name === 'check-onboarding-progress',
),
);
}
if (derived.intent.kind === 'status-question') {
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'check-onboarding-progress'),
);
}
if (derived.intent.kind === 'discord-question') {
return maybeUseToolSearch(
this.tools.filter(
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
),
);
}
return maybeUseToolSearch(this.tools);
}
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
return {
requestId,
+23
View File
@@ -18,6 +18,7 @@ function makeTempSkillRoot(): string {
"name: demo",
"description: Demo skill.",
"version: 1",
"priority: 65",
"intents: [hello, help]",
"triggers: [user-message]",
"tools: [demo-tool]",
@@ -41,6 +42,7 @@ describe("SkillRegistry", () => {
name: "demo",
description: "Demo skill.",
intents: ["hello", "help"],
priority: 65,
tools: ["demo-tool"],
trustTier: "T2",
version: 1,
@@ -49,6 +51,27 @@ describe("SkillRegistry", () => {
expect(registry.loadResourceSync("demo", "guide")).toBe("guide body");
});
it("defaults priority to 50 when omitted", () => {
const root = mkdtempSync(join(tmpdir(), "lolympus-skills-priority-"));
const skillDir = join(root, "default-priority");
mkdirSync(skillDir, { recursive: true });
writeFileSync(
join(skillDir, "SKILL.md"),
[
"---",
"name: default-priority",
"description: Uses default priority.",
"version: 1",
"trustTier: T2",
"---",
"",
"body",
].join("\n"),
);
expect(SkillRegistry.load(root).get("default-priority")?.priority).toBe(50);
});
it("ignores invalid frontmatter", () => {
const root = mkdtempSync(join(tmpdir(), "lolympus-skills-invalid-"));
const skillDir = join(root, "bad");
+21
View File
@@ -1,5 +1,6 @@
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { getLogger } from "@obs/logger";
export type TrustTier = "T1" | "T2" | "T3" | "T4";
@@ -8,6 +9,7 @@ export interface SkillManifest {
description: string;
intents: string[];
name: string;
priority: number;
resources: Record<string, string>;
tools: string[];
triggers: string[];
@@ -123,6 +125,15 @@ function isTrustTier(value: unknown): value is TrustTier {
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
}
function isPriority(value: unknown): value is number {
return (
typeof value === "number" &&
Number.isInteger(value) &&
value >= 0 &&
value <= 100
);
}
function parseSkillFile(path: string): ParsedSkillFile | null {
const raw = readFileSync(path, "utf8");
const parsed = parseFrontmatter(raw);
@@ -133,6 +144,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
const skillDir = dirname(path);
const name = parsed.fields["name"];
const description = parsed.fields["description"];
const priority = parsed.fields["priority"] ?? 50;
const version = parsed.fields["version"];
const trustTier = parsed.fields["trustTier"];
@@ -141,6 +153,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
!name.trim() ||
typeof description !== "string" ||
!description.trim() ||
!isPriority(priority) ||
typeof version !== "number" ||
!isTrustTier(trustTier)
) {
@@ -154,6 +167,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
description,
intents: readStringArray(parsed.fields, "intents"),
name,
priority,
resources: readResources(parsed.fields, skillDir),
tools: readStringArray(parsed.fields, "tools"),
triggers: readStringArray(parsed.fields, "triggers"),
@@ -200,6 +214,13 @@ export class SkillRegistry {
for (const item of parsed) {
if (seen.has(item.manifest.name)) {
getLogger().warn(
{
path: item.manifest.bodyPath,
skill: item.manifest.name,
},
"[SkillRegistry] Duplicate skill manifest ignored",
);
continue;
}
seen.add(item.manifest.name);
+1
View File
@@ -6,6 +6,7 @@ import type {
export interface Skill {
readonly name: string;
readonly priority: number;
canHandle(request: AgentRequest): boolean;
handle(
request: AgentRequest,
+18
View File
@@ -0,0 +1,18 @@
import { describe, expect, it } from "vitest";
import { lookupRuleTool } from "./lookup-rule.js";
describe("lookupRuleTool", () => {
it("returns the matching community rule section", async () => {
const result = await lookupRuleTool.execute({ topic: "ads" }, {});
expect(result.source).toBe("community-rules");
expect(result.text).toContain("광고");
expect(result.text).toContain("홍보");
});
it("validates topic enum values", () => {
expect(() =>
lookupRuleTool.argsSchema.parse({ topic: "unknown" }),
).toThrow();
});
});
+55
View File
@@ -0,0 +1,55 @@
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { z } from "zod";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
topic: z.enum([
"ads",
"appeal",
"minor-safety",
"off-topic",
"privacy",
"profanity",
"report",
"sanctions",
"spam",
]),
});
const resultSchema = z.object({
topic: argsSchema.shape.topic,
text: z.string(),
source: z.literal("community-rules"),
});
type RuleTopic = z.infer<typeof argsSchema>["topic"];
const RULES_PATH = fileURLToPath(
new URL("../../skills/moderation/resources/community-rules.md", import.meta.url),
);
const RULES_TEXT = readFileSync(RULES_PATH, "utf8");
function extractRuleSection(topic: RuleTopic): string {
const pattern = new RegExp(`^## ${topic}\\n([\\s\\S]*?)(?=\\n## |$)`, "mu");
const match = RULES_TEXT.match(pattern);
return match?.[1]?.trim() || "";
}
export const lookupRuleTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "lookup-rule",
description:
"광고, 도배, 욕설, 신고, 제재, 이의 제기, 개인정보 같은 moderation 규칙을 조회한다.",
argsSchema,
resultSchema,
execute(args) {
return Promise.resolve({
topic: args.topic,
text: extractRuleSection(args.topic),
source: "community-rules",
});
},
};