Iterate skill SSOT routing and eval support
Add skill priority routing and duplicate manifest handling. Split eval golden sets by skill and add moderation rule lookup coverage. Move onboarding runtime helpers out of the main skill class.
This commit is contained in:
1 parent
fb189330da
commit
12e731a428
35 files changed
+1260
-500
No files matched your search
+2
-1
@@ -12,7 +12,8 @@
|
|||||||
"lint": "eslint .",
|
"lint": "eslint .",
|
||||||
"lint:fix": "eslint . --fix",
|
"lint:fix": "eslint . --fix",
|
||||||
"format": "prettier --write .",
|
"format": "prettier --write .",
|
||||||
"eval": "node --import tsx src/eval/cli.ts run --set=golden",
|
"eval": "node --import tsx src/eval/cli.ts run --set=onboarding/cases",
|
||||||
|
"eval:moderation": "node --import tsx src/eval/cli.ts run --set=moderation/cases",
|
||||||
"eval:capture": "node --import tsx src/eval/cli.ts capture",
|
"eval:capture": "node --import tsx src/eval/cli.ts capture",
|
||||||
"eval:baseline": "node --import tsx src/eval/cli.ts baseline",
|
"eval:baseline": "node --import tsx src/eval/cli.ts baseline",
|
||||||
"eval:report": "node --import tsx src/eval/cli.ts report",
|
"eval:report": "node --import tsx src/eval/cli.ts report",
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { Skill } from "@skills/skill";
|
||||||
|
import { selectSkillForRequest } from "./agent.js";
|
||||||
|
|
||||||
|
function makeRequest(skill = "shared"): AgentRequest {
|
||||||
|
return {
|
||||||
|
chatroomId: "room-1",
|
||||||
|
nickname: "tester",
|
||||||
|
requestId: "request-1",
|
||||||
|
skill,
|
||||||
|
trigger: "user-message",
|
||||||
|
userId: "user-1",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeSkill(name: string, priority: number, canHandle = true): Skill {
|
||||||
|
return {
|
||||||
|
name,
|
||||||
|
priority,
|
||||||
|
canHandle: () => canHandle,
|
||||||
|
handle: (
|
||||||
|
request: AgentRequest,
|
||||||
|
_conversationHistory: ConversationMessage[],
|
||||||
|
): Promise<AgentResponse> =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 1,
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: name,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("selectSkillForRequest", () => {
|
||||||
|
it("selects the highest-priority matching skill", () => {
|
||||||
|
const request = makeRequest();
|
||||||
|
const selected = selectSkillForRequest(
|
||||||
|
[
|
||||||
|
makeSkill("low", 10),
|
||||||
|
makeSkill("high", 90),
|
||||||
|
makeSkill("disabled", 100, false),
|
||||||
|
],
|
||||||
|
request,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(selected?.name).toBe("high");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null when no skill can handle the request", () => {
|
||||||
|
expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
+12
-1
@@ -25,6 +25,17 @@ function isValidRequest(data: unknown): data is AgentRequest {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function selectSkillForRequest(
|
||||||
|
skills: readonly Skill[],
|
||||||
|
request: AgentRequest,
|
||||||
|
): Skill | null {
|
||||||
|
return (
|
||||||
|
skills
|
||||||
|
.filter((candidate) => candidate.canHandle(request))
|
||||||
|
.sort((left, right) => right.priority - left.priority)[0] ?? null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export class Agent {
|
export class Agent {
|
||||||
private readonly skills: Skill[] = [];
|
private readonly skills: Skill[] = [];
|
||||||
private readonly logger = getLogger();
|
private readonly logger = getLogger();
|
||||||
@@ -96,7 +107,7 @@ export class Agent {
|
|||||||
"[Agent] Request received",
|
"[Agent] Request received",
|
||||||
);
|
);
|
||||||
|
|
||||||
const skill = this.skills.find((candidate) => candidate.canHandle(request));
|
const skill = selectSkillForRequest(this.skills, request);
|
||||||
if (!skill) {
|
if (!skill) {
|
||||||
logger.info("[Agent] No skill found for request");
|
logger.info("[Agent] No skill found for request");
|
||||||
const response: AgentResponse = {
|
const response: AgentResponse = {
|
||||||
|
|||||||
+3
-1
@@ -14,6 +14,7 @@ import {
|
|||||||
interface CaptureOptions {
|
interface CaptureOptions {
|
||||||
limit?: number | undefined;
|
limit?: number | undefined;
|
||||||
outputPath?: string | undefined;
|
outputPath?: string | undefined;
|
||||||
|
skill?: string | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function asRecord(value: unknown): Record<string, unknown> {
|
function asRecord(value: unknown): Record<string, unknown> {
|
||||||
@@ -122,7 +123,8 @@ export async function captureGoldenSetScaffold(
|
|||||||
|
|
||||||
const outputPath = resolve(
|
const outputPath = resolve(
|
||||||
process.cwd(),
|
process.cwd(),
|
||||||
options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl",
|
options?.outputPath ??
|
||||||
|
`src/eval/golden/${options?.skill ?? "onboarding"}/capture.jsonl`,
|
||||||
);
|
);
|
||||||
await mkdir(dirname(outputPath), { recursive: true });
|
await mkdir(dirname(outputPath), { recursive: true });
|
||||||
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
|
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
|
||||||
|
|||||||
+13
-5
@@ -319,39 +319,46 @@ async function main(): Promise<void> {
|
|||||||
parsePositiveInteger(getFlagString(flags, "limit")) ??
|
parsePositiveInteger(getFlagString(flags, "limit")) ??
|
||||||
parsePositiveInteger(positionals[0]),
|
parsePositiveInteger(positionals[0]),
|
||||||
outputPath: getFlagString(flags, "output") ?? positionals[1],
|
outputPath: getFlagString(flags, "output") ?? positionals[1],
|
||||||
|
skill: getFlagString(flags, "skill"),
|
||||||
});
|
});
|
||||||
logger.info({ outputPath }, '[eval] "capture" completed');
|
logger.info({ outputPath }, '[eval] "capture" completed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (command === "run") {
|
if (command === "run") {
|
||||||
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
|
const { loadEvalCases, resolveEvalSetPath, runEval } = await import(
|
||||||
"./runner.js"
|
"./runner.js"
|
||||||
);
|
);
|
||||||
const { publishEvalReportIfPossible } = await import(
|
const { publishEvalReportIfPossible } = await import(
|
||||||
"./runner.js"
|
"./runner.js"
|
||||||
);
|
);
|
||||||
const setName = getFlagString(flags, "set") ?? "golden";
|
const setName = getFlagString(flags, "set") ?? "onboarding/cases";
|
||||||
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
|
const resolvedSet = resolveEvalSetPath(setName);
|
||||||
|
const skill = getFlagString(flags, "skill") ?? resolvedSet.skill;
|
||||||
|
const inputPath = getFlagString(flags, "input") ?? resolvedSet.path;
|
||||||
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
|
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
|
||||||
|
|
||||||
if (hasFlag(flags, "dry-run")) {
|
if (hasFlag(flags, "dry-run")) {
|
||||||
const cases = await loadEvalCases(inputPath, { limit });
|
const cases = await loadEvalCases(inputPath, { limit });
|
||||||
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
|
console.info(
|
||||||
|
`[eval] dry-run skill=${skill} parsed ${cases.length} cases from ${inputPath}`,
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { report, reportPath } = await runOnboardingEval({
|
const { report, reportPath } = await runEval({
|
||||||
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
|
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
|
||||||
inputPath,
|
inputPath,
|
||||||
limit,
|
limit,
|
||||||
outputPath: getFlagString(flags, "output"),
|
outputPath: getFlagString(flags, "output"),
|
||||||
setName,
|
setName,
|
||||||
|
skill,
|
||||||
});
|
});
|
||||||
const publishedReport = hasFlag(flags, "no-publish-report")
|
const publishedReport = hasFlag(flags, "no-publish-report")
|
||||||
? false
|
? false
|
||||||
: await publishEvalReportIfPossible(report);
|
: await publishEvalReportIfPossible(report);
|
||||||
printRunSummary(report, reportPath);
|
printRunSummary(report, reportPath);
|
||||||
|
console.info(`[eval] skill=${skill}`);
|
||||||
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
|
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -414,6 +421,7 @@ async function main(): Promise<void> {
|
|||||||
const result = await appendCandidatesToJsonl(
|
const result = await appendCandidatesToJsonl(
|
||||||
candidates,
|
candidates,
|
||||||
getFlagString(flags, "output"),
|
getFlagString(flags, "output"),
|
||||||
|
getFlagString(flags, "skill"),
|
||||||
);
|
);
|
||||||
console.info(
|
console.info(
|
||||||
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
|
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
|
||||||
|
|||||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,5 @@
|
|||||||
|
{"id":"moderation_rule_ads","name":"moderation:rule-question:ads","input":{"request":{"requestId":"mod_req_ads","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_1","nickname":"문의자","trigger":"user-message","userMessage":"광고 올려도 돼요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["광고","홍보","허가","제한"],"mustNotInclude":["무조건 가능","바로 올려도"]}}
|
||||||
|
{"id":"moderation_rule_profanity","name":"moderation:rule-question:profanity","input":{"request":{"requestId":"mod_req_profanity","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_2","nickname":"문의자","trigger":"user-message","userMessage":"욕설하면 어떻게 돼요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["욕설","금지","운영진"],"mustNotInclude":["괜찮아요","상관없"]}}
|
||||||
|
{"id":"moderation_rule_spam","name":"moderation:rule-question:spam","input":{"request":{"requestId":"mod_req_spam","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_3","nickname":"문의자","trigger":"user-message","userMessage":"도배 기준이 뭐예요?"}},"expected":{"intent":"rule-question","mustIncludeAny":["도배","반복","제재"],"mustNotInclude":["기준 없음","상관없"]}}
|
||||||
|
{"id":"moderation_report_profanity","name":"moderation:violation-report:profanity","input":{"request":{"requestId":"mod_req_report","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_4","nickname":"제보자","trigger":"user-message","userMessage":"어떤 분이 욕했어요 신고하고 싶어요"}},"expected":{"intent":"violation-report","mustIncludeAny":["관련 메시지","상황","운영진"],"mustNotInclude":["제가 처리","제재 확정"]}}
|
||||||
|
{"id":"moderation_appeal_sanction","name":"moderation:appeal:sanction","input":{"request":{"requestId":"mod_req_appeal","skill":"moderation","chatroomId":"mod_room","userId":"mod_user_5","nickname":"소명자","trigger":"user-message","userMessage":"정지당했는데 억울해요"}},"expected":{"intent":"appeal","mustIncludeAny":["이의 제기","소명","운영진"],"mustNotInclude":["해제됐","풀렸"]}}
|
||||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -10,6 +10,113 @@ function round(value: number): number {
|
|||||||
return Number(value.toFixed(3));
|
return Number(value.toFixed(3));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function judgeRuleCase(
|
||||||
|
expected: EvalCaseExpected,
|
||||||
|
responseText: string,
|
||||||
|
actual: {
|
||||||
|
intent: string;
|
||||||
|
responseMode?: string | undefined;
|
||||||
|
},
|
||||||
|
): RuleJudgeResult {
|
||||||
|
const checks: RuleJudgeCheck[] = [];
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: "validator",
|
||||||
|
passed: true,
|
||||||
|
skipped: true,
|
||||||
|
detail: "not configured for this skill",
|
||||||
|
});
|
||||||
|
|
||||||
|
const expectedIntent = expected.intent?.trim();
|
||||||
|
if (expectedIntent) {
|
||||||
|
checks.push({
|
||||||
|
name: "intent",
|
||||||
|
passed: normalizeText(expectedIntent) === normalizeText(actual.intent),
|
||||||
|
skipped: false,
|
||||||
|
detail: `expected=${expectedIntent}, actual=${actual.intent}`,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
checks.push({
|
||||||
|
name: "intent",
|
||||||
|
passed: true,
|
||||||
|
skipped: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const expectedResponseMode = expected.responseMode?.trim();
|
||||||
|
if (expectedResponseMode && actual.responseMode) {
|
||||||
|
checks.push({
|
||||||
|
name: "responseMode",
|
||||||
|
passed:
|
||||||
|
normalizeText(expectedResponseMode) ===
|
||||||
|
normalizeText(actual.responseMode),
|
||||||
|
skipped: false,
|
||||||
|
detail: `expected=${expectedResponseMode}, actual=${actual.responseMode}`,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
checks.push({
|
||||||
|
name: "responseMode",
|
||||||
|
passed: true,
|
||||||
|
skipped: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
|
||||||
|
if (mustIncludeAny.length > 0) {
|
||||||
|
const normalizedResponse = normalizeText(responseText);
|
||||||
|
const matched = mustIncludeAny.find((candidate) =>
|
||||||
|
normalizedResponse.includes(normalizeText(candidate)),
|
||||||
|
);
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: "mustIncludeAny",
|
||||||
|
passed: Boolean(matched),
|
||||||
|
skipped: false,
|
||||||
|
detail: matched
|
||||||
|
? `matched=${matched}`
|
||||||
|
: `expected one of: ${mustIncludeAny.join(", ")}`,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
checks.push({
|
||||||
|
name: "mustIncludeAny",
|
||||||
|
passed: true,
|
||||||
|
skipped: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
|
||||||
|
if (mustNotInclude.length > 0) {
|
||||||
|
const normalizedResponse = normalizeText(responseText);
|
||||||
|
const matched = mustNotInclude.find((candidate) =>
|
||||||
|
normalizedResponse.includes(normalizeText(candidate)),
|
||||||
|
);
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: "mustNotInclude",
|
||||||
|
passed: !matched,
|
||||||
|
skipped: false,
|
||||||
|
detail: matched ? `forbidden=${matched}` : undefined,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
checks.push({
|
||||||
|
name: "mustNotInclude",
|
||||||
|
passed: true,
|
||||||
|
skipped: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeChecks = checks.filter((check) => !check.skipped);
|
||||||
|
const passedChecks = activeChecks.filter((check) => check.passed).length;
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed: activeChecks.every((check) => check.passed),
|
||||||
|
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
||||||
|
checks,
|
||||||
|
validatorStatus: "pass",
|
||||||
|
validatorReasons: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function judgeOnboardingRuleCase(
|
export function judgeOnboardingRuleCase(
|
||||||
expected: EvalCaseExpected,
|
expected: EvalCaseExpected,
|
||||||
responseText: string,
|
responseText: string,
|
||||||
|
|||||||
+7
-3
@@ -20,7 +20,6 @@ import { getEvalArchiveCollection } from "./archive.js";
|
|||||||
import { queryArchivedReviewLabels } from "./review/label-archive.js";
|
import { queryArchivedReviewLabels } from "./review/label-archive.js";
|
||||||
import type { ReviewLabel } from "./review/queue.js";
|
import type { ReviewLabel } from "./review/queue.js";
|
||||||
|
|
||||||
const DEFAULT_CANDIDATE_PATH = "src/eval/golden/onboarding.candidate.jsonl";
|
|
||||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
export interface PromotionCandidate {
|
export interface PromotionCandidate {
|
||||||
@@ -47,6 +46,10 @@ function stableShortHash(value: string): string {
|
|||||||
return createHash("sha256").update(value).digest("hex").slice(0, 8);
|
return createHash("sha256").update(value).digest("hex").slice(0, 8);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function defaultCandidatePathFor(skill: string): string {
|
||||||
|
return `src/eval/golden/${skill}/candidate.jsonl`;
|
||||||
|
}
|
||||||
|
|
||||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||||
if (entry.llmJudge.skipped) {
|
if (entry.llmJudge.skipped) {
|
||||||
return null;
|
return null;
|
||||||
@@ -326,9 +329,10 @@ function extractExistingCaseIds(raw: string): Set<string> {
|
|||||||
|
|
||||||
export async function appendCandidatesToJsonl(
|
export async function appendCandidatesToJsonl(
|
||||||
candidates: PromotionCandidate[],
|
candidates: PromotionCandidate[],
|
||||||
outputPath = DEFAULT_CANDIDATE_PATH,
|
outputPath?: string,
|
||||||
|
skill = "onboarding",
|
||||||
): Promise<{ appendedCount: number; outputPath: string }> {
|
): Promise<{ appendedCount: number; outputPath: string }> {
|
||||||
const resolved = resolve(process.cwd(), outputPath);
|
const resolved = resolve(process.cwd(), outputPath ?? defaultCandidatePathFor(skill));
|
||||||
await mkdir(dirname(resolved), { recursive: true });
|
await mkdir(dirname(resolved), { recursive: true });
|
||||||
const existingRaw = existsSync(resolved)
|
const existingRaw = existsSync(resolved)
|
||||||
? await readFile(resolved, "utf8")
|
? await readFile(resolved, "utf8")
|
||||||
|
|||||||
+27
-1
@@ -1,7 +1,7 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||||
import { publishEvalReportIfPossible } from "./runner.js";
|
import { publishEvalReportIfPossible, resolveEvalSetPath } from "./runner.js";
|
||||||
import type { EvalReport } from "./types.js";
|
import type { EvalReport } from "./types.js";
|
||||||
|
|
||||||
const natsMocks = vi.hoisted(() => ({
|
const natsMocks = vi.hoisted(() => ({
|
||||||
@@ -54,3 +54,29 @@ describe("publishEvalReportIfPossible", () => {
|
|||||||
expect(natsMocks.close).toHaveBeenCalledTimes(1);
|
expect(natsMocks.close).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("resolveEvalSetPath", () => {
|
||||||
|
it("resolves skill/subset eval sets", () => {
|
||||||
|
const resolved = resolveEvalSetPath("moderation/cases");
|
||||||
|
|
||||||
|
expect(resolved).toMatchObject({
|
||||||
|
skill: "moderation",
|
||||||
|
subset: "cases",
|
||||||
|
});
|
||||||
|
expect(resolved.path).toContain("src/eval/golden/moderation/cases.jsonl");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps legacy golden set to onboarding cases", () => {
|
||||||
|
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||||
|
|
||||||
|
const resolved = resolveEvalSetPath("golden");
|
||||||
|
|
||||||
|
expect(resolved.skill).toBe("onboarding");
|
||||||
|
expect(resolved.subset).toBe("cases");
|
||||||
|
expect(warn).toHaveBeenCalledWith(
|
||||||
|
"[eval] --set=golden is deprecated; use --set=onboarding/cases",
|
||||||
|
);
|
||||||
|
|
||||||
|
warn.mockRestore();
|
||||||
|
});
|
||||||
|
});
|
||||||
+314
-122
@@ -13,6 +13,9 @@ import { runWithContext } from "@obs/context";
|
|||||||
import { getLogger } from "@obs/logger";
|
import { getLogger } from "@obs/logger";
|
||||||
import type { RoomMessage } from "@room/types";
|
import type { RoomMessage } from "@room/types";
|
||||||
import { RoomContextService } from "@room/room-context-service";
|
import { RoomContextService } from "@room/room-context-service";
|
||||||
|
import type { Skill } from "@skills/skill";
|
||||||
|
import { ModerationSkill } from "@skills/moderation/index";
|
||||||
|
import { classifyModerationIntent } from "@skills/moderation/intents";
|
||||||
import { OnboardingSkill } from "@skills/onboarding/index";
|
import { OnboardingSkill } from "@skills/onboarding/index";
|
||||||
import {
|
import {
|
||||||
ProcessTraceCollector,
|
ProcessTraceCollector,
|
||||||
@@ -29,7 +32,10 @@ import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
|||||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
||||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||||
import { judgeProcessRule } from "./judges/process-judge.js";
|
import { judgeProcessRule } from "./judges/process-judge.js";
|
||||||
import { judgeOnboardingRuleCase } from "./judges/rule-judge.js";
|
import {
|
||||||
|
judgeOnboardingRuleCase,
|
||||||
|
judgeRuleCase,
|
||||||
|
} from "./judges/rule-judge.js";
|
||||||
import type {
|
import type {
|
||||||
EvalBaselineFile,
|
EvalBaselineFile,
|
||||||
EvalCaseDefinition,
|
EvalCaseDefinition,
|
||||||
@@ -44,11 +50,32 @@ import type {
|
|||||||
} from "./types.js";
|
} from "./types.js";
|
||||||
|
|
||||||
const DEFAULT_REPORTS_DIR = "eval-reports";
|
const DEFAULT_REPORTS_DIR = "eval-reports";
|
||||||
const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
|
const LEGACY_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
|
||||||
const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl";
|
|
||||||
const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl";
|
|
||||||
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
||||||
|
|
||||||
|
export interface ResolvedEvalSetPath {
|
||||||
|
path: string;
|
||||||
|
skill: string;
|
||||||
|
subset: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SkillRunnerDeps {
|
||||||
|
garnet: GarnetClient;
|
||||||
|
llm: LlmClient;
|
||||||
|
room?: RoomContextService | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> = {
|
||||||
|
moderation: (deps) => new ModerationSkill(deps.llm),
|
||||||
|
onboarding: (deps) =>
|
||||||
|
new OnboardingSkill(
|
||||||
|
deps.garnet,
|
||||||
|
deps.llm,
|
||||||
|
deps.room,
|
||||||
|
noopShadowEvalPublisher,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
class InMemoryGarnetClient extends GarnetClient {
|
class InMemoryGarnetClient extends GarnetClient {
|
||||||
private readonly hashStore = new Map<string, Record<string, string>>();
|
private readonly hashStore = new Map<string, Record<string, string>>();
|
||||||
private readonly stringStore = new Map<string, string>();
|
private readonly stringStore = new Map<string, string>();
|
||||||
@@ -127,6 +154,7 @@ export interface EvalRunOptions {
|
|||||||
limit?: number | undefined;
|
limit?: number | undefined;
|
||||||
outputPath?: string | undefined;
|
outputPath?: string | undefined;
|
||||||
setName?: string | undefined;
|
setName?: string | undefined;
|
||||||
|
skill?: string | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function round(value: number): number {
|
function round(value: number): number {
|
||||||
@@ -580,30 +608,72 @@ async function writeReport(
|
|||||||
return resolved;
|
return resolved;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadBaselineFile(): Promise<EvalBaselineFile | null> {
|
function baselinePathFor(skill: string): string {
|
||||||
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
|
return `${DEFAULT_REPORTS_DIR}/baseline.${skill}.json`;
|
||||||
if (!existsSync(baselinePath)) {
|
}
|
||||||
|
|
||||||
|
async function loadBaselineFile(skill: string): Promise<EvalBaselineFile | null> {
|
||||||
|
const baselinePath = resolve(process.cwd(), baselinePathFor(skill));
|
||||||
|
const legacyBaselinePath = resolve(process.cwd(), LEGACY_BASELINE_FILE);
|
||||||
|
const path = existsSync(baselinePath)
|
||||||
|
? baselinePath
|
||||||
|
: skill === "onboarding" && existsSync(legacyBaselinePath)
|
||||||
|
? legacyBaselinePath
|
||||||
|
: null;
|
||||||
|
if (!path) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const raw = await readFile(baselinePath, "utf8");
|
const raw = await readFile(path, "utf8");
|
||||||
return JSON.parse(raw) as EvalBaselineFile;
|
return JSON.parse(raw) as EvalBaselineFile;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveEvalSetPath(setNameOrPath = "golden"): string {
|
export function resolveEvalSetPath(
|
||||||
|
setNameOrPath = "onboarding/cases",
|
||||||
|
): ResolvedEvalSetPath {
|
||||||
if (setNameOrPath === "golden") {
|
if (setNameOrPath === "golden") {
|
||||||
return resolve(process.cwd(), GOLDEN_SET_PATH);
|
console.warn(
|
||||||
|
'[eval] --set=golden is deprecated; use --set=onboarding/cases',
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
path: resolve(process.cwd(), "src/eval/golden/onboarding/cases.jsonl"),
|
||||||
|
skill: "onboarding",
|
||||||
|
subset: "cases",
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (setNameOrPath === "capture") {
|
if (setNameOrPath === "capture") {
|
||||||
return resolve(process.cwd(), CAPTURE_SET_PATH);
|
console.warn(
|
||||||
|
'[eval] --set=capture is deprecated; use --set=onboarding/capture',
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
path: resolve(process.cwd(), "src/eval/golden/onboarding/capture.jsonl"),
|
||||||
|
skill: "onboarding",
|
||||||
|
subset: "capture",
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) {
|
if (setNameOrPath.endsWith(".jsonl")) {
|
||||||
return resolve(process.cwd(), setNameOrPath);
|
return {
|
||||||
|
path: resolve(process.cwd(), setNameOrPath),
|
||||||
|
skill: "onboarding",
|
||||||
|
subset: basename(setNameOrPath, ".jsonl"),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`);
|
const [skill, subset = "cases"] = setNameOrPath.includes("/")
|
||||||
|
? setNameOrPath.split("/", 2)
|
||||||
|
: [setNameOrPath, "cases"];
|
||||||
|
|
||||||
|
if (!skill?.trim() || !subset.trim()) {
|
||||||
|
throw new Error(`Invalid eval set name: ${setNameOrPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
path: resolve(process.cwd(), "src/eval/golden", skill, `${subset}.jsonl`),
|
||||||
|
skill,
|
||||||
|
subset,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function loadEvalCases(
|
export async function loadEvalCases(
|
||||||
@@ -632,6 +702,7 @@ async function evaluateCase(
|
|||||||
evalCase: EvalCaseDefinition,
|
evalCase: EvalCaseDefinition,
|
||||||
llmClient: LlmClient,
|
llmClient: LlmClient,
|
||||||
enableLlmJudge: boolean,
|
enableLlmJudge: boolean,
|
||||||
|
skillName: string,
|
||||||
): Promise<EvalCaseResult> {
|
): Promise<EvalCaseResult> {
|
||||||
const input = evalCase.input;
|
const input = evalCase.input;
|
||||||
const inMemoryGarnet = new InMemoryGarnetClient();
|
const inMemoryGarnet = new InMemoryGarnetClient();
|
||||||
@@ -641,14 +712,16 @@ async function evaluateCase(
|
|||||||
input.roomContext && input.roomContext.length > 0
|
input.roomContext && input.roomContext.length > 0
|
||||||
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
|
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
|
||||||
: undefined;
|
: undefined;
|
||||||
const skill = new OnboardingSkill(
|
const effectiveSkill = input.request.skill || skillName;
|
||||||
inMemoryGarnet,
|
const factory = SKILL_RUNNER_FACTORIES[effectiveSkill];
|
||||||
llmClient,
|
if (!factory) {
|
||||||
roomContextService,
|
throw new Error(`No eval runner factory for skill: ${effectiveSkill}`);
|
||||||
noopShadowEvalPublisher,
|
}
|
||||||
);
|
const skill = factory({
|
||||||
const classifier = new OnboardingIntentClassifier(llmClient);
|
garnet: inMemoryGarnet,
|
||||||
const processTrace = new ProcessTraceCollector();
|
llm: llmClient,
|
||||||
|
room: roomContextService,
|
||||||
|
});
|
||||||
|
|
||||||
return runWithContext(
|
return runWithContext(
|
||||||
{
|
{
|
||||||
@@ -660,109 +733,219 @@ async function evaluateCase(
|
|||||||
trigger: input.request.trigger,
|
trigger: input.request.trigger,
|
||||||
userId: input.request.userId,
|
userId: input.request.userId,
|
||||||
},
|
},
|
||||||
() => runWithProcessTrace(processTrace, async () => {
|
() =>
|
||||||
const classified = await classifier.classify(
|
effectiveSkill === "onboarding"
|
||||||
input.request.userMessage,
|
? evaluateOnboardingCase(
|
||||||
input.request.trigger,
|
evalCase,
|
||||||
);
|
input,
|
||||||
const derived = deriveState(
|
skill,
|
||||||
input.session ?? null,
|
llmClient,
|
||||||
input.request.trigger,
|
enableLlmJudge,
|
||||||
classified.intent,
|
)
|
||||||
);
|
: evaluateGenericSkillCase(evalCase, input, skill),
|
||||||
|
|
||||||
const startedAt = Date.now();
|
|
||||||
try {
|
|
||||||
const response = await skill.handle(
|
|
||||||
input.request,
|
|
||||||
input.conversationHistory ?? [],
|
|
||||||
);
|
|
||||||
const latencyMs = Date.now() - startedAt;
|
|
||||||
const derivedSnapshot = toDerivedSnapshot(derived);
|
|
||||||
const ruleJudge = judgeOnboardingRuleCase(
|
|
||||||
evalCase.expected,
|
|
||||||
response.text,
|
|
||||||
derived,
|
|
||||||
);
|
|
||||||
const llmJudge = enableLlmJudge
|
|
||||||
? await judgeOnboardingWithLlm(
|
|
||||||
llmClient,
|
|
||||||
evalCase,
|
|
||||||
response.text,
|
|
||||||
derivedSnapshot,
|
|
||||||
)
|
|
||||||
: {
|
|
||||||
skipped: true as const,
|
|
||||||
reason: "disabled",
|
|
||||||
};
|
|
||||||
const processTraceSnapshot = processTrace.snapshot();
|
|
||||||
const processJudge = judgeProcessRule(
|
|
||||||
evalCase.expected,
|
|
||||||
processTraceSnapshot,
|
|
||||||
);
|
|
||||||
|
|
||||||
return {
|
|
||||||
id: evalCase.id,
|
|
||||||
lineNumber: evalCase.lineNumber,
|
|
||||||
name: evalCase.name,
|
|
||||||
latencyMs,
|
|
||||||
classified: toClassifiedSnapshot(classified),
|
|
||||||
derived: derivedSnapshot,
|
|
||||||
request: input.request,
|
|
||||||
response,
|
|
||||||
ruleJudge,
|
|
||||||
llmJudge,
|
|
||||||
processJudge,
|
|
||||||
processTrace: processTraceSnapshot,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
const latencyMs = Date.now() - startedAt;
|
|
||||||
const errorMessage =
|
|
||||||
error instanceof Error ? error.message : String(error);
|
|
||||||
const processTraceSnapshot = processTrace.snapshot();
|
|
||||||
|
|
||||||
return {
|
|
||||||
id: evalCase.id,
|
|
||||||
lineNumber: evalCase.lineNumber,
|
|
||||||
name: evalCase.name,
|
|
||||||
latencyMs,
|
|
||||||
classified: toClassifiedSnapshot(classified),
|
|
||||||
derived: toDerivedSnapshot(derived),
|
|
||||||
request: input.request,
|
|
||||||
response: {
|
|
||||||
requestId: input.request.requestId,
|
|
||||||
text: "",
|
|
||||||
confidence: 0,
|
|
||||||
source: "template",
|
|
||||||
metadata: {
|
|
||||||
error: errorMessage,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
|
|
||||||
llmJudge: {
|
|
||||||
skipped: true,
|
|
||||||
reason: "skill-error",
|
|
||||||
},
|
|
||||||
processJudge: judgeProcessRule(
|
|
||||||
evalCase.expected,
|
|
||||||
processTraceSnapshot,
|
|
||||||
),
|
|
||||||
processTrace: processTraceSnapshot,
|
|
||||||
error: errorMessage,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runOnboardingEval(
|
async function evaluateOnboardingCase(
|
||||||
|
evalCase: EvalCaseDefinition,
|
||||||
|
input: EvalCaseInput,
|
||||||
|
skill: Skill,
|
||||||
|
llmClient: LlmClient,
|
||||||
|
enableLlmJudge: boolean,
|
||||||
|
): Promise<EvalCaseResult> {
|
||||||
|
const classifier = new OnboardingIntentClassifier(llmClient);
|
||||||
|
const processTrace = new ProcessTraceCollector();
|
||||||
|
|
||||||
|
return runWithProcessTrace(processTrace, async () => {
|
||||||
|
const classified = await classifier.classify(
|
||||||
|
input.request.userMessage,
|
||||||
|
input.request.trigger,
|
||||||
|
);
|
||||||
|
const derived = deriveState(
|
||||||
|
input.session ?? null,
|
||||||
|
input.request.trigger,
|
||||||
|
classified.intent,
|
||||||
|
);
|
||||||
|
|
||||||
|
const startedAt = Date.now();
|
||||||
|
try {
|
||||||
|
const response = await skill.handle(
|
||||||
|
input.request,
|
||||||
|
input.conversationHistory ?? [],
|
||||||
|
);
|
||||||
|
const latencyMs = Date.now() - startedAt;
|
||||||
|
const derivedSnapshot = toDerivedSnapshot(derived);
|
||||||
|
const ruleJudge = judgeOnboardingRuleCase(
|
||||||
|
evalCase.expected,
|
||||||
|
response.text,
|
||||||
|
derived,
|
||||||
|
);
|
||||||
|
const llmJudge = enableLlmJudge
|
||||||
|
? await judgeOnboardingWithLlm(
|
||||||
|
llmClient,
|
||||||
|
evalCase,
|
||||||
|
response.text,
|
||||||
|
derivedSnapshot,
|
||||||
|
)
|
||||||
|
: {
|
||||||
|
skipped: true as const,
|
||||||
|
reason: "disabled",
|
||||||
|
};
|
||||||
|
const processTraceSnapshot = processTrace.snapshot();
|
||||||
|
const processJudge = judgeProcessRule(
|
||||||
|
evalCase.expected,
|
||||||
|
processTraceSnapshot,
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: evalCase.id,
|
||||||
|
lineNumber: evalCase.lineNumber,
|
||||||
|
name: evalCase.name,
|
||||||
|
latencyMs,
|
||||||
|
classified: toClassifiedSnapshot(classified),
|
||||||
|
derived: derivedSnapshot,
|
||||||
|
request: input.request,
|
||||||
|
response,
|
||||||
|
ruleJudge,
|
||||||
|
llmJudge,
|
||||||
|
processJudge,
|
||||||
|
processTrace: processTraceSnapshot,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const latencyMs = Date.now() - startedAt;
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : String(error);
|
||||||
|
const processTraceSnapshot = processTrace.snapshot();
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: evalCase.id,
|
||||||
|
lineNumber: evalCase.lineNumber,
|
||||||
|
name: evalCase.name,
|
||||||
|
latencyMs,
|
||||||
|
classified: toClassifiedSnapshot(classified),
|
||||||
|
derived: toDerivedSnapshot(derived),
|
||||||
|
request: input.request,
|
||||||
|
response: {
|
||||||
|
requestId: input.request.requestId,
|
||||||
|
text: "",
|
||||||
|
confidence: 0,
|
||||||
|
source: "template",
|
||||||
|
metadata: {
|
||||||
|
error: errorMessage,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
|
||||||
|
llmJudge: {
|
||||||
|
skipped: true,
|
||||||
|
reason: "skill-error",
|
||||||
|
},
|
||||||
|
processJudge: judgeProcessRule(
|
||||||
|
evalCase.expected,
|
||||||
|
processTraceSnapshot,
|
||||||
|
),
|
||||||
|
processTrace: processTraceSnapshot,
|
||||||
|
error: errorMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function evaluateGenericSkillCase(
|
||||||
|
evalCase: EvalCaseDefinition,
|
||||||
|
input: EvalCaseInput,
|
||||||
|
skill: Skill,
|
||||||
|
): Promise<EvalCaseResult> {
|
||||||
|
const classified = classifyModerationIntent(input.request.userMessage ?? "");
|
||||||
|
const derived: EvalDerivedSnapshot = {
|
||||||
|
allowShortResponse: true,
|
||||||
|
intent: classified.intent.kind,
|
||||||
|
phase: skill.name,
|
||||||
|
responseMode: classified.intent.kind === "off-topic" ? "template" : "llm",
|
||||||
|
shouldRespond: classified.intent.kind !== "off-topic",
|
||||||
|
suggestedMaxTokens: 192,
|
||||||
|
tier: 0,
|
||||||
|
};
|
||||||
|
const startedAt = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await skill.handle(
|
||||||
|
input.request,
|
||||||
|
input.conversationHistory ?? [],
|
||||||
|
);
|
||||||
|
const latencyMs = Date.now() - startedAt;
|
||||||
|
return {
|
||||||
|
id: evalCase.id,
|
||||||
|
lineNumber: evalCase.lineNumber,
|
||||||
|
name: evalCase.name,
|
||||||
|
latencyMs,
|
||||||
|
classified: {
|
||||||
|
confidence: classified.confidence,
|
||||||
|
intent: classified.intent.kind,
|
||||||
|
source: "regex",
|
||||||
|
},
|
||||||
|
derived,
|
||||||
|
request: input.request,
|
||||||
|
response,
|
||||||
|
ruleJudge: judgeRuleCase(evalCase.expected, response.text, {
|
||||||
|
intent: classified.intent.kind,
|
||||||
|
responseMode: derived.responseMode,
|
||||||
|
}),
|
||||||
|
llmJudge: {
|
||||||
|
skipped: true,
|
||||||
|
reason: "unsupported-skill",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const latencyMs = Date.now() - startedAt;
|
||||||
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||||
|
return {
|
||||||
|
id: evalCase.id,
|
||||||
|
lineNumber: evalCase.lineNumber,
|
||||||
|
name: evalCase.name,
|
||||||
|
latencyMs,
|
||||||
|
classified: {
|
||||||
|
confidence: classified.confidence,
|
||||||
|
intent: classified.intent.kind,
|
||||||
|
source: "regex",
|
||||||
|
},
|
||||||
|
derived,
|
||||||
|
request: input.request,
|
||||||
|
response: {
|
||||||
|
requestId: input.request.requestId,
|
||||||
|
text: "",
|
||||||
|
confidence: 0,
|
||||||
|
source: "template",
|
||||||
|
metadata: {
|
||||||
|
error: errorMessage,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ruleJudge: judgeRuleCase(evalCase.expected, "", {
|
||||||
|
intent: classified.intent.kind,
|
||||||
|
responseMode: derived.responseMode,
|
||||||
|
}),
|
||||||
|
llmJudge: {
|
||||||
|
skipped: true,
|
||||||
|
reason: "skill-error",
|
||||||
|
},
|
||||||
|
error: errorMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runEval(
|
||||||
options?: EvalRunOptions,
|
options?: EvalRunOptions,
|
||||||
): Promise<{ report: EvalReport; reportPath: string }> {
|
): Promise<{ report: EvalReport; reportPath: string }> {
|
||||||
const logger = getLogger();
|
const logger = getLogger();
|
||||||
const setName = options?.setName ?? "golden";
|
const setName = options?.setName ?? "onboarding/cases";
|
||||||
const inputPath = options?.inputPath
|
const resolvedSet = options?.inputPath
|
||||||
? resolve(process.cwd(), options.inputPath)
|
? {
|
||||||
|
path: resolve(process.cwd(), options.inputPath),
|
||||||
|
skill: options.skill ?? "onboarding",
|
||||||
|
subset: basename(options.inputPath, ".jsonl"),
|
||||||
|
}
|
||||||
: resolveEvalSetPath(setName);
|
: resolveEvalSetPath(setName);
|
||||||
|
const skill = options?.skill ?? resolvedSet.skill;
|
||||||
|
const inputPath = resolvedSet.path;
|
||||||
const evalCases = await loadEvalCases(inputPath, {
|
const evalCases = await loadEvalCases(inputPath, {
|
||||||
limit: options?.limit,
|
limit: options?.limit,
|
||||||
});
|
});
|
||||||
@@ -772,11 +955,11 @@ export async function runOnboardingEval(
|
|||||||
const caseResults: EvalCaseResult[] = [];
|
const caseResults: EvalCaseResult[] = [];
|
||||||
|
|
||||||
for (const evalCase of evalCases) {
|
for (const evalCase of evalCases) {
|
||||||
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge));
|
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge, skill));
|
||||||
}
|
}
|
||||||
|
|
||||||
const summary = createSummary(caseResults);
|
const summary = createSummary(caseResults);
|
||||||
const baseline = await loadBaselineFile();
|
const baseline = await loadBaselineFile(skill);
|
||||||
const report: EvalReport = {
|
const report: EvalReport = {
|
||||||
generatedAt: new Date().toISOString(),
|
generatedAt: new Date().toISOString(),
|
||||||
inputPath,
|
inputPath,
|
||||||
@@ -798,6 +981,7 @@ export async function runOnboardingEval(
|
|||||||
cases: summary.cases,
|
cases: summary.cases,
|
||||||
passRate: summary.passRate,
|
passRate: summary.passRate,
|
||||||
reportPath,
|
reportPath,
|
||||||
|
skill,
|
||||||
},
|
},
|
||||||
"[Eval] run completed",
|
"[Eval] run completed",
|
||||||
);
|
);
|
||||||
@@ -805,6 +989,12 @@ export async function runOnboardingEval(
|
|||||||
return { report, reportPath };
|
return { report, reportPath };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function runOnboardingEval(
|
||||||
|
options?: EvalRunOptions,
|
||||||
|
): Promise<{ report: EvalReport; reportPath: string }> {
|
||||||
|
return runEval({ ...options, skill: options?.skill ?? "onboarding" });
|
||||||
|
}
|
||||||
|
|
||||||
export async function findLatestEvalReportPath(): Promise<string | null> {
|
export async function findLatestEvalReportPath(): Promise<string | null> {
|
||||||
const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR);
|
const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR);
|
||||||
if (!existsSync(reportsDir)) {
|
if (!existsSync(reportsDir)) {
|
||||||
@@ -815,7 +1005,8 @@ export async function findLatestEvalReportPath(): Promise<string | null> {
|
|||||||
const candidates = entries
|
const candidates = entries
|
||||||
.filter((entry) => entry.isFile())
|
.filter((entry) => entry.isFile())
|
||||||
.map((entry) => entry.name)
|
.map((entry) => entry.name)
|
||||||
.filter((name) => name.endsWith(".json") && name !== "baseline.json")
|
.filter((name) => name.endsWith(".json") && !name.startsWith("baseline."))
|
||||||
|
.filter((name) => name !== "baseline.json")
|
||||||
.sort((left, right) => right.localeCompare(left));
|
.sort((left, right) => right.localeCompare(left));
|
||||||
|
|
||||||
if (candidates.length === 0) {
|
if (candidates.length === 0) {
|
||||||
@@ -845,6 +1036,7 @@ export async function updateEvalBaseline(
|
|||||||
|
|
||||||
const raw = await readFile(sourcePath, "utf8");
|
const raw = await readFile(sourcePath, "utf8");
|
||||||
const report = JSON.parse(raw) as EvalReport;
|
const report = JSON.parse(raw) as EvalReport;
|
||||||
|
const skill = resolveEvalSetPath(report.setName).skill;
|
||||||
const baseline: EvalBaselineFile = {
|
const baseline: EvalBaselineFile = {
|
||||||
reportPath: sourcePath,
|
reportPath: sourcePath,
|
||||||
setName: report.setName,
|
setName: report.setName,
|
||||||
@@ -852,7 +1044,7 @@ export async function updateEvalBaseline(
|
|||||||
updatedAt: new Date().toISOString(),
|
updatedAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
|
||||||
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
|
const baselinePath = resolve(process.cwd(), baselinePathFor(skill));
|
||||||
await mkdir(dirname(baselinePath), { recursive: true });
|
await mkdir(dirname(baselinePath), { recursive: true });
|
||||||
await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
|
await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
|
||||||
|
|
||||||
|
|||||||
@@ -80,7 +80,10 @@ const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
|
|||||||
const BACKGROUND_GROUP = "bg-worker";
|
const BACKGROUND_GROUP = "bg-worker";
|
||||||
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
||||||
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
||||||
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
|
const EVAL_BASELINE_FILES = [
|
||||||
|
"eval-reports/baseline.onboarding.json",
|
||||||
|
"eval-reports/baseline.json",
|
||||||
|
];
|
||||||
const ALERT_WINDOW_MS = 5 * 60_000;
|
const ALERT_WINDOW_MS = 5 * 60_000;
|
||||||
const ALERT_WINDOW_LABEL = "5m";
|
const ALERT_WINDOW_LABEL = "5m";
|
||||||
const ALERT_TICK_MS = 60_000;
|
const ALERT_TICK_MS = 60_000;
|
||||||
@@ -1123,8 +1126,10 @@ export class ShadowWorker {
|
|||||||
return redisParsed;
|
return redisParsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
|
const filePath = EVAL_BASELINE_FILES
|
||||||
if (!existsSync(filePath)) {
|
.map((candidate) => resolve(process.cwd(), candidate))
|
||||||
|
.find((candidate) => existsSync(candidate));
|
||||||
|
if (!filePath) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+10
@@ -228,7 +228,17 @@ async function bootstrap(): Promise<void> {
|
|||||||
moderation: createModerationSkill,
|
moderation: createModerationSkill,
|
||||||
onboarding: createOnboardingSkill,
|
onboarding: createOnboardingSkill,
|
||||||
};
|
};
|
||||||
|
const seenSkills = new Set<string>();
|
||||||
for (const manifest of skillRegistry.list()) {
|
for (const manifest of skillRegistry.list()) {
|
||||||
|
if (seenSkills.has(manifest.name)) {
|
||||||
|
logger.warn(
|
||||||
|
{ skill: manifest.name },
|
||||||
|
"[Skill] Duplicate manifest ignored",
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seenSkills.add(manifest.name);
|
||||||
|
|
||||||
const factory = factories[manifest.name];
|
const factory = factories[manifest.name];
|
||||||
if (!factory) {
|
if (!factory) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
|
|||||||
@@ -2,10 +2,12 @@
|
|||||||
name: moderation
|
name: moderation
|
||||||
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
|
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
|
||||||
version: 1
|
version: 1
|
||||||
|
priority: 50
|
||||||
intents: [greeting, rule-question, violation-report, appeal, off-topic]
|
intents: [greeting, rule-question, violation-report, appeal, off-topic]
|
||||||
triggers: [user-message]
|
triggers: [user-message]
|
||||||
tools: []
|
tools: [lookup-rule]
|
||||||
resources: {}
|
resources:
|
||||||
|
community-rules: ./resources/community-rules.md
|
||||||
trustTier: T4
|
trustTier: T4
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -6,19 +6,21 @@ interface ModerationIntentDefinition {
|
|||||||
export const MODERATION_INTENT_TABLE = {
|
export const MODERATION_INTENT_TABLE = {
|
||||||
greeting: {
|
greeting: {
|
||||||
confidence: 0.9,
|
confidence: 0.9,
|
||||||
regex: [/^(안녕|하이|반갑|문의드립니다)/u],
|
regex: [/^(안녕|하이|반갑|문의드립|운영진|모더|관리자)/u],
|
||||||
},
|
},
|
||||||
"rule-question": {
|
"rule-question": {
|
||||||
confidence: 0.92,
|
confidence: 0.92,
|
||||||
regex: [/(규칙|룰|금지|가능|해도\s*돼|하면\s*안|제재|경고)/u],
|
regex: [
|
||||||
|
/(규칙|룰|금지|가능|해도\s*돼|하면\s*안|제재|경고|광고|홍보|도배|링크|욕설|혐오|차별)/u,
|
||||||
|
],
|
||||||
},
|
},
|
||||||
"violation-report": {
|
"violation-report": {
|
||||||
confidence: 0.94,
|
confidence: 0.94,
|
||||||
regex: [/(신고|제보|위반|욕설|분쟁|비매너|문제\s*있|처리해)/u],
|
regex: [/(신고|제보|위반|욕설|분쟁|비매너|문제\s*있|처리해|누가|사람이|유저가|이용자)/u],
|
||||||
},
|
},
|
||||||
appeal: {
|
appeal: {
|
||||||
confidence: 0.94,
|
confidence: 0.94,
|
||||||
regex: [/(이의|소명|억울|해제|풀어|재검토|항소|정지)/u],
|
regex: [/(이의|소명|억울|해제|풀어|재검토|항소|정지|벤|밴|차단\s*풀|차단)/u],
|
||||||
},
|
},
|
||||||
"off-topic": {
|
"off-topic": {
|
||||||
confidence: 0,
|
confidence: 0,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ export const MODERATION_PROMPT_DATA = {
|
|||||||
policy: {
|
policy: {
|
||||||
base: [
|
base: [
|
||||||
"- 답변은 1~3문장으로 짧게 한다.",
|
"- 답변은 1~3문장으로 짧게 한다.",
|
||||||
|
"- 참고 규칙이 제공되면 그 범위 안에서만 답한다.",
|
||||||
"- 처벌, 제재, 신고 처리 결과를 확정해서 말하지 않는다.",
|
"- 처벌, 제재, 신고 처리 결과를 확정해서 말하지 않는다.",
|
||||||
"- 개인정보, 계정 정보, 특정 이용자 평가를 요구하면 운영진 확인으로 돌린다.",
|
"- 개인정보, 계정 정보, 특정 이용자 평가를 요구하면 운영진 확인으로 돌린다.",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
## ads
|
||||||
|
광고, 홍보, 외부 링크 반복 게시물은 운영진 허가가 없는 경우 제한될 수 있다.
|
||||||
|
개인 방송, 거래, 모집 홍보도 사전 확인이 필요하다.
|
||||||
|
|
||||||
|
## spam
|
||||||
|
같거나 유사한 메시지를 짧은 시간에 반복하는 도배는 제재 대상이 될 수 있다.
|
||||||
|
대화 흐름을 방해하는 과도한 멘션이나 이모지도 도배로 볼 수 있다.
|
||||||
|
|
||||||
|
## profanity
|
||||||
|
욕설, 혐오 표현, 차별적 표현, 특정 이용자 비하 발언은 금지된다.
|
||||||
|
분쟁이 생기면 맞대응하지 말고 운영진에게 상황을 전달한다.
|
||||||
|
|
||||||
|
## report
|
||||||
|
위반 제보는 문제 메시지, 시간, 관련 이용자, 맥락을 함께 정리해 운영진에게 전달한다.
|
||||||
|
처리 결과와 제재 여부는 운영진 확인 뒤 결정된다.
|
||||||
|
|
||||||
|
## sanctions
|
||||||
|
제재는 경고, 메시지 삭제, 임시 제한, 강퇴 등 단계적으로 적용될 수 있다.
|
||||||
|
사안의 반복성, 고의성, 피해 정도에 따라 운영진이 판단한다.
|
||||||
|
|
||||||
|
## appeal
|
||||||
|
정지나 차단이 억울하면 당시 상황, 근거, 반성 또는 소명 내용을 정리해 운영진에게 이의 제기한다.
|
||||||
|
assistant는 해제 여부를 확정하지 않는다.
|
||||||
|
|
||||||
|
## minor-safety
|
||||||
|
미성년자 보호를 위해 부적절한 대화, 사적 연락 강요, 성적 발언은 엄격히 제한된다.
|
||||||
|
불편하거나 위험한 상황은 즉시 운영진에게 알린다.
|
||||||
|
|
||||||
|
## privacy
|
||||||
|
전화번호, 주소, 계정 정보, 실명 등 개인정보 공유나 요구는 금지된다.
|
||||||
|
타인의 개인정보를 발견하면 확산하지 말고 운영진에게 제보한다.
|
||||||
|
|
||||||
|
## off-topic
|
||||||
|
규칙과 무관한 대화는 해당 채팅방의 목적에 맞는 곳으로 옮기도록 안내한다.
|
||||||
|
반복적으로 흐름을 방해하면 운영진 확인이 필요할 수 있다.
|
||||||
@@ -6,10 +6,22 @@ import type {
|
|||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import type { ChatMessage } from "@llm/types";
|
import type { ChatMessage } from "@llm/types";
|
||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
|
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||||
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
||||||
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
||||||
import { createModerationToolRegistry } from "./tools.js";
|
import { createModerationToolRegistry } from "./tools.js";
|
||||||
|
|
||||||
|
type RuleTopic =
|
||||||
|
| "ads"
|
||||||
|
| "appeal"
|
||||||
|
| "minor-safety"
|
||||||
|
| "off-topic"
|
||||||
|
| "privacy"
|
||||||
|
| "profanity"
|
||||||
|
| "report"
|
||||||
|
| "sanctions"
|
||||||
|
| "spam";
|
||||||
|
|
||||||
function formatHistory(history: ConversationMessage[]): string {
|
function formatHistory(history: ConversationMessage[]): string {
|
||||||
if (history.length === 0) {
|
if (history.length === 0) {
|
||||||
return "(none)";
|
return "(none)";
|
||||||
@@ -32,11 +44,15 @@ function buildMessages(
|
|||||||
request: AgentRequest,
|
request: AgentRequest,
|
||||||
history: ConversationMessage[],
|
history: ConversationMessage[],
|
||||||
intent: ModerationIntent,
|
intent: ModerationIntent,
|
||||||
|
referenceText?: string,
|
||||||
): ChatMessage[] {
|
): ChatMessage[] {
|
||||||
const policies = [
|
const policies = [
|
||||||
...MODERATION_PROMPT_DATA.policy.base,
|
...MODERATION_PROMPT_DATA.policy.base,
|
||||||
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
|
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
|
||||||
];
|
];
|
||||||
|
const referenceBlock = referenceText?.trim()
|
||||||
|
? ["", "[reference]", referenceText.trim()]
|
||||||
|
: [];
|
||||||
|
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
@@ -52,6 +68,7 @@ function buildMessages(
|
|||||||
"",
|
"",
|
||||||
"[examples]",
|
"[examples]",
|
||||||
formatExamples(intent),
|
formatExamples(intent),
|
||||||
|
...referenceBlock,
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -67,6 +84,52 @@ function buildMessages(
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveRuleTopic(message: string): RuleTopic | null {
|
||||||
|
if (/(광고|홍보|링크)/u.test(message)) {
|
||||||
|
return "ads";
|
||||||
|
}
|
||||||
|
if (/(도배|반복|스팸)/u.test(message)) {
|
||||||
|
return "spam";
|
||||||
|
}
|
||||||
|
if (/(욕설|욕했|혐오|차별|비하)/u.test(message)) {
|
||||||
|
return "profanity";
|
||||||
|
}
|
||||||
|
if (/(신고|제보|위반)/u.test(message)) {
|
||||||
|
return "report";
|
||||||
|
}
|
||||||
|
if (/(경고|제재|강퇴|처벌)/u.test(message)) {
|
||||||
|
return "sanctions";
|
||||||
|
}
|
||||||
|
if (/(이의|소명|억울|정지|벤|밴|차단)/u.test(message)) {
|
||||||
|
return "appeal";
|
||||||
|
}
|
||||||
|
if (/(미성년|미자|성적|보호)/u.test(message)) {
|
||||||
|
return "minor-safety";
|
||||||
|
}
|
||||||
|
if (/(개인정보|전화번호|주소|실명|계정)/u.test(message)) {
|
||||||
|
return "privacy";
|
||||||
|
}
|
||||||
|
if (/(잡담|주제|오프토픽|off-topic)/iu.test(message)) {
|
||||||
|
return "off-topic";
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function lookupReferenceText(
|
||||||
|
intent: ModerationIntent,
|
||||||
|
message: string,
|
||||||
|
): Promise<string | undefined> {
|
||||||
|
if (intent.kind !== "rule-question") {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const topic = resolveRuleTopic(message);
|
||||||
|
if (!topic) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const result = await lookupRuleTool.execute({ topic }, {});
|
||||||
|
return result.text;
|
||||||
|
}
|
||||||
|
|
||||||
function templateForIntent(intent: ModerationIntent): string {
|
function templateForIntent(intent: ModerationIntent): string {
|
||||||
switch (intent.kind) {
|
switch (intent.kind) {
|
||||||
case "appeal":
|
case "appeal":
|
||||||
@@ -84,6 +147,7 @@ function templateForIntent(intent: ModerationIntent): string {
|
|||||||
|
|
||||||
export class ModerationSkill implements Skill {
|
export class ModerationSkill implements Skill {
|
||||||
readonly name = "moderation";
|
readonly name = "moderation";
|
||||||
|
readonly priority = 50;
|
||||||
private readonly toolRegistry = createModerationToolRegistry();
|
private readonly toolRegistry = createModerationToolRegistry();
|
||||||
|
|
||||||
constructor(private readonly llmClient: LlmClient) {}
|
constructor(private readonly llmClient: LlmClient) {}
|
||||||
@@ -97,6 +161,10 @@ export class ModerationSkill implements Skill {
|
|||||||
conversationHistory: ConversationMessage[],
|
conversationHistory: ConversationMessage[],
|
||||||
): Promise<AgentResponse> {
|
): Promise<AgentResponse> {
|
||||||
const classified = classifyModerationIntent(request.userMessage ?? "");
|
const classified = classifyModerationIntent(request.userMessage ?? "");
|
||||||
|
const referenceText = await lookupReferenceText(
|
||||||
|
classified.intent,
|
||||||
|
request.userMessage ?? "",
|
||||||
|
);
|
||||||
const metadata = {
|
const metadata = {
|
||||||
intent: classified.intent.kind,
|
intent: classified.intent.kind,
|
||||||
toolCount: this.toolRegistry.list().length,
|
toolCount: this.toolRegistry.list().length,
|
||||||
@@ -113,7 +181,7 @@ export class ModerationSkill implements Skill {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const response = await this.llmClient.chatCompletion(
|
const response = await this.llmClient.chatCompletion(
|
||||||
buildMessages(request, conversationHistory, classified.intent),
|
buildMessages(request, conversationHistory, classified.intent, referenceText),
|
||||||
{
|
{
|
||||||
maxTokens: 192,
|
maxTokens: 192,
|
||||||
temperature: 0.2,
|
temperature: 0.2,
|
||||||
@@ -126,7 +194,10 @@ export class ModerationSkill implements Skill {
|
|||||||
metadata,
|
metadata,
|
||||||
requestId: request.requestId,
|
requestId: request.requestId,
|
||||||
source: response ? "llm" : "template",
|
source: response ? "llm" : "template",
|
||||||
text: response?.content.trim() || templateForIntent(classified.intent),
|
text:
|
||||||
|
response?.content.trim() ||
|
||||||
|
referenceText ||
|
||||||
|
templateForIntent(classified.intent),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { ToolRegistry } from "@tools/registry";
|
import { ToolRegistry } from "@tools/registry";
|
||||||
|
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||||
|
|
||||||
export function createModerationToolRegistry(): ToolRegistry {
|
export function createModerationToolRegistry(): ToolRegistry {
|
||||||
return new ToolRegistry();
|
return new ToolRegistry().register(lookupRuleTool);
|
||||||
}
|
}
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
name: onboarding
|
name: onboarding
|
||||||
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
|
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
|
||||||
version: 1
|
version: 1
|
||||||
|
priority: 70
|
||||||
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
|
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
|
||||||
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
|
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
|
||||||
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
|
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
import {
|
||||||
|
getAcknowledgementMessage,
|
||||||
|
getAgeRequirementMessage,
|
||||||
|
getDiscordMessage,
|
||||||
|
getGenericHelpMessage,
|
||||||
|
getGreetingMessage,
|
||||||
|
getHeartGuideMessage,
|
||||||
|
getHeartLocationMessage,
|
||||||
|
getNicknameInvalidMessage,
|
||||||
|
getNicknameFormatMessage,
|
||||||
|
getPhotoFailedMessage,
|
||||||
|
getRemainingStatusMessage,
|
||||||
|
} from '../onboarding-templates.js';
|
||||||
|
import type { DerivedState, Intent } from '../state/onboarding-state.js';
|
||||||
|
import type { OnboardingSession } from '../types.js';
|
||||||
|
|
||||||
|
export function getIntentFallbackText(
|
||||||
|
nickname: string,
|
||||||
|
session: OnboardingSession | null,
|
||||||
|
intent: Intent,
|
||||||
|
derived: DerivedState,
|
||||||
|
): string {
|
||||||
|
switch (intent.kind) {
|
||||||
|
case 'greeting':
|
||||||
|
return getGreetingMessage();
|
||||||
|
case 'acknowledgement':
|
||||||
|
return getAcknowledgementMessage();
|
||||||
|
case 'status-question':
|
||||||
|
return getRemainingStatusMessage(session);
|
||||||
|
case 'nickname-question':
|
||||||
|
return derived.phase.kind === 'nickname-rejected'
|
||||||
|
? getNicknameInvalidMessage(nickname, session)
|
||||||
|
: getNicknameFormatMessage();
|
||||||
|
case 'photo-question':
|
||||||
|
return derived.phase.kind === 'photo-rejected'
|
||||||
|
? getPhotoFailedMessage(nickname)
|
||||||
|
: getHeartGuideMessage();
|
||||||
|
case 'help-where':
|
||||||
|
if (
|
||||||
|
derived.phase.kind === 'photo-pending' ||
|
||||||
|
derived.phase.kind === 'photo-rejected' ||
|
||||||
|
(derived.phase.kind === 'deadline-near' &&
|
||||||
|
derived.phase.remaining.includes('photo'))
|
||||||
|
) {
|
||||||
|
return getHeartLocationMessage();
|
||||||
|
}
|
||||||
|
return getGenericHelpMessage(nickname, session);
|
||||||
|
case 'help-how':
|
||||||
|
if (
|
||||||
|
derived.phase.kind === 'nickname-pending' ||
|
||||||
|
derived.phase.kind === 'nickname-rejected' ||
|
||||||
|
(derived.phase.kind === 'deadline-near' &&
|
||||||
|
derived.phase.remaining.includes('nickname'))
|
||||||
|
) {
|
||||||
|
return getNicknameFormatMessage();
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
derived.phase.kind === 'photo-pending' ||
|
||||||
|
derived.phase.kind === 'photo-rejected' ||
|
||||||
|
(derived.phase.kind === 'deadline-near' &&
|
||||||
|
derived.phase.remaining.includes('photo'))
|
||||||
|
) {
|
||||||
|
return getHeartGuideMessage();
|
||||||
|
}
|
||||||
|
return getGenericHelpMessage(nickname, session);
|
||||||
|
case 'discord-question':
|
||||||
|
return getDiscordMessage(nickname, session);
|
||||||
|
case 'discord-verify-question':
|
||||||
|
return getDiscordMessage(nickname, session);
|
||||||
|
case 'age-question':
|
||||||
|
return getAgeRequirementMessage();
|
||||||
|
case 'chitchat':
|
||||||
|
case 'off-topic-to-other':
|
||||||
|
return '';
|
||||||
|
case 'unknown':
|
||||||
|
return getGenericHelpMessage(nickname, session);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import type { AgentRequest, ConversationMessage } from '@agent/types';
|
||||||
|
import type { GarnetClient } from '@common/garnet';
|
||||||
|
import { getConversationSummaryKey } from '@conv/conversation-store';
|
||||||
|
import type { ChatMessage } from '@llm/types';
|
||||||
|
import { getRequestLogger } from '@obs/context';
|
||||||
|
import type { RoomContextService } from '@room/room-context-service';
|
||||||
|
import { meetsTrustTier } from '../../../eval/governance/trust.js';
|
||||||
|
import {
|
||||||
|
getAgentTrustMinTier,
|
||||||
|
getAgentUserMemoryEnabled,
|
||||||
|
getPromptVariantId,
|
||||||
|
} from '../../../config.js';
|
||||||
|
import {
|
||||||
|
loadPromptVariantManifest,
|
||||||
|
PromptBuilder,
|
||||||
|
} from '../prompts/builder.js';
|
||||||
|
import type { OnboardingBlackboardStore } from '../state/blackboard.js';
|
||||||
|
import type { UserMemoryStore } from '../state/user-memory.js';
|
||||||
|
import type { DerivedState } from '../state/onboarding-state.js';
|
||||||
|
import type { OnboardingSession, PromptContext } from '../types.js';
|
||||||
|
|
||||||
|
interface MessageBuilderDeps {
|
||||||
|
blackboardStore: OnboardingBlackboardStore;
|
||||||
|
garnetClient: GarnetClient;
|
||||||
|
roomContextService?: RoomContextService | undefined;
|
||||||
|
userMemoryStore: UserMemoryStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MessageBuilderInput {
|
||||||
|
conversationHistory: ConversationMessage[];
|
||||||
|
derived: DerivedState;
|
||||||
|
request: AgentRequest;
|
||||||
|
session: OnboardingSession | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildPromptUserMessage(
|
||||||
|
request: AgentRequest,
|
||||||
|
derived: DerivedState,
|
||||||
|
): string {
|
||||||
|
switch (request.trigger) {
|
||||||
|
case 'nickname-invalid':
|
||||||
|
return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`;
|
||||||
|
case 'photo-failed':
|
||||||
|
return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.';
|
||||||
|
case 'progress-update':
|
||||||
|
return '현재 새내기 등록 진행상황만 짧게 알려줘.';
|
||||||
|
case 'timeout-warning':
|
||||||
|
return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.';
|
||||||
|
case 'completed':
|
||||||
|
return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.';
|
||||||
|
case 'user-message':
|
||||||
|
if (derived.intent.kind === 'status-question') {
|
||||||
|
return '완료된 것은 빼고 남은 절차만 짧게 말해줘.';
|
||||||
|
}
|
||||||
|
return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.';
|
||||||
|
default:
|
||||||
|
return '지금 필요한 온보딩 안내만 짧게 도와줘.';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadConversationSummary(
|
||||||
|
garnetClient: GarnetClient,
|
||||||
|
chatroomId: string,
|
||||||
|
userId: string,
|
||||||
|
): Promise<string | undefined> {
|
||||||
|
const key = getConversationSummaryKey(chatroomId, userId);
|
||||||
|
return (await garnetClient.getString(key)) ?? undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function buildLlmMessages(
|
||||||
|
deps: MessageBuilderDeps,
|
||||||
|
input: MessageBuilderInput,
|
||||||
|
): Promise<ChatMessage[]> {
|
||||||
|
const logger = getRequestLogger();
|
||||||
|
const { conversationHistory, derived, request, session } = input;
|
||||||
|
const roomContextPromise =
|
||||||
|
request.trigger === 'user-message' && deps.roomContextService
|
||||||
|
? deps.roomContextService
|
||||||
|
.getRecentMessages(request.chatroomId)
|
||||||
|
.catch((error) => {
|
||||||
|
logger.warn(
|
||||||
|
{ err: error },
|
||||||
|
'[OnboardingSkill] Failed to fetch room context',
|
||||||
|
);
|
||||||
|
return undefined;
|
||||||
|
})
|
||||||
|
: Promise.resolve(undefined);
|
||||||
|
const userProfilePromise = getAgentUserMemoryEnabled()
|
||||||
|
? deps.userMemoryStore.load(request.userId).catch((error) => {
|
||||||
|
logger.warn(
|
||||||
|
{ err: error },
|
||||||
|
'[OnboardingSkill] Failed to load user memory',
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
})
|
||||||
|
: Promise.resolve(null);
|
||||||
|
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
|
||||||
|
deps.blackboardStore.load(request.chatroomId, request.userId),
|
||||||
|
loadConversationSummary(deps.garnetClient, request.chatroomId, request.userId),
|
||||||
|
roomContextPromise,
|
||||||
|
userProfilePromise,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const context: PromptContext = {
|
||||||
|
blackboard,
|
||||||
|
conversationHistory,
|
||||||
|
roomContext,
|
||||||
|
summary,
|
||||||
|
userMessage: buildPromptUserMessage(request, derived),
|
||||||
|
userProfile,
|
||||||
|
};
|
||||||
|
|
||||||
|
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||||
|
const promptVariantId = getPromptVariantId();
|
||||||
|
if (!promptVariantId) {
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
const manifest = loadPromptVariantManifest(promptVariantId);
|
||||||
|
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder.applyVariant(manifest).build();
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
import type { AgentRequest } from '@agent/types';
|
||||||
|
import type { DerivedState } from '../state/onboarding-state.js';
|
||||||
|
import type { OnboardingBlackboardStore } from '../state/blackboard.js';
|
||||||
|
|
||||||
|
export async function rememberTurn(
|
||||||
|
blackboardStore: OnboardingBlackboardStore,
|
||||||
|
request: AgentRequest,
|
||||||
|
derived: DerivedState,
|
||||||
|
): Promise<void> {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const userMessage = request.userMessage?.trim();
|
||||||
|
|
||||||
|
await blackboardStore.update(request.chatroomId, request.userId, (current) => {
|
||||||
|
const sameIntentStreak =
|
||||||
|
current.lastIntent === derived.intent.kind
|
||||||
|
? (current.sameIntentStreak ?? 1) + 1
|
||||||
|
: 1;
|
||||||
|
const openQuestions =
|
||||||
|
userMessage &&
|
||||||
|
(derived.intent.kind === 'help-how' ||
|
||||||
|
derived.intent.kind === 'help-where' ||
|
||||||
|
derived.intent.kind === 'nickname-question' ||
|
||||||
|
derived.intent.kind === 'photo-question' ||
|
||||||
|
derived.intent.kind === 'discord-question' ||
|
||||||
|
derived.intent.kind === 'discord-verify-question')
|
||||||
|
? [...current.openQuestions, userMessage].slice(-3)
|
||||||
|
: current.openQuestions;
|
||||||
|
|
||||||
|
const recentNicknameAttempts =
|
||||||
|
request.trigger === 'nickname-invalid' ||
|
||||||
|
derived.intent.kind === 'nickname-question'
|
||||||
|
? Array.from(
|
||||||
|
new Set([...current.recentNicknameAttempts, request.nickname]),
|
||||||
|
).slice(-5)
|
||||||
|
: current.recentNicknameAttempts;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...current,
|
||||||
|
flaggedIssues:
|
||||||
|
derived.phase.kind === 'nickname-rejected'
|
||||||
|
? derived.phase.issues.map((issue) => issue.message)
|
||||||
|
: derived.phase.kind === 'completed'
|
||||||
|
? []
|
||||||
|
: current.flaggedIssues,
|
||||||
|
lastIntent: derived.intent.kind,
|
||||||
|
lastIntentAt: now,
|
||||||
|
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
|
||||||
|
recentNicknameAttempts,
|
||||||
|
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
|
||||||
|
toneMemory: {
|
||||||
|
...current.toneMemory,
|
||||||
|
greetedAt:
|
||||||
|
derived.intent.kind === 'greeting'
|
||||||
|
? now
|
||||||
|
: current.toneMemory.greetedAt,
|
||||||
|
thankedAt:
|
||||||
|
derived.intent.kind === 'acknowledgement'
|
||||||
|
? now
|
||||||
|
: current.toneMemory.thankedAt,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import type { AgentRequest } from '@agent/types';
|
||||||
|
import type { ChatMessage } from '@llm/types';
|
||||||
|
import type { Tool, ToolContext } from '@tools/tool';
|
||||||
|
import { getLlmTemperature } from '../../../config.js';
|
||||||
|
import type { DerivedState } from '../state/onboarding-state.js';
|
||||||
|
import type { OnboardingOrchestrator } from './orchestrator.js';
|
||||||
|
import { validateOnboardingResponse } from './validator.js';
|
||||||
|
|
||||||
|
export type LlmTextResult =
|
||||||
|
| { status: 'success'; text: string }
|
||||||
|
| { status: 'no_response' }
|
||||||
|
| { status: 'failed' };
|
||||||
|
|
||||||
|
export async function generateLlmText(
|
||||||
|
orchestrator: OnboardingOrchestrator,
|
||||||
|
messages: ChatMessage[],
|
||||||
|
derived: DerivedState,
|
||||||
|
options?: {
|
||||||
|
allowShortResponse?: boolean;
|
||||||
|
maxTokens?: number;
|
||||||
|
request: AgentRequest;
|
||||||
|
retryOnLowQuality?: boolean;
|
||||||
|
temperature?: number;
|
||||||
|
toolContext?: ToolContext | undefined;
|
||||||
|
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||||
|
},
|
||||||
|
): Promise<LlmTextResult> {
|
||||||
|
const attempts: Array<{
|
||||||
|
maxTokens?: number;
|
||||||
|
temperature?: number;
|
||||||
|
}> = [];
|
||||||
|
|
||||||
|
const firstAttempt: {
|
||||||
|
maxTokens?: number;
|
||||||
|
temperature?: number;
|
||||||
|
} = {};
|
||||||
|
if (options?.maxTokens !== undefined) {
|
||||||
|
firstAttempt.maxTokens = options.maxTokens;
|
||||||
|
}
|
||||||
|
if (options?.temperature !== undefined) {
|
||||||
|
firstAttempt.temperature = options.temperature;
|
||||||
|
}
|
||||||
|
attempts.push(firstAttempt);
|
||||||
|
|
||||||
|
if (options?.retryOnLowQuality) {
|
||||||
|
const retryAttempt: {
|
||||||
|
maxTokens?: number;
|
||||||
|
temperature?: number;
|
||||||
|
} = {
|
||||||
|
temperature: getLlmTemperature() + 0.1,
|
||||||
|
};
|
||||||
|
if (options.maxTokens !== undefined) {
|
||||||
|
retryAttempt.maxTokens = options.maxTokens;
|
||||||
|
}
|
||||||
|
attempts.push(retryAttempt);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const attempt of attempts) {
|
||||||
|
const requestOptions: {
|
||||||
|
maxTokens?: number;
|
||||||
|
temperature?: number;
|
||||||
|
} = {};
|
||||||
|
if (attempt.maxTokens !== undefined) {
|
||||||
|
requestOptions.maxTokens = attempt.maxTokens;
|
||||||
|
}
|
||||||
|
if (attempt.temperature !== undefined) {
|
||||||
|
requestOptions.temperature = attempt.temperature;
|
||||||
|
}
|
||||||
|
|
||||||
|
const toolContext = options?.toolContext ?? {};
|
||||||
|
const llmText = await orchestrator.run(derived, messages, {
|
||||||
|
maxTokens: requestOptions.maxTokens,
|
||||||
|
request: options?.request,
|
||||||
|
temperature: requestOptions.temperature,
|
||||||
|
toolContext,
|
||||||
|
tools: options?.tools,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!llmText) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const validation = validateOnboardingResponse(llmText, derived, {
|
||||||
|
allowShortResponse: options?.allowShortResponse,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (validation.text.includes('[NO_RESPONSE]')) {
|
||||||
|
return { status: 'no_response' };
|
||||||
|
}
|
||||||
|
if (validation.status === 'reject') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
text: validation.text,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { status: 'failed' };
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import type { Tool } from '@tools/tool';
|
||||||
|
import { createToolSearchTool } from '@tools/tool-search';
|
||||||
|
import type { ToolRegistry } from '@tools/registry';
|
||||||
|
import {
|
||||||
|
getAgentToolSearchEnabled,
|
||||||
|
getAgentToolSearchThreshold,
|
||||||
|
} from '../../../config.js';
|
||||||
|
import type { DerivedState } from '../state/onboarding-state.js';
|
||||||
|
|
||||||
|
export function selectToolsForDerived(
|
||||||
|
derived: DerivedState,
|
||||||
|
tools: readonly Tool<unknown, unknown>[],
|
||||||
|
toolRegistry: ToolRegistry,
|
||||||
|
): readonly Tool<unknown, unknown>[] {
|
||||||
|
const maybeUseToolSearch = (
|
||||||
|
selectedTools: readonly Tool<unknown, unknown>[],
|
||||||
|
): readonly Tool<unknown, unknown>[] => {
|
||||||
|
if (
|
||||||
|
getAgentToolSearchEnabled() &&
|
||||||
|
selectedTools.length >= getAgentToolSearchThreshold()
|
||||||
|
) {
|
||||||
|
return [createToolSearchTool(toolRegistry)];
|
||||||
|
}
|
||||||
|
return selectedTools;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (derived.intent.kind === 'discord-verify-question') {
|
||||||
|
return maybeUseToolSearch(
|
||||||
|
tools.filter((tool) => tool.name === 'verify-discord-nickname'),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (derived.phase.kind === 'nickname-rejected') {
|
||||||
|
return maybeUseToolSearch(
|
||||||
|
tools.filter(
|
||||||
|
(tool) =>
|
||||||
|
tool.name === 'validate-nickname' ||
|
||||||
|
tool.name === 'lookup-community-rule' ||
|
||||||
|
tool.name === 'check-onboarding-progress',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (derived.intent.kind === 'status-question') {
|
||||||
|
return maybeUseToolSearch(
|
||||||
|
tools.filter((tool) => tool.name === 'check-onboarding-progress'),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (derived.intent.kind === 'discord-question') {
|
||||||
|
return maybeUseToolSearch(
|
||||||
|
tools.filter(
|
||||||
|
(tool) =>
|
||||||
|
tool.name === 'lookup-community-rule' ||
|
||||||
|
tool.name === 'verify-discord-nickname',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return maybeUseToolSearch(tools);
|
||||||
|
}
|
||||||
+31
-354
@@ -1,38 +1,22 @@
|
|||||||
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
|
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
|
||||||
import { getConversationSummaryKey } from '@conv/conversation-store';
|
|
||||||
import type { GarnetClient } from '@common/garnet';
|
import type { GarnetClient } from '@common/garnet';
|
||||||
import type { NatsClient } from '@common/nats';
|
import type { NatsClient } from '@common/nats';
|
||||||
import { getContext, getRequestLogger } from '@obs/context';
|
import { getContext, getRequestLogger } from '@obs/context';
|
||||||
import type { LlmClient } from '@llm/llm-client';
|
import type { LlmClient } from '@llm/llm-client';
|
||||||
import type { ChatMessage } from '@llm/types';
|
|
||||||
import type { RoomContextService } from '@room/room-context-service';
|
import type { RoomContextService } from '@room/room-context-service';
|
||||||
import type { Skill } from '@skills/skill';
|
import type { Skill } from '@skills/skill';
|
||||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||||
import { meetsTrustTier } from '../../eval/governance/trust.js';
|
|
||||||
import { createOnboardingToolRegistry } from './tools.js';
|
import { createOnboardingToolRegistry } from './tools.js';
|
||||||
import type { ToolRegistry } from '@tools/registry';
|
import type { ToolRegistry } from '@tools/registry';
|
||||||
import type { Tool, ToolContext } from '@tools/tool';
|
import type { Tool, ToolContext } from '@tools/tool';
|
||||||
import { createToolSearchTool } from '@tools/tool-search';
|
|
||||||
import {
|
import {
|
||||||
loadPromptVariantManifest,
|
|
||||||
PromptBuilder,
|
|
||||||
} from './prompts/builder.js';
|
|
||||||
import {
|
|
||||||
getAcknowledgementMessage,
|
|
||||||
getAgeRequirementMessage,
|
|
||||||
getCompletedMessage,
|
getCompletedMessage,
|
||||||
getDiscordMessage,
|
|
||||||
getGreetingMessage,
|
|
||||||
getGenericHelpMessage,
|
getGenericHelpMessage,
|
||||||
getHeartGuideMessage,
|
|
||||||
getHeartLocationMessage,
|
|
||||||
getNicknameInvalidMessage,
|
getNicknameInvalidMessage,
|
||||||
getNicknameFormatMessage,
|
|
||||||
getOnboardingTemplate,
|
getOnboardingTemplate,
|
||||||
getPhotoFailedMessage,
|
getPhotoFailedMessage,
|
||||||
getProgressMessage,
|
getProgressMessage,
|
||||||
getRemainingStatusMessage,
|
|
||||||
getTimeoutWarningMessage,
|
getTimeoutWarningMessage,
|
||||||
} from './onboarding-templates.js';
|
} from './onboarding-templates.js';
|
||||||
import {
|
import {
|
||||||
@@ -41,13 +25,8 @@ import {
|
|||||||
getAgentAbstentionThreshold,
|
getAgentAbstentionThreshold,
|
||||||
getAgentRouterMode,
|
getAgentRouterMode,
|
||||||
getAgentRouterWeightsPath,
|
getAgentRouterWeightsPath,
|
||||||
getAgentTrustMinTier,
|
|
||||||
getAgentToolSearchEnabled,
|
|
||||||
getAgentToolSearchThreshold,
|
|
||||||
getAgentUserMemoryEnabled,
|
getAgentUserMemoryEnabled,
|
||||||
getOnboardingSessionKeyPrefix,
|
getOnboardingSessionKeyPrefix,
|
||||||
getLlmTemperature,
|
|
||||||
getPromptVariantId,
|
|
||||||
} from '../../config.js';
|
} from '../../config.js';
|
||||||
import { OnboardingBlackboardStore } from './state/blackboard.js';
|
import { OnboardingBlackboardStore } from './state/blackboard.js';
|
||||||
import { UserMemoryStore } from './state/user-memory.js';
|
import { UserMemoryStore } from './state/user-memory.js';
|
||||||
@@ -74,21 +53,24 @@ import {
|
|||||||
runWithProcessTrace,
|
runWithProcessTrace,
|
||||||
} from './runtime/trace.js';
|
} from './runtime/trace.js';
|
||||||
import { validateOnboardingResponse } from './runtime/validator.js';
|
import { validateOnboardingResponse } from './runtime/validator.js';
|
||||||
|
import { getIntentFallbackText } from './runtime/fallback-text-resolver.js';
|
||||||
|
import { buildLlmMessages } from './runtime/message-builder.js';
|
||||||
|
import {
|
||||||
|
generateLlmText,
|
||||||
|
type LlmTextResult,
|
||||||
|
} from './runtime/text-generator.js';
|
||||||
|
import { selectToolsForDerived } from './runtime/tool-selector.js';
|
||||||
|
import { rememberTurn } from './runtime/state-updater.js';
|
||||||
import { normalizeOnboardingSession } from './session.js';
|
import { normalizeOnboardingSession } from './session.js';
|
||||||
import type {
|
import type {
|
||||||
Blackboard,
|
Blackboard,
|
||||||
OnboardingSession,
|
OnboardingSession,
|
||||||
PromptContext,
|
|
||||||
StoredOnboardingSession,
|
StoredOnboardingSession,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
|
|
||||||
type LlmTextResult =
|
|
||||||
| { status: 'success'; text: string }
|
|
||||||
| { status: 'no_response' }
|
|
||||||
| { status: 'failed' };
|
|
||||||
|
|
||||||
export class OnboardingSkill implements Skill {
|
export class OnboardingSkill implements Skill {
|
||||||
readonly name = 'onboarding';
|
readonly name = 'onboarding';
|
||||||
|
readonly priority = 70;
|
||||||
private readonly classifier: OnboardingIntentClassifier;
|
private readonly classifier: OnboardingIntentClassifier;
|
||||||
private readonly blackboardStore: OnboardingBlackboardStore;
|
private readonly blackboardStore: OnboardingBlackboardStore;
|
||||||
private readonly orchestrator: OnboardingOrchestrator;
|
private readonly orchestrator: OnboardingOrchestrator;
|
||||||
@@ -357,9 +339,26 @@ export class OnboardingSkill implements Skill {
|
|||||||
conversationHistory: ConversationMessage[],
|
conversationHistory: ConversationMessage[],
|
||||||
derived: DerivedState,
|
derived: DerivedState,
|
||||||
): Promise<LlmTextResult> {
|
): Promise<LlmTextResult> {
|
||||||
const messages = await this.buildLlmMessages(request, session, conversationHistory, derived);
|
if (!this.llmClient.hasEndpoints()) {
|
||||||
|
return { status: 'failed' };
|
||||||
|
}
|
||||||
|
|
||||||
return this.generateLlmText(messages, derived, {
|
const messages = await buildLlmMessages(
|
||||||
|
{
|
||||||
|
blackboardStore: this.blackboardStore,
|
||||||
|
garnetClient: this.garnetClient,
|
||||||
|
roomContextService: this.roomContextService,
|
||||||
|
userMemoryStore: this.userMemoryStore,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
conversationHistory,
|
||||||
|
derived,
|
||||||
|
request,
|
||||||
|
session,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return generateLlmText(this.orchestrator, messages, derived, {
|
||||||
allowShortResponse: derived.allowShortResponse,
|
allowShortResponse: derived.allowShortResponse,
|
||||||
maxTokens: derived.suggestedMaxTokens,
|
maxTokens: derived.suggestedMaxTokens,
|
||||||
request,
|
request,
|
||||||
@@ -368,135 +367,12 @@ export class OnboardingSkill implements Skill {
|
|||||||
!derived.allowShortResponse &&
|
!derived.allowShortResponse &&
|
||||||
derived.responseMode !== 'llm-with-tools',
|
derived.responseMode !== 'llm-with-tools',
|
||||||
toolContext: this.buildToolContext(request, session),
|
toolContext: this.buildToolContext(request, session),
|
||||||
tools: this.selectToolsForDerived(derived),
|
tools: selectToolsForDerived(derived, this.tools, this.toolRegistry),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async buildLlmMessages(
|
|
||||||
request: AgentRequest,
|
|
||||||
session: OnboardingSession | null,
|
|
||||||
conversationHistory: ConversationMessage[],
|
|
||||||
derived: DerivedState,
|
|
||||||
): Promise<ChatMessage[]> {
|
|
||||||
const logger = getRequestLogger();
|
|
||||||
const roomContextPromise =
|
|
||||||
request.trigger === 'user-message' && this.roomContextService
|
|
||||||
? this.roomContextService.getRecentMessages(request.chatroomId).catch((error) => {
|
|
||||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to fetch room context');
|
|
||||||
return undefined;
|
|
||||||
})
|
|
||||||
: Promise.resolve(undefined);
|
|
||||||
const userProfilePromise = getAgentUserMemoryEnabled()
|
|
||||||
? this.userMemoryStore.load(request.userId).catch((error) => {
|
|
||||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory');
|
|
||||||
return null;
|
|
||||||
})
|
|
||||||
: Promise.resolve(null);
|
|
||||||
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
|
|
||||||
this.blackboardStore.load(request.chatroomId, request.userId),
|
|
||||||
this.loadConversationSummary(request.chatroomId, request.userId),
|
|
||||||
roomContextPromise,
|
|
||||||
userProfilePromise,
|
|
||||||
]);
|
|
||||||
|
|
||||||
const context: PromptContext = {
|
|
||||||
blackboard,
|
|
||||||
conversationHistory,
|
|
||||||
roomContext,
|
|
||||||
summary,
|
|
||||||
userMessage: this.buildPromptUserMessage(request, derived),
|
|
||||||
userProfile,
|
|
||||||
};
|
|
||||||
|
|
||||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
|
||||||
const promptVariantId = getPromptVariantId();
|
|
||||||
if (!promptVariantId) {
|
|
||||||
return builder.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
const manifest = loadPromptVariantManifest(promptVariantId);
|
|
||||||
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
|
|
||||||
return builder.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
return builder.applyVariant(manifest).build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
|
|
||||||
switch (request.trigger) {
|
|
||||||
case 'nickname-invalid':
|
|
||||||
return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`;
|
|
||||||
case 'photo-failed':
|
|
||||||
return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.';
|
|
||||||
case 'progress-update':
|
|
||||||
return '현재 새내기 등록 진행상황만 짧게 알려줘.';
|
|
||||||
case 'timeout-warning':
|
|
||||||
return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.';
|
|
||||||
case 'completed':
|
|
||||||
return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.';
|
|
||||||
case 'user-message':
|
|
||||||
if (derived.intent.kind === 'status-question') {
|
|
||||||
return '완료된 것은 빼고 남은 절차만 짧게 말해줘.';
|
|
||||||
}
|
|
||||||
return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.';
|
|
||||||
default:
|
|
||||||
return '지금 필요한 온보딩 안내만 짧게 도와줘.';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async loadConversationSummary(
|
|
||||||
chatroomId: string,
|
|
||||||
userId: string,
|
|
||||||
): Promise<string | undefined> {
|
|
||||||
const key = getConversationSummaryKey(chatroomId, userId);
|
|
||||||
return (await this.garnetClient.getString(key)) ?? undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise<void> {
|
private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise<void> {
|
||||||
const now = new Date().toISOString();
|
return rememberTurn(this.blackboardStore, request, derived);
|
||||||
const userMessage = request.userMessage?.trim();
|
|
||||||
|
|
||||||
await this.blackboardStore.update(request.chatroomId, request.userId, (current) => {
|
|
||||||
const sameIntentStreak =
|
|
||||||
current.lastIntent === derived.intent.kind
|
|
||||||
? (current.sameIntentStreak ?? 1) + 1
|
|
||||||
: 1;
|
|
||||||
const openQuestions =
|
|
||||||
userMessage &&
|
|
||||||
(derived.intent.kind === 'help-how' ||
|
|
||||||
derived.intent.kind === 'help-where' ||
|
|
||||||
derived.intent.kind === 'nickname-question' ||
|
|
||||||
derived.intent.kind === 'photo-question' ||
|
|
||||||
derived.intent.kind === 'discord-question' ||
|
|
||||||
derived.intent.kind === 'discord-verify-question')
|
|
||||||
? [...current.openQuestions, userMessage].slice(-3)
|
|
||||||
: current.openQuestions;
|
|
||||||
|
|
||||||
const recentNicknameAttempts =
|
|
||||||
request.trigger === 'nickname-invalid' || derived.intent.kind === 'nickname-question'
|
|
||||||
? Array.from(new Set([...current.recentNicknameAttempts, request.nickname])).slice(-5)
|
|
||||||
: current.recentNicknameAttempts;
|
|
||||||
|
|
||||||
return {
|
|
||||||
...current,
|
|
||||||
flaggedIssues:
|
|
||||||
derived.phase.kind === 'nickname-rejected'
|
|
||||||
? derived.phase.issues.map((issue) => issue.message)
|
|
||||||
: derived.phase.kind === 'completed'
|
|
||||||
? []
|
|
||||||
: current.flaggedIssues,
|
|
||||||
lastIntent: derived.intent.kind,
|
|
||||||
lastIntentAt: now,
|
|
||||||
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
|
|
||||||
recentNicknameAttempts,
|
|
||||||
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
|
|
||||||
toneMemory: {
|
|
||||||
...current.toneMemory,
|
|
||||||
greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt,
|
|
||||||
thankedAt: derived.intent.kind === 'acknowledgement' ? now : current.toneMemory.thankedAt,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async observeUserMemory(
|
private async observeUserMemory(
|
||||||
@@ -552,163 +428,11 @@ export class OnboardingSkill implements Skill {
|
|||||||
default:
|
default:
|
||||||
return this.makeTemplateResponse(
|
return this.makeTemplateResponse(
|
||||||
request.requestId,
|
request.requestId,
|
||||||
this.getIntentFallbackText(request.nickname, session, intent, derived),
|
getIntentFallbackText(request.nickname, session, intent, derived),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private getIntentFallbackText(
|
|
||||||
nickname: string,
|
|
||||||
session: OnboardingSession | null,
|
|
||||||
intent: Intent,
|
|
||||||
derived: DerivedState,
|
|
||||||
): string {
|
|
||||||
switch (intent.kind) {
|
|
||||||
case 'greeting':
|
|
||||||
return getGreetingMessage();
|
|
||||||
case 'acknowledgement':
|
|
||||||
return getAcknowledgementMessage();
|
|
||||||
case 'status-question':
|
|
||||||
return getRemainingStatusMessage(session);
|
|
||||||
case 'nickname-question':
|
|
||||||
return derived.phase.kind === 'nickname-rejected'
|
|
||||||
? getNicknameInvalidMessage(nickname, session)
|
|
||||||
: getNicknameFormatMessage();
|
|
||||||
case 'photo-question':
|
|
||||||
return derived.phase.kind === 'photo-rejected'
|
|
||||||
? getPhotoFailedMessage(nickname)
|
|
||||||
: getHeartGuideMessage();
|
|
||||||
case 'help-where':
|
|
||||||
if (
|
|
||||||
derived.phase.kind === 'photo-pending' ||
|
|
||||||
derived.phase.kind === 'photo-rejected' ||
|
|
||||||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
|
|
||||||
) {
|
|
||||||
return getHeartLocationMessage();
|
|
||||||
}
|
|
||||||
return getGenericHelpMessage(nickname, session);
|
|
||||||
case 'help-how':
|
|
||||||
if (
|
|
||||||
derived.phase.kind === 'nickname-pending' ||
|
|
||||||
derived.phase.kind === 'nickname-rejected' ||
|
|
||||||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('nickname'))
|
|
||||||
) {
|
|
||||||
return getNicknameFormatMessage();
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
derived.phase.kind === 'photo-pending' ||
|
|
||||||
derived.phase.kind === 'photo-rejected' ||
|
|
||||||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
|
|
||||||
) {
|
|
||||||
return getHeartGuideMessage();
|
|
||||||
}
|
|
||||||
return getGenericHelpMessage(nickname, session);
|
|
||||||
case 'discord-question':
|
|
||||||
return getDiscordMessage(nickname, session);
|
|
||||||
case 'discord-verify-question':
|
|
||||||
return getDiscordMessage(nickname, session);
|
|
||||||
case 'age-question':
|
|
||||||
return getAgeRequirementMessage();
|
|
||||||
case 'chitchat':
|
|
||||||
case 'off-topic-to-other':
|
|
||||||
return '';
|
|
||||||
case 'unknown':
|
|
||||||
return getGenericHelpMessage(nickname, session);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async generateLlmText(
|
|
||||||
messages: ChatMessage[],
|
|
||||||
derived: DerivedState,
|
|
||||||
options?: {
|
|
||||||
allowShortResponse?: boolean;
|
|
||||||
maxTokens?: number;
|
|
||||||
request: AgentRequest;
|
|
||||||
retryOnLowQuality?: boolean;
|
|
||||||
temperature?: number;
|
|
||||||
toolContext?: ToolContext | undefined;
|
|
||||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
|
||||||
},
|
|
||||||
): Promise<LlmTextResult> {
|
|
||||||
if (!this.llmClient.hasEndpoints()) {
|
|
||||||
return { status: 'failed' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const attempts: Array<{
|
|
||||||
maxTokens?: number;
|
|
||||||
temperature?: number;
|
|
||||||
}> = [];
|
|
||||||
|
|
||||||
const firstAttempt: {
|
|
||||||
maxTokens?: number;
|
|
||||||
temperature?: number;
|
|
||||||
} = {};
|
|
||||||
if (options?.maxTokens !== undefined) {
|
|
||||||
firstAttempt.maxTokens = options.maxTokens;
|
|
||||||
}
|
|
||||||
if (options?.temperature !== undefined) {
|
|
||||||
firstAttempt.temperature = options.temperature;
|
|
||||||
}
|
|
||||||
attempts.push(firstAttempt);
|
|
||||||
|
|
||||||
if (options?.retryOnLowQuality) {
|
|
||||||
const retryAttempt: {
|
|
||||||
maxTokens?: number;
|
|
||||||
temperature?: number;
|
|
||||||
} = {
|
|
||||||
temperature: getLlmTemperature() + 0.1,
|
|
||||||
};
|
|
||||||
if (options.maxTokens !== undefined) {
|
|
||||||
retryAttempt.maxTokens = options.maxTokens;
|
|
||||||
}
|
|
||||||
attempts.push(retryAttempt);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const attempt of attempts) {
|
|
||||||
const requestOptions: {
|
|
||||||
maxTokens?: number;
|
|
||||||
temperature?: number;
|
|
||||||
} = {};
|
|
||||||
if (attempt.maxTokens !== undefined) {
|
|
||||||
requestOptions.maxTokens = attempt.maxTokens;
|
|
||||||
}
|
|
||||||
if (attempt.temperature !== undefined) {
|
|
||||||
requestOptions.temperature = attempt.temperature;
|
|
||||||
}
|
|
||||||
|
|
||||||
const toolContext = options?.toolContext ?? {};
|
|
||||||
const llmText = await this.orchestrator.run(derived, messages, {
|
|
||||||
maxTokens: requestOptions.maxTokens,
|
|
||||||
request: options?.request,
|
|
||||||
temperature: requestOptions.temperature,
|
|
||||||
toolContext,
|
|
||||||
tools: options?.tools,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!llmText) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const validation = validateOnboardingResponse(llmText, derived, {
|
|
||||||
allowShortResponse: options?.allowShortResponse,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (validation.text.includes('[NO_RESPONSE]')) {
|
|
||||||
return { status: 'no_response' };
|
|
||||||
}
|
|
||||||
if (validation.status === 'reject') {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
text: validation.text,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return { status: 'failed' };
|
|
||||||
}
|
|
||||||
|
|
||||||
private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext {
|
private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext {
|
||||||
return {
|
return {
|
||||||
chatroomId: request.chatroomId,
|
chatroomId: request.chatroomId,
|
||||||
@@ -721,53 +445,6 @@ export class OnboardingSkill implements Skill {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private selectToolsForDerived(derived: DerivedState): readonly Tool<unknown, unknown>[] {
|
|
||||||
const maybeUseToolSearch = (
|
|
||||||
tools: readonly Tool<unknown, unknown>[],
|
|
||||||
): readonly Tool<unknown, unknown>[] => {
|
|
||||||
if (
|
|
||||||
getAgentToolSearchEnabled() &&
|
|
||||||
tools.length >= getAgentToolSearchThreshold()
|
|
||||||
) {
|
|
||||||
return [createToolSearchTool(this.toolRegistry)];
|
|
||||||
}
|
|
||||||
return tools;
|
|
||||||
};
|
|
||||||
|
|
||||||
if (derived.intent.kind === 'discord-verify-question') {
|
|
||||||
return maybeUseToolSearch(
|
|
||||||
this.tools.filter((tool) => tool.name === 'verify-discord-nickname'),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (derived.phase.kind === 'nickname-rejected') {
|
|
||||||
return maybeUseToolSearch(
|
|
||||||
this.tools.filter(
|
|
||||||
(tool) =>
|
|
||||||
tool.name === 'validate-nickname' ||
|
|
||||||
tool.name === 'lookup-community-rule' ||
|
|
||||||
tool.name === 'check-onboarding-progress',
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (derived.intent.kind === 'status-question') {
|
|
||||||
return maybeUseToolSearch(
|
|
||||||
this.tools.filter((tool) => tool.name === 'check-onboarding-progress'),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (derived.intent.kind === 'discord-question') {
|
|
||||||
return maybeUseToolSearch(
|
|
||||||
this.tools.filter(
|
|
||||||
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return maybeUseToolSearch(this.tools);
|
|
||||||
}
|
|
||||||
|
|
||||||
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
|
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
|
||||||
return {
|
return {
|
||||||
requestId,
|
requestId,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ function makeTempSkillRoot(): string {
|
|||||||
"name: demo",
|
"name: demo",
|
||||||
"description: Demo skill.",
|
"description: Demo skill.",
|
||||||
"version: 1",
|
"version: 1",
|
||||||
|
"priority: 65",
|
||||||
"intents: [hello, help]",
|
"intents: [hello, help]",
|
||||||
"triggers: [user-message]",
|
"triggers: [user-message]",
|
||||||
"tools: [demo-tool]",
|
"tools: [demo-tool]",
|
||||||
@@ -41,6 +42,7 @@ describe("SkillRegistry", () => {
|
|||||||
name: "demo",
|
name: "demo",
|
||||||
description: "Demo skill.",
|
description: "Demo skill.",
|
||||||
intents: ["hello", "help"],
|
intents: ["hello", "help"],
|
||||||
|
priority: 65,
|
||||||
tools: ["demo-tool"],
|
tools: ["demo-tool"],
|
||||||
trustTier: "T2",
|
trustTier: "T2",
|
||||||
version: 1,
|
version: 1,
|
||||||
@@ -49,6 +51,27 @@ describe("SkillRegistry", () => {
|
|||||||
expect(registry.loadResourceSync("demo", "guide")).toBe("guide body");
|
expect(registry.loadResourceSync("demo", "guide")).toBe("guide body");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("defaults priority to 50 when omitted", () => {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "lolympus-skills-priority-"));
|
||||||
|
const skillDir = join(root, "default-priority");
|
||||||
|
mkdirSync(skillDir, { recursive: true });
|
||||||
|
writeFileSync(
|
||||||
|
join(skillDir, "SKILL.md"),
|
||||||
|
[
|
||||||
|
"---",
|
||||||
|
"name: default-priority",
|
||||||
|
"description: Uses default priority.",
|
||||||
|
"version: 1",
|
||||||
|
"trustTier: T2",
|
||||||
|
"---",
|
||||||
|
"",
|
||||||
|
"body",
|
||||||
|
].join("\n"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(SkillRegistry.load(root).get("default-priority")?.priority).toBe(50);
|
||||||
|
});
|
||||||
|
|
||||||
it("ignores invalid frontmatter", () => {
|
it("ignores invalid frontmatter", () => {
|
||||||
const root = mkdtempSync(join(tmpdir(), "lolympus-skills-invalid-"));
|
const root = mkdtempSync(join(tmpdir(), "lolympus-skills-invalid-"));
|
||||||
const skillDir = join(root, "bad");
|
const skillDir = join(root, "bad");
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||||
import { dirname, resolve } from "node:path";
|
import { dirname, resolve } from "node:path";
|
||||||
|
import { getLogger } from "@obs/logger";
|
||||||
|
|
||||||
export type TrustTier = "T1" | "T2" | "T3" | "T4";
|
export type TrustTier = "T1" | "T2" | "T3" | "T4";
|
||||||
|
|
||||||
@@ -8,6 +9,7 @@ export interface SkillManifest {
|
|||||||
description: string;
|
description: string;
|
||||||
intents: string[];
|
intents: string[];
|
||||||
name: string;
|
name: string;
|
||||||
|
priority: number;
|
||||||
resources: Record<string, string>;
|
resources: Record<string, string>;
|
||||||
tools: string[];
|
tools: string[];
|
||||||
triggers: string[];
|
triggers: string[];
|
||||||
@@ -123,6 +125,15 @@ function isTrustTier(value: unknown): value is TrustTier {
|
|||||||
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
|
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isPriority(value: unknown): value is number {
|
||||||
|
return (
|
||||||
|
typeof value === "number" &&
|
||||||
|
Number.isInteger(value) &&
|
||||||
|
value >= 0 &&
|
||||||
|
value <= 100
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function parseSkillFile(path: string): ParsedSkillFile | null {
|
function parseSkillFile(path: string): ParsedSkillFile | null {
|
||||||
const raw = readFileSync(path, "utf8");
|
const raw = readFileSync(path, "utf8");
|
||||||
const parsed = parseFrontmatter(raw);
|
const parsed = parseFrontmatter(raw);
|
||||||
@@ -133,6 +144,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
|||||||
const skillDir = dirname(path);
|
const skillDir = dirname(path);
|
||||||
const name = parsed.fields["name"];
|
const name = parsed.fields["name"];
|
||||||
const description = parsed.fields["description"];
|
const description = parsed.fields["description"];
|
||||||
|
const priority = parsed.fields["priority"] ?? 50;
|
||||||
const version = parsed.fields["version"];
|
const version = parsed.fields["version"];
|
||||||
const trustTier = parsed.fields["trustTier"];
|
const trustTier = parsed.fields["trustTier"];
|
||||||
|
|
||||||
@@ -141,6 +153,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
|||||||
!name.trim() ||
|
!name.trim() ||
|
||||||
typeof description !== "string" ||
|
typeof description !== "string" ||
|
||||||
!description.trim() ||
|
!description.trim() ||
|
||||||
|
!isPriority(priority) ||
|
||||||
typeof version !== "number" ||
|
typeof version !== "number" ||
|
||||||
!isTrustTier(trustTier)
|
!isTrustTier(trustTier)
|
||||||
) {
|
) {
|
||||||
@@ -154,6 +167,7 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
|||||||
description,
|
description,
|
||||||
intents: readStringArray(parsed.fields, "intents"),
|
intents: readStringArray(parsed.fields, "intents"),
|
||||||
name,
|
name,
|
||||||
|
priority,
|
||||||
resources: readResources(parsed.fields, skillDir),
|
resources: readResources(parsed.fields, skillDir),
|
||||||
tools: readStringArray(parsed.fields, "tools"),
|
tools: readStringArray(parsed.fields, "tools"),
|
||||||
triggers: readStringArray(parsed.fields, "triggers"),
|
triggers: readStringArray(parsed.fields, "triggers"),
|
||||||
@@ -200,6 +214,13 @@ export class SkillRegistry {
|
|||||||
|
|
||||||
for (const item of parsed) {
|
for (const item of parsed) {
|
||||||
if (seen.has(item.manifest.name)) {
|
if (seen.has(item.manifest.name)) {
|
||||||
|
getLogger().warn(
|
||||||
|
{
|
||||||
|
path: item.manifest.bodyPath,
|
||||||
|
skill: item.manifest.name,
|
||||||
|
},
|
||||||
|
"[SkillRegistry] Duplicate skill manifest ignored",
|
||||||
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
seen.add(item.manifest.name);
|
seen.add(item.manifest.name);
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import type {
|
|||||||
|
|
||||||
export interface Skill {
|
export interface Skill {
|
||||||
readonly name: string;
|
readonly name: string;
|
||||||
|
readonly priority: number;
|
||||||
canHandle(request: AgentRequest): boolean;
|
canHandle(request: AgentRequest): boolean;
|
||||||
handle(
|
handle(
|
||||||
request: AgentRequest,
|
request: AgentRequest,
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { lookupRuleTool } from "./lookup-rule.js";
|
||||||
|
|
||||||
|
describe("lookupRuleTool", () => {
|
||||||
|
it("returns the matching community rule section", async () => {
|
||||||
|
const result = await lookupRuleTool.execute({ topic: "ads" }, {});
|
||||||
|
|
||||||
|
expect(result.source).toBe("community-rules");
|
||||||
|
expect(result.text).toContain("광고");
|
||||||
|
expect(result.text).toContain("홍보");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("validates topic enum values", () => {
|
||||||
|
expect(() =>
|
||||||
|
lookupRuleTool.argsSchema.parse({ topic: "unknown" }),
|
||||||
|
).toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { z } from "zod";
|
||||||
|
import type { Tool } from "../tool.js";
|
||||||
|
|
||||||
|
const argsSchema = z.object({
|
||||||
|
topic: z.enum([
|
||||||
|
"ads",
|
||||||
|
"appeal",
|
||||||
|
"minor-safety",
|
||||||
|
"off-topic",
|
||||||
|
"privacy",
|
||||||
|
"profanity",
|
||||||
|
"report",
|
||||||
|
"sanctions",
|
||||||
|
"spam",
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
|
||||||
|
const resultSchema = z.object({
|
||||||
|
topic: argsSchema.shape.topic,
|
||||||
|
text: z.string(),
|
||||||
|
source: z.literal("community-rules"),
|
||||||
|
});
|
||||||
|
|
||||||
|
type RuleTopic = z.infer<typeof argsSchema>["topic"];
|
||||||
|
|
||||||
|
const RULES_PATH = fileURLToPath(
|
||||||
|
new URL("../../skills/moderation/resources/community-rules.md", import.meta.url),
|
||||||
|
);
|
||||||
|
const RULES_TEXT = readFileSync(RULES_PATH, "utf8");
|
||||||
|
|
||||||
|
function extractRuleSection(topic: RuleTopic): string {
|
||||||
|
const pattern = new RegExp(`^## ${topic}\\n([\\s\\S]*?)(?=\\n## |$)`, "mu");
|
||||||
|
const match = RULES_TEXT.match(pattern);
|
||||||
|
return match?.[1]?.trim() || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
export const lookupRuleTool: Tool<
|
||||||
|
z.infer<typeof argsSchema>,
|
||||||
|
z.infer<typeof resultSchema>
|
||||||
|
> = {
|
||||||
|
name: "lookup-rule",
|
||||||
|
description:
|
||||||
|
"광고, 도배, 욕설, 신고, 제재, 이의 제기, 개인정보 같은 moderation 규칙을 조회한다.",
|
||||||
|
argsSchema,
|
||||||
|
resultSchema,
|
||||||
|
execute(args) {
|
||||||
|
return Promise.resolve({
|
||||||
|
topic: args.topic,
|
||||||
|
text: extractRuleSection(args.topic),
|
||||||
|
source: "community-rules",
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
Reference in new issue
Block a user